From 51ec837e61bb2ac2b710bc6f617854de1941591b Mon Sep 17 00:00:00 2001 From: Nick Parker Date: Mon, 5 Dec 2016 14:29:15 -0600 Subject: [PATCH] Snapshot of upstream SQLite 3.15.2 --- Makefile.in | 69 +- Makefile.msc | 303 +- VERSION | 2 +- autoconf/Makefile.am | 9 +- autoconf/Makefile.msc | 106 +- autoconf/README.txt | 4 +- autoconf/configure.ac | 93 +- autoconf/tea/configure.ac | 1 - config.h.in | 12 + configure | 120 +- configure.ac | 50 +- doc/lemon.html | 291 +- ext/fts2/fts2_tokenizer.c | 6 +- ext/fts3/fts3_test.c | 22 +- ext/fts3/fts3_tokenizer.c | 77 +- ext/fts3/fts3_write.c | 10 +- ext/fts3/tool/fts3view.c | 2 +- ext/fts5/fts5.h | 15 +- ext/fts5/fts5Int.h | 15 +- ext/fts5/fts5_aux.c | 232 +- ext/fts5/fts5_config.c | 18 +- ext/fts5/fts5_expr.c | 263 +- ext/fts5/fts5_hash.c | 4 +- ext/fts5/fts5_index.c | 363 +- ext/fts5/fts5_main.c | 57 +- ext/fts5/fts5_storage.c | 15 +- ext/fts5/fts5_tcl.c | 33 +- ext/fts5/fts5_test_mi.c | 39 +- ext/fts5/fts5_vocab.c | 13 +- ext/fts5/fts5parse.y | 11 +- ext/fts5/test/fts5_common.tcl | 19 +- ext/fts5/test/fts5aa.test | 9 +- ext/fts5/test/fts5af.test | 91 +- ext/fts5/test/fts5aux.test | 32 + ext/fts5/test/fts5bigtok.test | 1 + ext/fts5/test/fts5colset.test | 59 + ext/fts5/test/fts5config.test | 16 + ext/fts5/test/fts5corrupt2.test | 6 +- ext/fts5/test/fts5corrupt3.test | 4 + ext/fts5/test/fts5determin.test | 67 + ext/fts5/test/fts5dlidx.test | 4 +- ext/fts5/test/fts5eb.test | 29 +- ext/fts5/test/fts5fault4.test | 2 +- ext/fts5/test/fts5fault8.test | 25 + ext/fts5/test/fts5faultB.test | 83 + ext/fts5/test/fts5fuzz1.test | 93 + ext/fts5/test/fts5matchinfo.test | 24 + ext/fts5/test/fts5merge.test | 68 +- ext/fts5/test/fts5merge2.test | 5 +- ext/fts5/test/fts5multiclient.test | 48 + ext/fts5/test/fts5optimize.test | 47 +- ext/fts5/test/fts5rank.test | 54 + ext/fts5/test/fts5simple.test | 68 +- ext/fts5/test/fts5simple3.test | 34 + ext/fts5/test/fts5unicode2.test | 8 +- ext/fts5/test/fts5vocab.test | 38 +- ext/fts5/tool/fts5txt2db.tcl | 5 +- ext/icu/icu.c | 142 +- ext/misc/amatch.c | 8 +- ext/misc/carray.c | 364 ++ ext/misc/csv.c | 869 +++ ext/misc/fuzzer.c | 4 +- ext/misc/json1.c | 55 +- ext/misc/memvfs.c | 491 ++ ext/misc/percentile.c | 2 +- ext/misc/regexp.c | 10 +- ext/misc/scrub.c | 610 +++ ext/misc/series.c | 16 +- ext/misc/spellfix.c | 34 +- ext/misc/vfsstat.c | 816 +++ ext/rbu/rbu.c | 56 +- ext/rbu/rbu1.test | 46 +- ext/rbu/rbu5.test | 27 +- ext/rbu/rbuC.test | 142 + ext/rbu/rbu_common.tcl | 54 + ext/rbu/rbudiff.test | 177 +- ext/rbu/rbudor.test | 59 + ext/rbu/rbufault3.test | 98 + ext/rbu/rbuprogress.test | 417 ++ ext/rbu/rbuvacuum.test | 392 ++ ext/rbu/rbuvacuum2.test | 204 + ext/rbu/sqlite3rbu.c | 1225 ++++- ext/rbu/sqlite3rbu.h | 120 +- ext/rbu/test_rbu.c | 101 +- ext/rtree/rtree.c | 64 +- ext/rtree/rtree1.test | 20 +- ext/rtree/rtree3.test | 31 +- ext/rtree/rtreeC.test | 4 - ext/rtree/rtreeG.test | 66 + ext/session/changeset.c | 416 ++ ext/session/session1.test | 585 +++ ext/session/session2.test | 591 +++ ext/session/session3.test | 211 + ext/session/session4.test | 67 + ext/session/session5.test | 408 ++ ext/session/session6.test | 90 + ext/session/session8.test | 91 + ext/session/session9.test | 287 + ext/session/sessionA.test | 106 + ext/session/sessionB.test | 508 ++ ext/session/sessionC.test | 198 + ext/session/sessionD.test | 225 + ext/session/sessionE.test | 117 + ext/session/sessionF.test | 295 ++ ext/session/sessionG.test | 177 + ext/session/session_common.tcl | 170 + ext/session/sessionfault.test | 589 +++ ext/session/sessionfault2.test | 107 + ext/session/sqlite3session.c | 4641 +++++++++++++++++ ext/session/sqlite3session.h | 1281 +++++ ext/session/test_session.c | 1046 ++++ main.mk | 47 +- manifest | 844 +-- manifest.uuid | 2 +- src/alter.c | 40 +- src/analyze.c | 62 +- src/attach.c | 27 +- src/auth.c | 7 +- src/backup.c | 57 +- src/bitvec.c | 4 +- src/btree.c | 354 +- src/btree.h | 52 +- src/build.c | 415 +- src/callback.c | 70 +- src/complete.c | 2 +- src/ctime.c | 11 +- src/date.c | 22 +- src/dbstat.c | 30 +- src/delete.c | 56 +- src/expr.c | 1813 +++++-- src/fkey.c | 12 +- src/func.c | 127 +- src/global.c | 34 +- src/hash.c | 10 +- src/hash.h | 6 +- src/hwtime.h | 6 +- src/in-operator.md | 107 + src/insert.c | 57 +- src/journal.c | 256 - src/legacy.c | 2 +- src/loadext.c | 77 +- src/main.c | 219 +- src/malloc.c | 2 +- src/memjournal.c | 327 +- src/msvc.h | 6 +- src/os.c | 27 +- src/os.h | 5 +- src/os_setup.h | 6 +- src/os_unix.c | 166 +- src/os_win.c | 752 ++- src/os_win.h | 6 +- src/pager.c | 315 +- src/pager.h | 23 +- src/parse.y | 609 ++- src/pcache.c | 242 +- src/pcache.h | 18 +- src/pcache1.c | 52 +- src/pragma.c | 76 +- src/prepare.c | 80 +- src/printf.c | 36 +- src/resolve.c | 79 +- src/rowset.c | 140 +- src/select.c | 268 +- src/shell.c | 1332 ++++- src/sqlite.h.in | 487 +- src/sqlite3ext.h | 28 +- src/sqliteInt.h | 674 ++- src/sqliteLimit.h | 13 +- src/status.c | 9 +- src/table.c | 6 +- src/tclsqlite.c | 669 ++- src/test1.c | 738 ++- src/test2.c | 46 +- src/test3.c | 90 +- src/test4.c | 36 +- src/test5.c | 14 +- src/test6.c | 70 +- src/test7.c | 32 +- src/test8.c | 10 +- src/test9.c | 12 +- src/test_async.c | 19 +- src/test_autoext.c | 35 +- src/test_backup.c | 15 +- src/test_bestindex.c | 610 +++ src/test_blob.c | 18 +- src/test_btree.c | 8 +- src/test_config.c | 43 +- src/test_delete.c | 138 + src/test_demovfs.c | 13 +- src/test_devsym.c | 10 +- src/test_fs.c | 20 +- src/test_func.c | 30 +- src/test_hexio.c | 20 +- src/test_init.c | 14 +- src/test_intarray.c | 13 +- src/test_intarray.h | 6 +- src/test_journal.c | 30 +- src/test_malloc.c | 75 +- src/test_multiplex.c | 29 +- src/test_multiplex.h | 6 +- src/test_mutex.c | 28 +- src/test_onefile.c | 2 +- src/test_osinst.c | 17 +- src/test_quota.c | 79 +- src/test_rtree.c | 16 +- src/test_schema.c | 12 +- src/test_superlock.c | 15 +- src/test_syscall.c | 53 +- src/test_tclvar.c | 8 +- src/test_thread.c | 28 +- src/test_vfs.c | 17 +- src/test_windirent.c | 30 +- src/test_windirent.h | 7 + src/threads.c | 14 +- src/tokenize.c | 50 +- src/treeview.c | 44 +- src/trigger.c | 20 +- src/update.c | 30 +- src/utf.c | 4 +- src/util.c | 245 +- src/vacuum.c | 196 +- src/vdbe.c | 1111 ++-- src/vdbe.h | 14 +- src/vdbeInt.h | 120 +- src/vdbeapi.c | 262 +- src/vdbeaux.c | 547 +- src/vdbeblob.c | 28 + src/vdbemem.c | 164 +- src/vdbesort.c | 77 +- src/vdbetrace.c | 12 +- src/vtab.c | 96 +- src/wal.c | 42 +- src/wal.h | 6 +- src/walker.c | 20 +- src/where.c | 950 ++-- src/whereInt.h | 52 +- src/wherecode.c | 580 +- src/whereexpr.c | 124 +- test/alter3.test | 12 +- test/alter4.test | 19 + test/analyze9.test | 2 +- test/analyzeD.test | 5 +- test/analyzer1.test | 9 +- test/auth.test | 49 +- test/auth2.test | 12 - test/auth3.test | 21 +- test/autoinc.test | 12 + test/autoindex2.test | 7 +- test/autovacuum.test | 17 +- test/backcompat.test | 13 +- test/backup.test | 21 +- test/backup4.test | 5 + test/backup_malloc.test | 32 + test/bc_common.tcl | 5 +- test/bestindex1.test | 275 + test/bestindex2.test | 141 + test/bestindex3.test | 175 + test/capi3.test | 30 +- test/capi3c.test | 21 +- test/cffault.test | 2 +- test/close.test | 4 + test/collateB.test | 63 + test/conflict.test | 12 +- test/conflict2.test | 23 +- test/corrupt2.test | 44 +- test/corrupt3.test | 7 +- test/corrupt4.test | 7 +- test/corrupt6.test | 7 +- test/corrupt7.test | 7 +- test/corruptE.test | 7 +- test/corruptG.test | 7 +- test/corruptH.test | 8 +- test/corruptI.test | 8 +- test/corruptJ.test | 8 +- test/coveridxscan.test | 31 + test/crash8.test | 1 + test/csv01.test | 112 + test/ctime.test | 5 +- test/cursorhint2.test | 182 + test/dbstatus.test | 48 +- test/dbstatus2.test | 2 +- test/delete.test | 14 + test/delete4.test | 42 + test/delete_db.test | 217 + test/e_expr.test | 7 +- test/e_fkey.test | 8 +- test/e_uri.test | 2 +- test/e_vacuum.test | 79 +- test/e_walauto.test | 5 + test/e_walhook.test | 6 +- test/eqp.test | 66 +- test/exclusive.test | 9 +- test/exists.test | 1 + test/expr.test | 27 + test/filectrl.test | 26 + test/filefmt.test | 16 +- test/fkey6.test | 56 +- test/fts3atoken.test | 83 +- test/fts3aux1.test | 6 +- test/fts3conf.test | 38 + test/fts3expr.test | 4 + test/fts4growth.test | 40 +- test/fts4langid.test | 49 +- test/fts4merge.test | 26 +- test/fts4merge3.test | 6 +- test/fts4opt.test | 213 + test/fuzz-oss1.test | 4 + test/fuzzcheck.c | 52 +- test/hook.test | 410 ++ test/icu.test | 9 + test/in.test | 26 +- test/in5.test | 54 + test/incrblob.test | 23 +- test/incrblob_err.test | 4 +- test/incrvacuum2.test | 2 +- test/index6.test | 18 +- test/index7.test | 17 +- test/index8.test | 60 + test/indexexpr1.test | 46 + test/instrfault.test | 68 + test/intpkey.test | 39 +- test/io.test | 2 +- test/journal2.test | 2 +- test/json101.test | 332 ++ test/limit2.test | 153 + test/loadext.test | 14 +- test/lock.test | 5 +- test/main.test | 2 +- test/mallocI.test | 11 + test/memsubsys1.test | 12 +- test/minmax.test | 14 + test/misc1.test | 2 +- test/misc5.test | 2 + test/misc8.test | 39 + test/mmap1.test | 6 +- test/mmap3.test | 3 + test/mutex1.test | 2 +- test/nan.test | 80 +- test/nolock.test | 35 + test/pager1.test | 177 +- test/pagerfault.test | 30 +- test/pageropt.test | 5 + test/parser1.test | 23 + test/permutations.test | 58 +- test/pragma.test | 145 +- test/pragma3.test | 3 +- test/quota.test | 2 + test/regexp2.test | 125 + test/releasetest.tcl | 169 +- test/rowvalue.test | 296 ++ test/rowvalue2.test | 279 + test/rowvalue3.test | 220 + test/rowvalue4.test | 313 ++ test/rowvalue5.test | 115 + test/rowvalue6.test | 36 + test/rowvalue7.test | 58 + test/rowvalue8.test | 59 + test/rowvalue9.test | 302 ++ test/rowvaluefault.test | 72 + test/savepoint7.test | 34 + test/select4.test | 73 +- test/select7.test | 12 +- test/selectA.test | 9 + test/session.test | 22 + test/shell1.test | 104 +- test/shell2.test | 12 +- test/shell3.test | 22 +- test/shell4.test | 31 +- test/shell5.test | 10 +- test/skipscan1.test | 31 +- test/snapshot.test | 99 + test/snapshot_fault.test | 2 +- test/sort5.test | 74 + test/speed3.test | 8 +- test/speedtest1.c | 98 +- test/spellfix3.test | 11 +- test/sqldiff1.test | 12 +- test/stat.test | 22 +- test/stmt.test | 10 +- test/subselect.test | 2 +- test/superlock.test | 25 +- test/sync.test | 2 + test/tabfunc01.test | 73 + test/tclsqlite.test | 40 +- test/tempdb.test | 2 +- test/tempdb2.test | 77 + test/tempfault.test | 134 + test/temptable2.test | 357 ++ test/temptable3.test | 41 + test/tester.tcl | 155 +- test/time-wordcount.sh | 34 + test/tkt-2d1a5c67d.test | 3 +- test/tkt-313723c356.test | 2 +- test/tkt-5d863f876e.test | 2 +- test/tkt-9d68c883.test | 2 + test/tkt4018.test | 1 + test/trace3.test | 233 + test/types2.test | 2 +- test/unixexcl.test | 1 + test/vacuum5.test | 153 + test/vacuummem.test | 73 + test/view.test | 20 + test/vtab6.test | 4 +- test/vtabF.test | 4 +- test/vtabH.test | 28 +- test/wal.test | 2 + test/wal2.test | 5 +- test/wal3.test | 1 + test/wal5.test | 7 +- test/wal8.test | 1 + test/walbak.test | 2 + test/walcrash4.test | 85 + test/walprotocol.test | 2 +- test/walro.test | 2 +- test/where.test | 11 +- test/where2.test | 11 + test/where3.test | 4 +- test/whereD.test | 77 + test/win32nolock.test | 126 + test/wordcount.c | 54 +- test/zerodamage.test | 3 +- tool/GetTclKit.bat | 10 +- tool/Replace.cs | 223 + tool/addopcodes.tcl | 3 + tool/build-all-msvc.bat | 133 +- tool/dbhash.c | 506 ++ tool/fuzzershell.c | 321 ++ tool/lemon.c | 386 +- tool/lempar.c | 318 +- tool/libvers.c | 15 + tool/logest.c | 2 +- tool/mkautoconfamal.sh | 1 + tool/mkmsvcmin.tcl | 18 +- tool/mkopcodeh.tcl | 35 +- tool/mksqlite3c-noext.tcl | 59 +- tool/mksqlite3c.tcl | 34 +- tool/mksqlite3h.tcl | 44 +- tool/mkvsix.tcl | 4 +- tool/opcodesum.tcl | 34 + tool/replace.tcl | 5 +- tool/run-speed-test.sh | 36 +- tool/showdb.c | 4 +- tool/spaceanal.tcl | 78 +- tool/speed-check.sh | 122 + tool/sqldiff.c | 193 +- tool/srcck1.c | 2 +- tool/symbols.sh | 2 +- tool/warnings.sh | 23 +- vsixtest/App.xaml | 8 + vsixtest/App.xaml.cpp | 120 + vsixtest/App.xaml.h | 27 + vsixtest/Assets/LockScreenLogo.scale-200.png | Bin 0 -> 1430 bytes vsixtest/Assets/SplashScreen.scale-200.png | Bin 0 -> 7700 bytes .../Assets/Square150x150Logo.scale-200.png | Bin 0 -> 2937 bytes vsixtest/Assets/Square44x44Logo.scale-200.png | Bin 0 -> 1647 bytes ...x44Logo.targetsize-24_altform-unplated.png | Bin 0 -> 1255 bytes vsixtest/Assets/StoreLogo.png | Bin 0 -> 1451 bytes vsixtest/Assets/Wide310x150Logo.scale-200.png | Bin 0 -> 3204 bytes vsixtest/MainPage.xaml | 13 + vsixtest/MainPage.xaml.cpp | 53 + vsixtest/MainPage.xaml.h | 22 + vsixtest/Package.appxmanifest | 49 + vsixtest/pch.cpp | 6 + vsixtest/pch.h | 11 + vsixtest/vsixtest.sln | 39 + vsixtest/vsixtest.tcl | 373 ++ vsixtest/vsixtest.vcxproj.data | 198 + vsixtest/vsixtest.vcxproj.filters | 57 + vsixtest/vsixtest_TemporaryKey.pfx | Bin 0 -> 2520 bytes 469 files changed, 45135 insertions(+), 8295 deletions(-) create mode 100644 ext/fts5/test/fts5colset.test create mode 100644 ext/fts5/test/fts5determin.test create mode 100644 ext/fts5/test/fts5faultB.test create mode 100644 ext/fts5/test/fts5fuzz1.test create mode 100644 ext/fts5/test/fts5multiclient.test create mode 100644 ext/misc/carray.c create mode 100644 ext/misc/csv.c create mode 100644 ext/misc/memvfs.c create mode 100644 ext/misc/scrub.c create mode 100644 ext/misc/vfsstat.c create mode 100644 ext/rbu/rbuC.test create mode 100644 ext/rbu/rbudor.test create mode 100644 ext/rbu/rbufault3.test create mode 100644 ext/rbu/rbuprogress.test create mode 100644 ext/rbu/rbuvacuum.test create mode 100644 ext/rbu/rbuvacuum2.test create mode 100644 ext/rtree/rtreeG.test create mode 100644 ext/session/changeset.c create mode 100644 ext/session/session1.test create mode 100644 ext/session/session2.test create mode 100644 ext/session/session3.test create mode 100644 ext/session/session4.test create mode 100644 ext/session/session5.test create mode 100644 ext/session/session6.test create mode 100644 ext/session/session8.test create mode 100644 ext/session/session9.test create mode 100644 ext/session/sessionA.test create mode 100644 ext/session/sessionB.test create mode 100644 ext/session/sessionC.test create mode 100644 ext/session/sessionD.test create mode 100644 ext/session/sessionE.test create mode 100644 ext/session/sessionF.test create mode 100644 ext/session/sessionG.test create mode 100644 ext/session/session_common.tcl create mode 100644 ext/session/sessionfault.test create mode 100644 ext/session/sessionfault2.test create mode 100644 ext/session/sqlite3session.c create mode 100644 ext/session/sqlite3session.h create mode 100644 ext/session/test_session.c create mode 100644 src/in-operator.md delete mode 100644 src/journal.c create mode 100644 src/test_bestindex.c create mode 100644 src/test_delete.c create mode 100644 test/bestindex1.test create mode 100644 test/bestindex2.test create mode 100644 test/bestindex3.test create mode 100644 test/collateB.test create mode 100644 test/csv01.test create mode 100644 test/cursorhint2.test create mode 100644 test/delete_db.test create mode 100644 test/fts4opt.test create mode 100644 test/index8.test create mode 100644 test/instrfault.test create mode 100644 test/limit2.test create mode 100644 test/regexp2.test mode change 100644 => 100755 test/releasetest.tcl create mode 100644 test/rowvalue.test create mode 100644 test/rowvalue2.test create mode 100644 test/rowvalue3.test create mode 100644 test/rowvalue4.test create mode 100644 test/rowvalue5.test create mode 100644 test/rowvalue6.test create mode 100644 test/rowvalue7.test create mode 100644 test/rowvalue8.test create mode 100644 test/rowvalue9.test create mode 100644 test/rowvaluefault.test create mode 100644 test/session.test create mode 100644 test/tempdb2.test create mode 100644 test/tempfault.test create mode 100644 test/temptable2.test create mode 100644 test/temptable3.test create mode 100644 test/time-wordcount.sh create mode 100644 test/trace3.test create mode 100644 test/vacuum5.test create mode 100644 test/vacuummem.test create mode 100644 test/walcrash4.test create mode 100644 test/win32nolock.test create mode 100644 tool/Replace.cs create mode 100644 tool/dbhash.c create mode 100644 tool/libvers.c create mode 100644 tool/opcodesum.tcl create mode 100644 tool/speed-check.sh create mode 100644 vsixtest/App.xaml create mode 100644 vsixtest/App.xaml.cpp create mode 100644 vsixtest/App.xaml.h create mode 100644 vsixtest/Assets/LockScreenLogo.scale-200.png create mode 100644 vsixtest/Assets/SplashScreen.scale-200.png create mode 100644 vsixtest/Assets/Square150x150Logo.scale-200.png create mode 100644 vsixtest/Assets/Square44x44Logo.scale-200.png create mode 100644 vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png create mode 100644 vsixtest/Assets/StoreLogo.png create mode 100644 vsixtest/Assets/Wide310x150Logo.scale-200.png create mode 100644 vsixtest/MainPage.xaml create mode 100644 vsixtest/MainPage.xaml.cpp create mode 100644 vsixtest/MainPage.xaml.h create mode 100644 vsixtest/Package.appxmanifest create mode 100644 vsixtest/pch.cpp create mode 100644 vsixtest/pch.h create mode 100644 vsixtest/vsixtest.sln create mode 100644 vsixtest/vsixtest.tcl create mode 100644 vsixtest/vsixtest.vcxproj.data create mode 100644 vsixtest/vsixtest.vcxproj.filters create mode 100644 vsixtest/vsixtest_TemporaryKey.pfx diff --git a/Makefile.in b/Makefile.in index 013cf442..bb1f14b6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -30,7 +30,8 @@ BCC = @BUILD_CC@ @BUILD_CFLAGS@ # CC = @CC@ CFLAGS = @CPPFLAGS@ @CFLAGS@ -TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3 +TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu +TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session # Define this for the autoconf-based build, so that the code knows it can # include the generated config.h @@ -173,13 +174,14 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ - icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \ + icu.lo insert.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ + random.lo resolve.lo rowset.lo rtree.lo \ + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -224,7 +226,6 @@ SRC = \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ - $(TOP)/src/journal.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ $(TOP)/src/main.c \ @@ -342,14 +343,15 @@ SRC += \ SRC += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c +SRC += \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h \ $(TOP)/ext/rbu/sqlite3rbu.c SRC += \ $(TOP)/ext/misc/json1.c - - # Generated source code files # SRC += \ @@ -376,9 +378,11 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_bestindex.c \ $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ + $(TOP)/src/test_delete.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ @@ -405,14 +409,17 @@ TESTSRC = \ $(TOP)/src/test_windirent.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ - $(TOP)/ext/fts3/fts3_test.c \ + $(TOP)/ext/fts3/fts3_test.c \ + $(TOP)/ext/session/test_session.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/csv.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ @@ -474,7 +481,8 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c + $(TOP)/ext/async/sqlite3async.c \ + $(TOP)/ext/session/sqlite3session.c # Header files used by all library source files. # @@ -533,7 +541,8 @@ TESTPROGS = \ testfixture$(TEXE) \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ - sqldiff$(TEXE) + sqldiff$(TEXE) \ + dbhash$(TEXE) # Databases containing fuzzer test cases # @@ -550,7 +559,9 @@ TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 +# SHELL_OPT += -DSQLITE_ENABLE_FTS5 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 @@ -581,8 +592,15 @@ sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c $(TOP)/src/shell.c sqlite3.c \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" -sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) +sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS) + +dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) + +scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo + $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ + $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) srcck1$(BEXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c @@ -597,8 +615,8 @@ fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) -mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c - $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ +mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c + $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \ $(TLIBS) -rpath "$(libdir)" MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 @@ -634,6 +652,7 @@ mptest: mptester$(TEXE) sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . + cp $(TOP)/ext/session/sqlite3session.h . sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . @@ -738,9 +757,6 @@ hash.lo: $(TOP)/src/hash.c $(HDR) insert.lo: $(TOP)/src/insert.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c -journal.lo: $(TOP)/src/journal.c $(HDR) - $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/journal.c - legacy.lo: $(TOP)/src/legacy.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/legacy.c @@ -1000,6 +1016,9 @@ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c +sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c + json1.lo: $(TOP)/ext/misc/json1.c $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c @@ -1025,7 +1044,7 @@ FTS5_SRC = \ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h - ./lemon $(OPTS) fts5parse.y + ./lemon$(BEXE) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c @@ -1050,6 +1069,8 @@ sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE TESTFIXTURE_FLAGS += -DBUILD_sqlite +TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 +TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c @@ -1130,14 +1151,17 @@ showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) +changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) + rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c -wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c - $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) +wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) @@ -1152,8 +1176,9 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # +VALIDIDS=' sqlite3(changeset|changegroup|session)?_' checksymbols: sqlite3.lo - nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. The amalamgation-tarball target builds @@ -1216,13 +1241,12 @@ clean: rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz rm -f mkkeywordhash$(BEXE) keywordhash.h rm -f *.da *.bb *.bbg gmon.out - rm -rf quota2a quota2b quota2c rm -rf tsrc .target_source rm -f tclsqlite3$(TEXE) rm -f testfixture$(TEXE) test.db rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) - rm -f wordcount$(TEXE) + rm -f wordcount$(TEXE) changeset$(TEXE) rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3rc.h @@ -1235,6 +1259,7 @@ clean: rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe + rm -f dbhash dbhash.exe rm -f fts5.* fts5parse.* distclean: clean diff --git a/Makefile.msc b/Makefile.msc index 843f9a25..0d42d41a 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -24,6 +24,20 @@ USE_AMALGAMATION = 1 USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable full runtime error checks (-RTC1, etc). This +# has no effect if (any) optimizations are enabled. +# +!IFNDEF USE_RUNTIME_CHECKS +USE_RUNTIME_CHECKS = 0 +!ENDIF + +# Set this non-0 to create a SQLite amalgamation file that excludes the +# various built-in extensions. +# +!IFNDEF MINIMAL_AMALGAMATION +MINIMAL_AMALGAMATION = 0 +!ENDIF + # Set this non-0 to use "stdcall" calling convention for the core library # and shell executable. # @@ -198,6 +212,12 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# Set this to non-0 to enable support for the session extension. +# +!IFNDEF SESSION +SESSION = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -219,37 +239,88 @@ SQLITE3H = sqlite3.h # This is the name to use for the SQLite dynamic link library (DLL). # !IFNDEF SQLITE3DLL +!IF $(FOR_WIN10)!=0 +SQLITE3DLL = winsqlite3.dll +!ELSE SQLITE3DLL = sqlite3.dll !ENDIF +!ENDIF # This is the name to use for the SQLite import library (LIB). # !IFNDEF SQLITE3LIB +!IF $(FOR_WIN10)!=0 +SQLITE3LIB = winsqlite3.lib +!ELSE SQLITE3LIB = sqlite3.lib !ENDIF +!ENDIF # This is the name to use for the SQLite shell executable (EXE). # !IFNDEF SQLITE3EXE +!IF $(FOR_WIN10)!=0 +SQLITE3EXE = winsqlite3shell.exe +!ELSE SQLITE3EXE = sqlite3.exe !ENDIF +!ENDIF # This is the argument used to set the program database (PDB) file for the # SQLite shell executable (EXE). # !IFNDEF SQLITE3EXEPDB +!IF $(FOR_WIN10)!=0 +SQLITE3EXEPDB = +!ELSE SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF +!ENDIF + +# <> +# These are the names of the customized Tcl header files used by various parts +# of this makefile when the stdcall calling convention is in use. It is not +# used for any other purpose. +# +!IFNDEF SQLITETCLH +SQLITETCLH = sqlite_tcl.h +!ENDIF + +!IFNDEF SQLITETCLDECLSH +SQLITETCLDECLSH = sqlite_tclDecls.h +!ENDIF + +# These are the additional targets that the targets that integrate with the +# Tcl library should depend on when compiling, etc. +# +!IFNDEF SQLITE_TCL_DEP +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +SQLITE_TCL_DEP = $(SQLITETCLDECLSH) $(SQLITETCLH) +!ELSE +SQLITE_TCL_DEP = +!ENDIF +!ENDIF +# <> # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +!IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +!ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Should the session extension be enabled? If so, add compilation options +# to enable it. +# +!IF $(SESSION)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 +!ENDIF + # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # @@ -290,6 +361,14 @@ PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) CC = cl.exe !ENDIF +# Check for the predefined command macro CSC. This should point to a working +# C Sharp compiler binary. If it is not defined, simply define it to the +# legacy default value 'csc.exe'. +# +!IFNDEF CSC +CSC = csc.exe +!ENDIF + # Check for the command macro LD. This should point to the linker binary for # the target platform. If it is not defined, simply define it to the legacy # default value 'link.exe'. @@ -417,15 +496,6 @@ TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) -# Adjust the names of the primary targets for use with Windows 10. -# -!IF $(FOR_WIN10)!=0 -SQLITE3DLL = winsqlite3.dll -SQLITE3LIB = winsqlite3.lib -SQLITE3EXE = winsqlite3shell.exe -SQLITE3EXEPDB = -!ENDIF - # Check if we want to use the "stdcall" calling convention when compiling. # This is not supported by the compilers for non-x86 platforms. It should # also be noted here that building any target with these "stdcall" options @@ -435,20 +505,32 @@ SQLITE3EXEPDB = # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +# <> +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +# <> !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +# <> +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +# <> !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = +# <> +TEST_CCONV_OPTS = +# <> !ENDIF !ENDIF !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = +# <> +TEST_CCONV_OPTS = +# <> !ENDIF # These are additional compiler options used for the core library. @@ -465,20 +547,24 @@ CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) # when linking. # !IFNDEF CORE_LINK_DEP -!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +!IF $(DYNAMIC_SHELL)!=0 CORE_LINK_DEP = -!ELSE +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" CORE_LINK_DEP = sqlite3.def +!ELSE +CORE_LINK_DEP = !ENDIF !ENDIF # These are additional linker options used for the core library. # !IFNDEF CORE_LINK_OPTS -!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +!IF $(DYNAMIC_SHELL)!=0 CORE_LINK_OPTS = -!ELSE +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" CORE_LINK_OPTS = /DEF:sqlite3.def +!ELSE +CORE_LINK_OPTS = !ENDIF !ENDIF @@ -588,6 +674,8 @@ TCC = $(TCC) -I$(TOP)\ext\fts3 RCC = $(RCC) -I$(TOP)\ext\fts3 TCC = $(TCC) -I$(TOP)\ext\rtree RCC = $(RCC) -I$(TOP)\ext\rtree +TCC = $(TCC) -I$(TOP)\ext\session +RCC = $(RCC) -I$(TOP)\ext\session !ENDIF # The mksqlite3c.tcl script accepts some options on the command @@ -595,12 +683,35 @@ RCC = $(RCC) -I$(TOP)\ext\rtree # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # +!IFNDEF MKSQLITE3C_TOOL +!IF $(MINIMAL_AMALGAMATION)!=0 +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c-noext.tcl +!ELSE +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c.tcl +!ENDIF +!ENDIF + !IFNDEF MKSQLITE3C_ARGS !IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = !ENDIF +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +MKSQLITE3C_ARGS = $(MKSQLITE3C_ARGS) --useapicall +!ENDIF +!ENDIF + +# The mksqlite3h.tcl script accepts some options on the command line. +# When compiling with stdcall support, some of these options are +# necessary. +# +!IFNDEF MKSQLITE3H_ARGS +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +MKSQLITE3H_ARGS = --useapicall +!ELSE +MKSQLITE3H_ARGS = +!ENDIF !ENDIF # <> @@ -683,11 +794,11 @@ TCLLIBDIR = c:\tcl\lib !ENDIF !IFNDEF LIBTCL -LIBTCL = tcl85.lib +LIBTCL = tcl86.lib !ENDIF !IFNDEF LIBTCLSTUB -LIBTCLSTUB = tclstub85.lib +LIBTCLSTUB = tclstub86.lib !ENDIF !IFNDEF LIBTCLPATH @@ -717,7 +828,7 @@ LIBICU = icuuc.lib icuin.lib # specific Tcl shell to use. # !IFNDEF TCLSH_CMD -TCLSH_CMD = tclsh85 +TCLSH_CMD = tclsh !ENDIF # <> @@ -800,6 +911,10 @@ RCC = $(RCC) -D_DEBUG !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od +!IF $(USE_RUNTIME_CHECKS)!=0 +TCC = $(TCC) -RTC1 +BCC = $(BCC) -RTC1 +!ENDIF !ELSEIF $(OPTIMIZATIONS)>=3 TCC = $(TCC) -Ox BCC = $(BCC) -Ox @@ -966,13 +1081,14 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ - icu.lo insert.lo journal.lo legacy.lo loadext.lo \ + icu.lo insert.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ + random.lo resolve.lo rowset.lo rtree.lo \ + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -1030,7 +1146,6 @@ SRC00 = \ $(TOP)\src\global.c \ $(TOP)\src\hash.c \ $(TOP)\src\insert.c \ - $(TOP)\src\journal.c \ $(TOP)\src\legacy.c \ $(TOP)\src\loadext.c \ $(TOP)\src\main.c \ @@ -1160,6 +1275,7 @@ SRC07 = \ $(TOP)\ext\fts3\fts3_write.c \ $(TOP)\ext\icu\icu.c \ $(TOP)\ext\rtree\rtree.c \ + $(TOP)\ext\session\sqlite3session.c \ $(TOP)\ext\rbu\sqlite3rbu.c \ $(TOP)\ext\misc\json1.c @@ -1182,7 +1298,8 @@ SRC09 = \ $(TOP)\ext\fts3\fts3_tokenizer.h \ $(TOP)\ext\icu\sqliteicu.h \ $(TOP)\ext\rtree\rtree.h \ - $(TOP)\ext\rbu\sqlite3rbu.h + $(TOP)\ext\rbu\sqlite3rbu.h \ + $(TOP)\ext\session\sqlite3session.h # Generated source code files # @@ -1198,6 +1315,16 @@ SRC11 = \ parse.h \ $(SQLITE3H) +# Generated Tcl header files +# +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +SRC12 = \ + $(SQLITETCLH) \ + $(SQLITETCLDECLSH) +!ELSE +SRC12 = +!ENDIF + # All source code files. # SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11) @@ -1217,9 +1344,11 @@ TESTSRC = \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ + $(TOP)\src\test_bestindex.c \ $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ + $(TOP)\src\test_delete.c \ $(TOP)\src\test_demovfs.c \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ @@ -1247,13 +1376,16 @@ TESTSRC = \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ - $(TOP)\ext\rbu\test_rbu.c + $(TOP)\ext\rbu\test_rbu.c \ + $(TOP)\ext\session\test_session.c # Statically linked extensions. # TESTEXT = \ $(TOP)\ext\misc\amatch.c \ + $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\closure.c \ + $(TOP)\ext\misc\csv.c \ $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ @@ -1270,6 +1402,7 @@ TESTEXT = \ $(TOP)\ext\misc\wholenumber.c # Source code to the library files needed by the test fixture +# (non-amalgamation) # TESTSRC2 = \ $(SRC00) \ @@ -1299,7 +1432,7 @@ HDR = \ parse.h \ $(TOP)\src\pragma.h \ $(SQLITE3H) \ - $(TOP)\src\sqlite3ext.h \ + sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ @@ -1328,6 +1461,8 @@ EXTHDR = $(EXTHDR) \ $(TOP)\ext\icu\sqliteicu.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\sqlite3rtree.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\session\sqlite3session.h # executables needed for testing # @@ -1335,7 +1470,8 @@ TESTPROGS = \ testfixture.exe \ $(SQLITE3EXE) \ sqlite3_analyzer.exe \ - sqldiff.exe + sqldiff.exe \ + dbhash.exe # Databases containing fuzzer test cases # @@ -1380,30 +1516,30 @@ all: dll libsqlite3.lib shell $(ALL_TCL_TARGETS) # Dynamic link library section. # -dll: $(SQLITE3DLL) +dll: $(SQLITE3DLL) # Shell executable. # -shell: $(SQLITE3EXE) +shell: $(SQLITE3EXE) +# <> libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) -# <> libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) # <> -$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) +$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) -# <> -sqlite3.def: libsqlite3.lib +# <> +sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \ | sort >> sqlite3.def -# <> +# <> $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c $(SHELL_CORE_SRC) \ @@ -1413,6 +1549,12 @@ $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + srcck1.exe: $(TOP)\tool\srcck1.c $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c @@ -1448,7 +1590,7 @@ mptest: mptester.exe # files are automatically generated. This target takes care of # all that automatic generation. # -.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP) -rmdir /Q/S tsrc 2>NUL -mkdir tsrc for %i in ($(SRC00)) do copy /Y %i tsrc @@ -1463,6 +1605,7 @@ mptest: mptester.exe for %i in ($(SRC09)) do copy /Y %i tsrc for %i in ($(SRC10)) do copy /Y %i tsrc for %i in ($(SRC11)) do copy /Y %i tsrc + for %i in ($(SRC12)) do copy /Y %i tsrc copy /Y fts5.c tsrc copy /Y fts5.h tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL @@ -1470,9 +1613,10 @@ mptest: mptester.exe move vdbe.new tsrc\vdbe.c echo > .target_source -sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl - $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS) +sqlite3.c: .target_source sqlite3ext.h $(MKSQLITE3C_TOOL) + $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) copy tsrc\shell.c . + copy $(TOP)\ext\session\sqlite3session.h . sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl @@ -1590,9 +1734,6 @@ hash.lo: $(TOP)\src\hash.c $(HDR) insert.lo: $(TOP)\src\insert.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\insert.c -journal.lo: $(TOP)\src\journal.c $(HDR) - $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\journal.c - legacy.lo: $(TOP)\src\legacy.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\legacy.c @@ -1746,10 +1887,10 @@ wherecode.lo: $(TOP)\src\wherecode.c $(HDR) whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c -tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c -tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) @@ -1775,10 +1916,16 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h $(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS) -sqlite3ext.h: .target_source - copy tsrc\sqlite3ext.h . +sqlite3ext.h: .target_source +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 + type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h + copy /Y sqlite3ext.h tsrc\sqlite3ext.h +!ELSE + copy /Y tsrc\sqlite3ext.h sqlite3ext.h +!ENDIF mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \ @@ -1854,6 +2001,9 @@ fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c +sqlite3session.lo: $(TOP)\ext\session\sqlite3session.c $(HDR) $(EXTHDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c + # FTS5 things # FTS5_SRC = \ @@ -1878,9 +2028,9 @@ fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe del /Q fts5parse.h 2>NUL .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) fts5parse.y -fts5parse.h: fts5parse.c +fts5parse.h: fts5parse.c -fts5.c: $(FTS5_SRC) +fts5.c: $(FTS5_SRC) $(TCLSH_CMD) $(TOP)\ext\fts5\tool\mkfts5c.tcl copy $(TOP)\ext\fts5\fts5.h . @@ -1906,6 +2056,9 @@ sqlite3rbu.lo: $(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR) TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) @@ -1915,13 +2068,33 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF -testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +sqlite_tclDecls.h: + echo #ifndef SQLITE_TCLAPI > $(SQLITETCLDECLSH) + echo # define SQLITE_TCLAPI >> $(SQLITETCLDECLSH) + echo #endif >> $(SQLITETCLDECLSH) + type "$(TCLINCDIR)\tclDecls.h" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN(?: CONST\d+?)?\s+?[^\(]*?\s+?)Tcl_" "\1 SQLITE_TCLAPI Tcl_" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN\s+?(?:void|VOID)\s+?)TclFreeObj" "\1 SQLITE_TCLAPI TclFreeObj" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tcl_" "(SQLITE_TCLAPI *tcl_" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tclFreeObj" "(SQLITE_TCLAPI *tclFreeObj" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_TCLAPI *" >> $(SQLITETCLDECLSH) + +sqlite_tcl.h: + type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH) +!ENDIF + +testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) -extensiontest: testfixture.exe testloadext.dll +extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS) @@ -1964,7 +2137,7 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP) echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ @@ -1980,7 +2153,7 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c -testloadext.dll: testloadext.lo +testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) @@ -1999,6 +2172,10 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2018,38 +2195,40 @@ speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) +rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +moreclean: clean + del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL # <> clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL - del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL # <> - del /Q $(SQLITE3C) $(SQLITE3H) opcodes.c opcodes.h 2>NUL + del /Q sqlite3.c sqlite3.h 2>NUL + del /Q opcodes.c opcodes.h 2>NUL del /Q lemon.* lempar.c parse.* 2>NUL del /Q mkkeywordhash.* keywordhash.h 2>NUL del /Q notasharedlib.* 2>NUL -rmdir /Q/S .deps 2>NUL -rmdir /Q/S .libs 2>NUL - -rmdir /Q/S quota2a 2>NUL - -rmdir /Q/S quota2b 2>NUL - -rmdir /Q/S quota2c 2>NUL -rmdir /Q/S tsrc 2>NUL del /Q .target_source 2>NUL - del /Q tclsqlite3.exe 2>NUL + del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL + del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL - del /Q $(SQLITE3EXE) $(SQLITE3DLL) sqlite3.def 2>NUL del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL - del /Q shell.c sqlite3ext.h 2>NUL + del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL - del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL + del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL # <> diff --git a/VERSION b/VERSION index afad8186..861845e4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.11.0 +3.15.2 diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am index 0e09cfcd..e8211596 100644 --- a/autoconf/Makefile.am +++ b/autoconf/Makefile.am @@ -1,19 +1,20 @@ -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE lib_LTLIBRARIES = libsqlite3.la libsqlite3_la_SOURCES = sqlite3.c libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 bin_PROGRAMS = sqlite3 -sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h -sqlite3_LDADD = @READLINE_LIBS@ +sqlite3_SOURCES = shell.c sqlite3.h +EXTRA_sqlite3_SOURCES = sqlite3.c +sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS include_HEADERS = sqlite3.h sqlite3ext.h -EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt +EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt Replace.cs pkgconfigdir = ${libdir}/pkgconfig pkgconfig_DATA = sqlite3.pc diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 0be42b4e..b53e2370 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -24,6 +24,20 @@ TOP = . USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable full runtime error checks (-RTC1, etc). This +# has no effect if (any) optimizations are enabled. +# +!IFNDEF USE_RUNTIME_CHECKS +USE_RUNTIME_CHECKS = 0 +!ENDIF + +# Set this non-0 to create a SQLite amalgamation file that excludes the +# various built-in extensions. +# +!IFNDEF MINIMAL_AMALGAMATION +MINIMAL_AMALGAMATION = 0 +!ENDIF + # Set this non-0 to use "stdcall" calling convention for the core library # and shell executable. # @@ -183,6 +197,12 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# Set this to non-0 to enable support for the session extension. +# +!IFNDEF SESSION +SESSION = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -204,37 +224,64 @@ SQLITE3H = sqlite3.h # This is the name to use for the SQLite dynamic link library (DLL). # !IFNDEF SQLITE3DLL +!IF $(FOR_WIN10)!=0 +SQLITE3DLL = winsqlite3.dll +!ELSE SQLITE3DLL = sqlite3.dll !ENDIF +!ENDIF # This is the name to use for the SQLite import library (LIB). # !IFNDEF SQLITE3LIB +!IF $(FOR_WIN10)!=0 +SQLITE3LIB = winsqlite3.lib +!ELSE SQLITE3LIB = sqlite3.lib !ENDIF +!ENDIF # This is the name to use for the SQLite shell executable (EXE). # !IFNDEF SQLITE3EXE +!IF $(FOR_WIN10)!=0 +SQLITE3EXE = winsqlite3shell.exe +!ELSE SQLITE3EXE = sqlite3.exe !ENDIF +!ENDIF # This is the argument used to set the program database (PDB) file for the # SQLite shell executable (EXE). # !IFNDEF SQLITE3EXEPDB +!IF $(FOR_WIN10)!=0 +SQLITE3EXEPDB = +!ELSE SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF +!ENDIF + # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +!IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +!ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Should the session extension be enabled? If so, add compilation options +# to enable it. +# +!IF $(SESSION)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 +!ENDIF + # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # @@ -275,6 +322,14 @@ PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\) CC = cl.exe !ENDIF +# Check for the predefined command macro CSC. This should point to a working +# C Sharp compiler binary. If it is not defined, simply define it to the +# legacy default value 'csc.exe'. +# +!IFNDEF CSC +CSC = csc.exe +!ENDIF + # Check for the command macro LD. This should point to the linker binary for # the target platform. If it is not defined, simply define it to the legacy # default value 'link.exe'. @@ -402,15 +457,6 @@ TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) -# Adjust the names of the primary targets for use with Windows 10. -# -!IF $(FOR_WIN10)!=0 -SQLITE3DLL = winsqlite3.dll -SQLITE3LIB = winsqlite3.lib -SQLITE3EXE = winsqlite3shell.exe -SQLITE3EXEPDB = -!ENDIF - # Check if we want to use the "stdcall" calling convention when compiling. # This is not supported by the compilers for non-x86 platforms. It should # also be noted here that building any target with these "stdcall" options @@ -420,12 +466,12 @@ SQLITE3EXEPDB = # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -450,8 +496,10 @@ CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) # when linking. # !IFNDEF CORE_LINK_DEP -!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +!IF $(DYNAMIC_SHELL)!=0 CORE_LINK_DEP = +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" +CORE_LINK_DEP = sqlite3.def !ELSE CORE_LINK_DEP = !ENDIF @@ -460,8 +508,10 @@ CORE_LINK_DEP = # These are additional linker options used for the core library. # !IFNDEF CORE_LINK_OPTS -!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0 +!IF $(DYNAMIC_SHELL)!=0 CORE_LINK_OPTS = +!ELSEIF $(FOR_WIN10)==0 || "$(PLATFORM)"=="x86" +CORE_LINK_OPTS = /DEF:sqlite3.def !ELSE CORE_LINK_OPTS = !ENDIF @@ -707,6 +757,10 @@ RCC = $(RCC) -D_DEBUG !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od +!IF $(USE_RUNTIME_CHECKS)!=0 +TCC = $(TCC) -RTC1 +BCC = $(BCC) -RTC1 +!ENDIF !ELSEIF $(OPTIMIZATIONS)>=3 TCC = $(TCC) -Ox BCC = $(BCC) -Ox @@ -863,23 +917,28 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_ # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: dll libsqlite3.lib shell +all: dll shell # Dynamic link library section. # -dll: $(SQLITE3DLL) +dll: $(SQLITE3DLL) # Shell executable. # -shell: $(SQLITE3EXE) - -libsqlite3.lib: $(LIBOBJ) - $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) +shell: $(SQLITE3EXE) -$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) +$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +Replace.exe: + $(CSC) /target:exe $(TOP)\Replace.cs + +sqlite3.def: Replace.exe $(LIBOBJ) + echo EXPORTS > sqlite3.def + dumpbin /all $(LIBOBJ) \ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ + | sort >> sqlite3.def $(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) $(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \ @@ -918,4 +977,5 @@ $(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL - del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q *.bsc *.def *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL + del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL diff --git a/autoconf/README.txt b/autoconf/README.txt index 2d6db4f0..6e62a4e1 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -6,8 +6,8 @@ This package contains: * the shell.c file used to build the sqlite3 command-line shell program * autoconf/automake installation infrastucture for building on POSIX compliant systems - * a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ on - Windows + * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft + Visual C++ on Windows SUMMARY OF HOW TO BUILD ======================= diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 94925306..b9a11aac 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -30,39 +30,52 @@ AC_FUNC_STRERROR_R AC_CONFIG_FILES([Makefile sqlite3.pc]) AC_SUBST(BUILD_CFLAGS) -#----------------------------------------------------------------------- +#------------------------------------------------------------------------- +# Two options to enable readline compatible libraries: +# # --enable-editline # --enable-readline # -AC_ARG_ENABLE(editline, [AS_HELP_STRING( - [--enable-editline], - [use BSD libedit])], - [], [enable_editline=yes]) -AC_ARG_ENABLE(readline, [AS_HELP_STRING( - [--enable-readline], - [use readline])], - [], [enable_readline=no]) -if test x"$enable_editline" != xno ; then - sLIBS=$LIBS - LIBS="" - AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no]) - READLINE_LIBS=$LIBS - if test x"$LIBS" != "x"; then - AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) - else - unset ac_cv_search_readline - fi - LIBS=$sLIBS -fi -if test x"$enable_readline" != xno ; then - sLIBS=$LIBS - LIBS="" - AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], []) - AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no]) - AC_CHECK_FUNCS(readline, [], []) - READLINE_LIBS=$LIBS - LIBS=$sLIBS -fi +# Both are enabled by default. If, after command line processing both are +# still enabled, the script searches for editline first and automatically +# disables readline if it is found. So, to use readline explicitly, the +# user must pass "--disable-editline". To disable command line editing +# support altogether, "--disable-editline --disable-readline". +# +# When searching for either library, check for headers before libraries +# as some distros supply packages that contain libraries but not header +# files, which come as a separate development package. +# +AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])]) +AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])]) + +AS_IF([ test x"$enable_editline" != xno ],[ + AC_CHECK_HEADERS([editline/readline.h],[ + sLIBS=$LIBS + LIBS="" + AC_SEARCH_LIBS([readline],[edit],[ + AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) + READLINE_LIBS=$LIBS + enable_readline=no + ]) + AS_UNSET(ac_cv_search_readline) + LIBS=$sLIBS + ]) +]) + +AS_IF([ test x"$enable_readline" != xno ],[ + AC_CHECK_HEADERS([readline/readline.h],[ + sLIBS=$LIBS + LIBS="" + AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], []) + AC_SEARCH_LIBS(readline,[readline edit], [ + AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper) + READLINE_LIBS=$LIBS + ]) + LIBS=$sLIBS + ]) +]) + AC_SUBST(READLINE_LIBS) #----------------------------------------------------------------------- @@ -103,7 +116,7 @@ AC_SUBST(DYNAMIC_EXTENSION_FLAGS) AC_ARG_ENABLE(fts5, [AS_HELP_STRING( [--enable-fts5], [include fts5 support [default=no]])], [], [enable_fts5=no]) -if test x"$enable_fts5" == "xyes"; then +if test x"$enable_fts5" = "xyes"; then AC_SEARCH_LIBS(log, m) FTS5_FLAGS=-DSQLITE_ENABLE_FTS5 fi @@ -116,12 +129,24 @@ AC_SUBST(FTS5_FLAGS) AC_ARG_ENABLE(json1, [AS_HELP_STRING( [--enable-json1], [include json1 support [default=no]])], [], [enable_json1=no]) -if test x"$enable_json1" == "xyes"; then +if test x"$enable_json1" = "xyes"; then JSON1_FLAGS=-DSQLITE_ENABLE_JSON1 fi AC_SUBST(JSON1_FLAGS) #----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# --enable-session +# +AC_ARG_ENABLE(session, [AS_HELP_STRING( + [--enable-session], [enable the session extension [default=no]])], + [], [enable_session=no]) +if test x"$enable_session" = "xyes"; then + SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK" +fi +AC_SUBST(SESSION_FLAGS) +#----------------------------------------------------------------------- + #----------------------------------------------------------------------- # --enable-static-shell # @@ -129,8 +154,8 @@ AC_ARG_ENABLE(static-shell, [AS_HELP_STRING( [--enable-static-shell], [statically link libsqlite3 into shell tool [default=yes]])], [], [enable_static_shell=yes]) -if test x"$enable_static_shell" == "xyes"; then - EXTRA_SHELL_OBJ=sqlite3.$OBJEXT +if test x"$enable_static_shell" = "xyes"; then + EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT else EXTRA_SHELL_OBJ=libsqlite3.la fi diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index 8df0af61..7fca05b2 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -78,7 +78,6 @@ TEA_ADD_LIBS([]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1]) -TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1]) TEA_ADD_STUB_SOURCES([]) TEA_ADD_TCL_SOURCES([]) diff --git a/config.h.in b/config.h.in index 36fd6078..f2ba7d4f 100644 --- a/config.h.in +++ b/config.h.in @@ -45,6 +45,18 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H +/* Define to 1 if you have the pread() function. */ +#undef HAVE_PREAD + +/* Define to 1 if you have the pread64() function. */ +#undef HAVE_PREAD64 + +/* Define to 1 if you have the pwrite() function. */ +#undef HAVE_PWRITE + +/* Define to 1 if you have the pwrite64() function. */ +#undef HAVE_PWRITE64 + /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H diff --git a/configure b/configure index b2cb4d13..e22c4dc8 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.11.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.15.2. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.11.0' -PACKAGE_STRING='sqlite 3.11.0' +PACKAGE_VERSION='3.15.2' +PACKAGE_STRING='sqlite 3.15.2' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -903,11 +903,14 @@ with_readline_inc enable_debug enable_amalgamation enable_load_extension +enable_memsys5 +enable_memsys3 enable_fts3 enable_fts4 enable_fts5 enable_json1 enable_rtree +enable_session enable_gcov ' ac_precious_vars='build_alias @@ -1460,7 +1463,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.11.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.15.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1525,7 +1528,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.11.0:";; + short | recursive ) echo "Configuration of sqlite 3.15.2:";; esac cat <<\_ACEOF @@ -1551,11 +1554,14 @@ Optional Features: separately --disable-load-extension Disable loading of external extensions + --enable-memsys5 Enable MEMSYS5 + --enable-memsys3 Enable MEMSYS3 --enable-fts3 Enable the FTS3 extension --enable-fts4 Enable the FTS4 extension --enable-fts5 Enable the FTS5 extension --enable-json1 Enable the JSON1 extension --enable-rtree Enable the RTREE extension + --enable-session Enable the SESSION extension --enable-gcov Enable coverage testing using gcov Optional Packages: @@ -1646,7 +1652,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.11.0 +sqlite configure 3.15.2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2065,7 +2071,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.11.0, which was +It was created by sqlite $as_me 3.15.2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3923,13 +3929,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3926: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3932: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3932: output\"" >&5) + (eval echo "\"\$as_me:3938: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5135,7 +5141,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5138 "configure"' > conftest.$ac_ext + echo '#line 5144 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6660,11 +6666,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6663: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6669: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6667: \$? = $ac_status" >&5 + echo "$as_me:6673: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -6999,11 +7005,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7002: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7008: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7006: \$? = $ac_status" >&5 + echo "$as_me:7012: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7104,11 +7110,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7107: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7113: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7111: \$? = $ac_status" >&5 + echo "$as_me:7117: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7159,11 +7165,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7162: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7168: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7166: \$? = $ac_status" >&5 + echo "$as_me:7172: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9539,7 +9545,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9542 "configure" +#line 9548 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9635,7 +9641,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9638 "configure" +#line 9644 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10273,7 +10279,7 @@ done ######### # Figure out whether or not we have these functions # -for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime +for ac_func in fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64 do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -10755,6 +10761,20 @@ else fi fi + # Recent versions of Xcode on Macs hid the tclConfig.sh file + # in a strange place. + if test x"${ac_cv_c_tclconfig}" = x ; then + if test x"$cross_compiling" = xno; then + for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib + do + if test -f "$i/tclConfig.sh" ; then + ac_cv_c_tclconfig="$i" + break + fi + done + fi + fi + # then check for a private Tcl installation if test x"${ac_cv_c_tclconfig}" = x ; then for i in \ @@ -11323,6 +11343,44 @@ else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi +########## +# Do we want to support memsys3 and/or memsys5 +# +# Check whether --enable-memsys5 was given. +if test "${enable_memsys5+set}" = set; then : + enableval=$enable_memsys5; enable_memsys5=yes +else + enable_memsys5=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5 +$as_echo_n "checking whether to support MEMSYS5... " >&6; } +if test "${enable_memsys5}" = "yes"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +# Check whether --enable-memsys3 was given. +if test "${enable_memsys3+set}" = set; then : + enableval=$enable_memsys3; enable_memsys3=yes +else + enable_memsys3=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5 +$as_echo_n "checking whether to support MEMSYS3... " >&6; } +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + ######### # See whether we should enable Full Text Search extensions # Check whether --enable-fts3 was given. @@ -11494,6 +11552,20 @@ if test "${enable_rtree}" = "yes" ; then OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" fi +######### +# See whether we should enable the SESSION extension +# Check whether --enable-session was given. +if test "${enable_session+set}" = set; then : + enableval=$enable_session; enable_session=yes +else + enable_session=no +fi + +if test "${enable_session}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION" + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK" +fi + ######### # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter for option in $CFLAGS $CPPFLAGS @@ -12079,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.11.0, which was +This file was extended by sqlite $as_me 3.15.2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12145,7 +12217,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.11.0 +sqlite config.status 3.15.2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 19f85dc8..c9e3af92 100644 --- a/configure.ac +++ b/configure.ac @@ -108,7 +108,7 @@ AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h malloc.h]) ######### # Figure out whether or not we have these functions # -AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime]) +AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64]) ######### # By default, we use the amalgamation (this may be changed below...) @@ -334,6 +334,20 @@ if test "${use_tcl}" = "yes" ; then fi fi + # Recent versions of Xcode on Macs hid the tclConfig.sh file + # in a strange place. + if test x"${ac_cv_c_tclconfig}" = x ; then + if test x"$cross_compiling" = xno; then + for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib + do + if test -f "$i/tclConfig.sh" ; then + ac_cv_c_tclconfig="$i" + break + fi + done + fi + fi + # then check for a private Tcl installation if test x"${ac_cv_c_tclconfig}" = x ; then for i in \ @@ -574,6 +588,30 @@ else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi +########## +# Do we want to support memsys3 and/or memsys5 +# +AC_ARG_ENABLE(memsys5, + AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]), + [enable_memsys5=yes],[enable_memsys5=no]) +AC_MSG_CHECKING([whether to support MEMSYS5]) +if test "${enable_memsys5}" = "yes"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5" + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi +AC_ARG_ENABLE(memsys3, + AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]), + [enable_memsys3=yes],[enable_memsys3=no]) +AC_MSG_CHECKING([whether to support MEMSYS3]) +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3" + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi + ######### # See whether we should enable Full Text Search extensions AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3], @@ -615,6 +653,16 @@ if test "${enable_rtree}" = "yes" ; then OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" fi +######### +# See whether we should enable the SESSION extension +AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session], + [Enable the SESSION extension]), + [enable_session=yes],[enable_session=no]) +if test "${enable_session}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION" + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK" +fi + ######### # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter for option in $CFLAGS $CPPFLAGS diff --git a/doc/lemon.html b/doc/lemon.html index b16e3596..114526f3 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -5,14 +5,17 @@

The Lemon Parser Generator

-

Lemon is an LALR(1) parser generator for C or C++. -It does the same job as ``bison'' and ``yacc''. -But lemon is not another bison or yacc clone. It +

Lemon is an LALR(1) parser generator for C. +It does the same job as "bison" and "yacc". +But lemon is not a bison or yacc clone. Lemon uses a different grammar syntax which is designed to -reduce the number of coding errors. Lemon also uses a more -sophisticated parsing engine that is faster than yacc and -bison and which is both reentrant and thread-safe. -Furthermore, Lemon implements features that can be used +reduce the number of coding errors. Lemon also uses a +parsing engine that is faster than yacc and +bison and which is both reentrant and threadsafe. +(Update: Since the previous sentence was written, bison +has also been updated so that it too can generate a +reentrant and threadsafe parser.) +Lemon also implements features that can be used to eliminate resource leaks, making is suitable for use in long-running programs such as graphical user interfaces or embedded controllers.

@@ -44,18 +47,18 @@ one and three files of outputs. automaton. By default, all three of these output files are generated. -The header file is suppressed if the ``-m'' command-line option is -used and the report file is omitted when ``-q'' is selected.

+The header file is suppressed if the "-m" command-line option is +used and the report file is omitted when "-q" is selected.

-

The grammar specification file uses a ``.y'' suffix, by convention. +

The grammar specification file uses a ".y" suffix, by convention. In the examples used in this document, we'll assume the name of the -grammar file is ``gram.y''. A typical use of Lemon would be the +grammar file is "gram.y". A typical use of Lemon would be the following command: 

    lemon gram.y
-This command will generate three output files named ``gram.c'', -``gram.h'' and ``gram.out''. +This command will generate three output files named "gram.c", +"gram.h" and "gram.out". The first is C code to implement the parser. The second is the header file that defines numerical values for all terminal symbols, and the last is the report that explains @@ -71,39 +74,35 @@ with a brief explanation of what each does by typing As of this writing, the following command-line options are supported:
    -
  • -b -
  • -c -
  • -g -
  • -m -
  • -q -
  • -s -
  • -x +
  • -b +Show only the basis for each parser state in the report file. +
  • -c +Do not compress the generated action tables. +
  • -Dname +Define C preprocessor macro name. This macro is useable by +"%ifdef" lines in the grammar file. +
  • -g +Do not generate a parser. Instead write the input grammar to standard +output with all comments, actions, and other extraneous text removed. +
  • -l +Omit "#line" directives in the generated parser C code. +
  • -m +Cause the output C source code to be compatible with the "makeheaders" +program. +
  • -p +Display all conflicts that are resolved by +precedence rules. +
  • -q +Suppress generation of the report file. +
  • -r +Do not sort or renumber the parser states as part of optimization. +
  • -s +Show parser statistics before existing. +
  • -Tfile +Use file as the template for the generated C-code parser implementation. +
  • -x +Print the Lemon version number.
-The ``-b'' option reduces the amount of text in the report file by -printing only the basis of each parser state, rather than the full -configuration. -The ``-c'' option suppresses action table compression. Using -c -will make the parser a little larger and slower but it will detect -syntax errors sooner. -The ``-g'' option causes no output files to be generated at all. -Instead, the input grammar file is printed on standard output but -with all comments, actions and other extraneous text deleted. This -is a useful way to get a quick summary of a grammar. -The ``-m'' option causes the output C source file to be compatible -with the ``makeheaders'' program. -Makeheaders is a program that automatically generates header files -from C source code. When the ``-m'' option is used, the header -file is not output since the makeheaders program will take care -of generated all header files automatically. -The ``-q'' option suppresses the report file. -Using ``-s'' causes a brief summary of parser statistics to be -printed. Like this: -
-   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
-                      340 states, 2026 parser table entries, 0 conflicts
-
-Finally, the ``-x'' option causes Lemon to print its version number -and then stops without attempting to read the grammar or generate a parser.

The Parser Interface

@@ -121,12 +120,12 @@ A new parser is created as follows: The ParseAlloc() routine allocates and initializes a new parser and returns a pointer to it. -The actual data structure used to represent a parser is opaque -- +The actual data structure used to represent a parser is opaque — its internal structure is not visible or usable by the calling routine. For this reason, the ParseAlloc() routine returns a pointer to void rather than a pointer to some particular structure. The sole argument to the ParseAlloc() routine is a pointer to the -subroutine used to allocate memory. Typically this means ``malloc()''.

+subroutine used to allocate memory. Typically this means malloc().

After a program is finished using a parser, it can reclaim all memory allocated by that parser by calling @@ -151,18 +150,19 @@ type of the next token in the data stream. There is one token type for each terminal symbol in the grammar. The gram.h file generated by Lemon contains #define statements that map symbolic terminal symbol names into appropriate integer values. -(A value of 0 for the second argument is a special flag to the -parser to indicate that the end of input has been reached.) +A value of 0 for the second argument is a special flag to the +parser to indicate that the end of input has been reached. The third argument is the value of the given token. By default, the type of the third argument is integer, but the grammar will usually redefine this type to be some kind of structure. Typically the second argument will be a broad category of tokens -such as ``identifier'' or ``number'' and the third argument will +such as "identifier" or "number" and the third argument will be the name of the identifier or the value of the number.

The Parse() function may have either three or four arguments, -depending on the grammar. If the grammar specification file request -it, the Parse() function will have a fourth parameter that can be +depending on the grammar. If the grammar specification file requests +it (via the extra_argument directive), +the Parse() function will have a fourth parameter that can be of any type chosen by the programmer. The parser doesn't do anything with this argument except to pass it through to action routines. This is a convenient mechanism for passing state information down @@ -192,7 +192,7 @@ following: This example shows a user-written routine that parses a file of text and returns a pointer to the parse tree. -(We've omitted all error-handling from this example to keep it +(All error-handling code is omitted from this example to keep it simple.) We assume the existence of some kind of tokenizer which is created using TokenizerCreate() on line 8 and deleted by TokenizerFree() @@ -263,6 +263,12 @@ with prior yacc and bison experience. But after years of experience using Lemon, I firmly believe that the Lemon way of doing things is better.

+

Updated as of 2016-02-16: +The text above was written in the 1990s. +We are told that Bison has lately been enhanced to support the +tokenizer-calls-parser paradigm used by Lemon, and to obviate the +need for global variables.

+

Input File Syntax

The main purpose of the grammar specification file for Lemon is @@ -280,7 +286,7 @@ tokens) and it honors the same commenting conventions as C and C++.

Terminals and Nonterminals

A terminal symbol (token) is any string of alphanumeric -and underscore characters +and/or underscore characters that begins with an upper case letter. A terminal can contain lowercase letters after the first character, but the usual convention is to make terminals all upper case. @@ -307,7 +313,7 @@ must have alphanumeric names.

The main component of a Lemon grammar file is a sequence of grammar rules. Each grammar rule consists of a nonterminal symbol followed by -the special symbol ``::='' and then a list of terminals and/or nonterminals. +the special symbol "::=" and then a list of terminals and/or nonterminals. The rule is terminated by a period. The list of terminals and nonterminals on the right-hand side of the rule can be empty. @@ -323,9 +329,9 @@ A typical sequence of grammar rules might look something like this:

-

There is one non-terminal in this example, ``expr'', and five -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'', -``RPAREN'' and ``VALUE''.

+

There is one non-terminal in this example, "expr", and five +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN", +"RPAREN" and "VALUE".

Like yacc and bison, Lemon allows the grammar to specify a block of C code that will be executed whenever a grammar rule is reduced @@ -341,15 +347,15 @@ For example:

In order to be useful, grammar actions must normally be linked to their associated grammar rules. -In yacc and bison, this is accomplished by embedding a ``$$'' in the +In yacc and bison, this is accomplished by embedding a "$$" in the action to stand for the value of the left-hand side of the rule and -symbols ``$1'', ``$2'', and so forth to stand for the value of +symbols "$1", "$2", and so forth to stand for the value of the terminal or nonterminal at position 1, 2 and so forth on the right-hand side of the rule. This idea is very powerful, but it is also very error-prone. The single most common source of errors in a yacc or bison grammar is to miscount the number of symbols on the right-hand side of a grammar -rule and say ``$7'' when you really mean ``$8''.

+rule and say "$7" when you really mean "$8".

Lemon avoids the need to count grammar symbols by assigning symbolic names to each symbol in a grammar rule and then using those symbolic @@ -379,7 +385,7 @@ For example, the rule 

   expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
-will generate an error because the linking symbol ``C'' is used +will generate an error because the linking symbol "C" is used in the grammar rule but not in the reduce action.

The Lemon notation for linking grammar rules to reduce actions @@ -387,6 +393,7 @@ also facilitates the use of destructors for reclaiming memory allocated by the values of terminals and nonterminals on the right-hand side of a rule.

+

Precedence Rules

Lemon resolves parsing ambiguities in exactly the same way as @@ -398,7 +405,10 @@ whichever rule comes first in the grammar file.

yacc and bison, Lemon allows a measure of control over the resolution of paring conflicts using precedence rules. A precedence value can be assigned to any terminal symbol -using the %left, %right or %nonassoc directives. Terminal symbols +using the +%left, +%right or +%nonassoc directives. Terminal symbols mentioned in earlier directives have a lower precedence that terminal symbols mentioned in later directives. For example:

@@ -518,7 +528,11 @@ other than that, the order of directives in Lemon is arbitrary.

  • %default_destructor
  • %default_type
  • %destructor +
  • %endif
  • %extra_argument +
  • %fallback +
  • %ifdef +
  • %ifndef
  • %include
  • %left
  • %name @@ -530,49 +544,57 @@ other than that, the order of directives in Lemon is arbitrary.

  • %stack_size
  • %start_symbol
  • %syntax_error +
  • %token_class
  • %token_destructor
  • %token_prefix
  • %token_type
  • %type +
  • %wildcard Each of these directives will be described separately in the following sections:

    +

    The %code directive

    -

    The %code directive is used to specify addition C/C++ code that +

    The %code directive is used to specify addition C code that is added to the end of the main output file. This is similar to -the %include directive except that %include is inserted at the -beginning of the main output file.

    +the %include directive except that %include +is inserted at the beginning of the main output file.

    %code is typically used to include some action routines or perhaps -a tokenizer as part of the output file.

    +a tokenizer or even the "main()" function +as part of the output file.

    +

    The %default_destructor directive

    The %default_destructor directive specifies a destructor to use for non-terminals that do not have their own destructor specified by a separate %destructor directive. See the documentation -on the %destructor directive below for additional information.

    +on the %destructor directive below for +additional information.

    In some grammers, many different non-terminal symbols have the same datatype and hence the same destructor. This directive is a convenience way to specify the same destructor for all those non-terminals using a single statement.

    +

    The %default_type directive

    The %default_type directive specifies the datatype of non-terminal symbols that do no have their own datatype defined using a separate -%type directive. See the documentation on %type below for addition -information.

    +%type directive. +

    +

    The %destructor directive

    The %destructor directive is used to specify a destructor for a non-terminal symbol. -(See also the %token_destructor directive which is used to -specify a destructor for terminal symbols.)

    +(See also the %token_destructor +directive which is used to specify a destructor for terminal symbols.)

    A non-terminal's destructor is called to dispose of the non-terminal's value whenever the non-terminal is popped from @@ -595,28 +617,28 @@ or other resources held by that non-terminal.

    This example is a bit contrived but it serves to illustrate how destructors work. The example shows a non-terminal named -``nt'' that holds values of type ``void*''. When the rule for -an ``nt'' reduces, it sets the value of the non-terminal to +"nt" that holds values of type "void*". When the rule for +an "nt" reduces, it sets the value of the non-terminal to space obtained from malloc(). Later, when the nt non-terminal is popped from the stack, the destructor will fire and call free() on this malloced space, thus avoiding a memory leak. -(Note that the symbol ``$$'' in the destructor code is replaced +(Note that the symbol "$$" in the destructor code is replaced by the value of the non-terminal.)

    It is important to note that the value of a non-terminal is passed to the destructor whenever the non-terminal is removed from the stack, unless the non-terminal is used in a C-code action. If the non-terminal is used by C-code, then it is assumed that the -C-code will take care of destroying it if it should really -be destroyed. More commonly, the value is used to build some +C-code will take care of destroying it. +More commonly, the value is used to build some larger structure and we don't want to destroy it, which is why the destructor is not called in this circumstance.

    -

    By appropriate use of destructors, it is possible to -build a parser using Lemon that can be used within a long-running -program, such as a GUI, that will not leak memory or other resources. +

    Destructors help avoid memory leaks by automatically freeing +allocated objects when they go out of scope. To do the same using yacc or bison is much more difficult.

    +

    The %extra_argument directive

    The %extra_argument directive instructs Lemon to add a 4th parameter @@ -630,17 +652,66 @@ and so forth. For example, if the grammar file contains:

    Then the Parse() function generated will have an 4th parameter -of type ``MyStruct*'' and all action routines will have access to -a variable named ``pAbc'' that is the value of the 4th parameter +of type "MyStruct*" and all action routines will have access to +a variable named "pAbc" that is the value of the 4th parameter in the most recent call to Parse().

    + +

    The %fallback directive

    + +

    The %fallback directive specifies an alternative meaning for one +or more tokens. The alternative meaning is tried if the original token +would have generated a syntax error. + +

    The %fallback directive was added to support robust parsing of SQL +syntax in SQLite. +The SQL language contains a large assortment of keywords, each of which +appears as a different token to the language parser. SQL contains so +many keywords, that it can be difficult for programmers to keep up with +them all. Programmers will, therefore, sometimes mistakenly use an +obscure language keyword for an identifier. The %fallback directive +provides a mechanism to tell the parser: "If you are unable to parse +this keyword, try treating it as an identifier instead." + +

    The syntax of %fallback is as follows: + +

    +%fallback ID TOKEN... . +
    + +

    In words, the %fallback directive is followed by a list of token names +terminated by a period. The first token name is the fallback token - the +token to which all the other tokens fall back to. The second and subsequent +arguments are tokens which fall back to the token identified by the first +argument. + + +

    The %ifdef, %ifndef, and %endif directives.

    + +

    The %ifdef, %ifndef, and %endif directives are similar to +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general. +Each of these directives must begin at the left margin. No whitespace +is allowed between the "%" and the directive name. + +

    Grammar text in between "%ifdef MACRO" and the next nested "%endif" is +ignored unless the "-DMACRO" command-line option is used. Grammar text +betwen "%ifndef MACRO" and the next nested "%endif" is included except when +the "-DMACRO" command-line option is used. + +

    Note that the argument to %ifdef and %ifndef must be a single +preprocessor symbol name, not a general expression. There is no "%else" +directive. + + +

    The %include directive

    The %include directive specifies C code that is included at the top of the generated parser. You can include any text you want -- the Lemon parser generator copies it blindly. If you have multiple -%include directives in your grammar file the value of the last -%include directive overwrites all the others.

    The %include directive is very handy for getting some extra #include preprocessor statements at the beginning of the generated parser. @@ -653,12 +724,13 @@ For example:

    This might be needed, for example, if some of the C actions in the grammar call functions that are prototyed in unistd.h.

    +

    The %left directive

    -The %left directive is used (along with the %right and -%nonassoc directives) to declare precedences of terminal -symbols. Every terminal symbol whose name appears after -a %left directive but before the next period (``.'') is +The %left directive is used (along with the %right and +%nonassoc directives) to declare precedences of +terminal symbols. Every terminal symbol whose name appears after +a %left directive but before the next period (".") is given the same left-associative precedence value. Subsequent %left directives have higher precedence. For example:

    @@ -679,10 +751,11 @@ a large amount of stack space if you make heavy use or right-associative operators. For this reason, it is recommended that you use %left rather than %right whenever possible.

    +

    The %name directive

    By default, the functions generated by Lemon all begin with the -five-character string ``Parse''. You can change this string to something +five-character string "Parse". You can change this string to something different using the %name directive. For instance:

    @@ -701,16 +774,19 @@ The %name directive allows you to generator two or more different
 parsers and link them all into the same executable.
 
    
 
+
 
    The %nonassoc directive
    
 
 
    This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on precedence rules
-or on the %left directive for additional information.
    
+one or more terminal symbols.  See the section on 
+precedence rules
+or on the %left directive for additional information.
    
 
+
 
    The %parse_accept directive
    
 
 
    The %parse_accept directive specifies a block of C code that is
-executed whenever the parser accepts its input string.  To ``accept''
+executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.
    
 
@@ -722,7 +798,7 @@ without error.
    
    }

    - +

    The %parse_failure directive

    The %parse_failure directive specifies a block of C code that @@ -737,12 +813,15 @@ only invoked when parsing is unable to continue.

    }

    +

    The %right directive

    This directive is used to assign right-associative precedence to -one or more terminal symbols. See the section on precedence rules -or on the %left directive for additional information.

    +one or more terminal symbols. See the section on +precedence rules +or on the %left directive for additional information.

    +

    The %stack_overflow directive

    The %stack_overflow directive specifies a block of C code that @@ -771,6 +850,7 @@ Not like this: list ::= . +

    The %stack_size directive

    If stack overflow is a problem and you can't resolve the trouble @@ -783,6 +863,7 @@ with a stack of the requested size. The default value is 100.

    %stack_size 2000

    +

    The %start_symbol directive

    By default, the start-symbol for the grammar that Lemon generates @@ -793,6 +874,7 @@ can choose a different start-symbol using the %start_symbol directive.

    %start_symbol prog

    +

    The %token_destructor directive

    The %destructor directive assigns a destructor to a non-terminal @@ -805,6 +887,7 @@ the %token_type directive) and so they use a common destructor. Other than that, the token destructor works just like the non-terminal destructors.

    +

    The %token_prefix directive

    Lemon generates #defines that assign small integer constants @@ -830,6 +913,7 @@ to cause Lemon to produce these symbols instead: #define TOKEN_PLUS 4 +

    The %token_type and %type directives

    These directives are used to specify the data types for values @@ -845,7 +929,7 @@ token structure. Like this:

    If the data type of terminals is not specified, the default value -is ``int''.

    +is "void*".

    Non-terminal symbols can each have their own data types. Typically the data type of a non-terminal is a pointer to the root of a parse-tree @@ -866,6 +950,17 @@ non-terminal whose data type requires 1K of storage, then your 100 entry parser stack will require 100K of heap space. If you are willing and able to pay that price, fine. You just need to know.

    + +

    The %wildcard directive

    + +

    The %wildcard directive is followed by a single token name and a +period. This directive specifies that the identified token should +match any input token. + +

    When the generated parser has the choice of matching an input against +the wildcard token and some other token, the other token is always used. +The wildcard token is only matched if there are no other alternatives. +

    Error Processing

    After extensive experimentation over several years, it has been @@ -877,7 +972,7 @@ first invokes the code specified by the %syntax_error directive, if any. It then enters its error recovery strategy. The error recovery strategy is to begin popping the parsers stack until it enters a state where it is permitted to shift a special non-terminal symbol -named ``error''. It then shifts this non-terminal and continues +named "error". It then shifts this non-terminal and continues parsing. But the %syntax_error routine will not be called again until at least three new tokens have been successfully shifted.

    @@ -886,7 +981,7 @@ is unable to shift the error symbol, then the %parse_failed routine is invoked and the parser resets itself to its start state, ready to begin parsing a new file. This is what will happen at the very first syntax error, of course, if there are no instances of the -``error'' non-terminal in your grammar.

    +"error" non-terminal in your grammar.

    diff --git a/ext/fts2/fts2_tokenizer.c b/ext/fts2/fts2_tokenizer.c index a3d6a631..dda33a72 100644 --- a/ext/fts2/fts2_tokenizer.c +++ b/ext/fts2/fts2_tokenizer.c @@ -99,7 +99,11 @@ static void scalarFunc( #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif #include /* diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 36dcc94e..a48a556c 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -18,7 +18,14 @@ ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. */ -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif #include #include @@ -143,7 +150,7 @@ static int nm_match_count( /* ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS? */ -static int fts3_near_match_cmd( +static int SQLITE_TCLAPI fts3_near_match_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -278,7 +285,7 @@ static int fts3_near_match_cmd( ** # Restore initial incr-load settings: ** eval fts3_configure_incr_load $cfg */ -static int fts3_configure_incr_load_cmd( +static int SQLITE_TCLAPI fts3_configure_incr_load_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -458,7 +465,7 @@ static int testTokenizerNext( if( pCsr->iLangid & 0x00000001 ){ for(i=0; iaBuffer[i] = pToken[i]; }else{ - for(i=0; iaBuffer[i] = testTolower(pToken[i]); + for(i=0; iaBuffer[i] = (char)testTolower(pToken[i]); } pCsr->iToken++; pCsr->iInput = (int)(p - pCsr->aInput); @@ -488,7 +495,7 @@ static int testTokenizerLanguage( } #endif -static int fts3_test_tokenizer_cmd( +static int SQLITE_TCLAPI fts3_test_tokenizer_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -517,7 +524,7 @@ static int fts3_test_tokenizer_cmd( return TCL_OK; } -static int fts3_test_varint_cmd( +static int SQLITE_TCLAPI fts3_test_varint_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -526,7 +533,8 @@ static int fts3_test_varint_cmd( #ifdef SQLITE_ENABLE_FTS3 char aBuf[24]; int rc; - Tcl_WideInt w, w2; + Tcl_WideInt w; + sqlite3_int64 w2; int nByte, nByte2; if( objc!=2 ){ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index fcabe5cc..bfc36af3 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -29,6 +29,18 @@ #include #include +/* +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +*/ +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} + /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: @@ -49,7 +61,7 @@ ** is a blob containing the pointer stored as the hash data corresponding ** to string (after the hash-table is updated, if applicable). */ -static void scalarFunc( +static void fts3TokenizerFunc( sqlite3_context *context, int argc, sqlite3_value **argv @@ -67,27 +79,23 @@ static void scalarFunc( nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ -#ifdef SQLITE_ENABLE_FTS3_TOKENIZER - void *pOld; - int n = sqlite3_value_bytes(argv[1]); - if( zName==0 || n!=sizeof(pPtr) ){ - sqlite3_result_error(context, "argument type mismatch", -1); + if( fts3TokenizerEnabled(context) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); return; } - pPtr = *(void **)sqlite3_value_blob(argv[1]); - pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); - if( pOld==pPtr ){ - sqlite3_result_error(context, "out of memory", -1); - return; - } -#else - sqlite3_result_error(context, "fts3tokenize: " - "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1 - ); - return; -#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */ - }else - { + }else{ if( zName ){ pPtr = sqlite3Fts3HashFind(pHash, zName, nName); } @@ -98,7 +106,6 @@ static void scalarFunc( return; } } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } @@ -217,7 +224,11 @@ int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif #include /* @@ -336,7 +347,6 @@ finish: Tcl_DecrRefCount(pRet); } -#ifdef SQLITE_ENABLE_FTS3_TOKENIZER static int registerTokenizer( sqlite3 *db, @@ -358,7 +368,6 @@ int registerTokenizer( return sqlite3_finalize(pStmt); } -#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */ static @@ -431,13 +440,13 @@ static void intTestFunc( assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); /* Test the storage function */ -#ifdef SQLITE_ENABLE_FTS3_TOKENIZER - rc = registerTokenizer(db, "nosuchtokenizer", p1); - assert( rc==SQLITE_OK ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_OK ); - assert( p2==p1 ); -#endif + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + } sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } @@ -453,7 +462,7 @@ static void intTestFunc( ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above -** scalarFunc() in this file for details) and, if ENABLE_TABLE is +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is ** defined at compilation time, a temporary virtual table (see header ** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. @@ -482,10 +491,10 @@ int sqlite3Fts3InitHashTable( #endif if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); } if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); } #ifdef SQLITE_TEST if( SQLITE_OK==rc ){ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index d5a40822..3ff481b0 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -333,7 +333,8 @@ static int fts3SqlStmt( ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ -/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" " ORDER BY (level %% 1024) ASC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment @@ -3194,7 +3195,7 @@ static int fts3SegmentMerge( ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ - if( csr.nSegment==1 ){ + if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ rc = SQLITE_DONE; goto finished; } @@ -4836,10 +4837,11 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, nMin); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - nSeg = nMin; + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); }else{ nSeg = -1; } diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index a8d7981a..37f9b739 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -398,8 +398,8 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){ if( sqlite3_step(pStmt)==SQLITE_ROW && (nLeaf = sqlite3_column_int(pStmt, 0))>0 ){ - nIdx = sqlite3_column_int(pStmt, 5); sqlite3_int64 sz; + nIdx = sqlite3_column_int(pStmt, 5); printf("For level %d:\n", i); printf(" Number of indexes...................... %9d\n", nIdx); printf(" Number of leaf segments................ %9d\n", nLeaf); diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 96ecb38e..a45c145d 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -143,11 +143,13 @@ struct Fts5PhraseIter { ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid ** ** with $p set to a phrase equivalent to the phrase iPhrase of the -** current query is executed. For each row visited, the callback function -** passed as the fourth argument is invoked. The context and API objects -** passed to the callback function may be used to access the properties of -** each matched row. Invoking Api.xUserData() returns a copy of the pointer -** passed as the third argument to pUserData. +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. ** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. @@ -316,7 +318,7 @@ struct Fts5ExtensionApi { ** behaviour. The structure methods are expected to function as follows: ** ** xCreate: -** This function is used to allocate and inititalize a tokenizer instance. +** This function is used to allocate and initialize a tokenizer instance. ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) @@ -575,4 +577,3 @@ struct fts5_api { #endif #endif /* _FTS5_H */ - diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index af404121..9ef338e8 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -47,6 +47,10 @@ typedef sqlite3_uint64 u64; #endif +/* Truncate very long tokens to this many bytes. Hard limit is +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset +** field that occurs at the start of each leaf page (see fts5_index.c). */ +#define FTS5_MAX_TOKEN_SIZE 32768 /* ** Maximum number of prefix indexes on single FTS5 table. This must be @@ -172,6 +176,7 @@ struct Fts5Config { int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ + int nUsermerge; /* 'usermerge' setting */ int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ @@ -479,6 +484,7 @@ int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); +int sqlite3Fts5IndexReset(Fts5Index *p); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); @@ -621,6 +627,7 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); +int sqlite3Fts5StorageReset(Fts5Storage *p); /* ** End of interface to code in fts5_storage.c. @@ -679,7 +686,6 @@ int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); -void sqlite3Fts5ExprClearEof(Fts5Expr*); int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); @@ -700,6 +706,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode( Fts5ExprNearset *pNear ); +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, + Fts5ExprNode *pLeft, + Fts5ExprNode *pRight +); + Fts5ExprPhrase *sqlite3Fts5ParseTerm( Fts5Parse *pParse, Fts5ExprPhrase *pPhrase, @@ -725,6 +737,7 @@ void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 836de8a0..219ea6ff 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -189,7 +189,7 @@ static int fts5HighlightCb( if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; - if( iPositer.iEnd ){ + if( iPos>=p->iter.iStart && iPositer.iEnd ){ fts5HighlightAppend(&rc, p, p->zClose, -1); } } @@ -246,6 +246,118 @@ static void fts5HighlightFunction( ** End of highlight() implementation. **************************************************************************/ +/* +** Context object passed to the fts5SentenceFinderCb() function. +*/ +typedef struct Fts5SFinder Fts5SFinder; +struct Fts5SFinder { + int iPos; /* Current token position */ + int nFirstAlloc; /* Allocated size of aFirst[] */ + int nFirst; /* Number of entries in aFirst[] */ + int *aFirst; /* Array of first token in each sentence */ + const char *zDoc; /* Document being tokenized */ +}; + +/* +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an +** error occurs. +*/ +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ + if( p->nFirstAlloc==p->nFirst ){ + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; + int *aNew; + + aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + p->aFirst = aNew; + p->nFirstAlloc = nNew; + } + p->aFirst[p->nFirst++] = iAdd; + return SQLITE_OK; +} + +/* +** This function is an xTokenize() callback used by the auxiliary snippet() +** function. Its job is to identify tokens that are the first in a sentence. +** For each such token, an entry is added to the SFinder.aFirst[] array. +*/ +static int fts5SentenceFinderCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + int rc = SQLITE_OK; + + UNUSED_PARAM2(pToken, nToken); + UNUSED_PARAM(iEndOff); + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + Fts5SFinder *p = (Fts5SFinder*)pContext; + if( p->iPos>0 ){ + int i; + char c = 0; + for(i=iStartOff-1; i>=0; i--){ + c = p->zDoc[i]; + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; + } + if( i!=iStartOff-1 && (c=='.' || c==':') ){ + rc = fts5SentenceFinderAdd(p, p->iPos); + } + }else{ + rc = fts5SentenceFinderAdd(p, 0); + } + p->iPos++; + } + return rc; +} + +static int fts5SnippetScore( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + int nDocsize, /* Size of column in tokens */ + unsigned char *aSeen, /* Array with one element per query phrase */ + int iCol, /* Column to score */ + int iPos, /* Starting offset to score */ + int nToken, /* Max tokens per snippet */ + int *pnScore, /* OUT: Score */ + int *piPos /* OUT: Adjusted offset */ +){ + int rc; + int i; + int ip = 0; + int ic = 0; + int iOff = 0; + int iFirst = -1; + int nInst; + int nScore = 0; + int iLast = 0; + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; ixInst(pFts, i, &ip, &ic, &iOff); + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){ + nScore += (aSeen[ip] ? 1 : 1000); + aSeen[ip] = 1; + if( iFirst<0 ) iFirst = iOff; + iLast = iOff + pApi->xPhraseSize(pFts, ip); + } + } + + *pnScore = nScore; + if( piPos ){ + int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; + if( iAdj<0 ) iAdj = 0; + *piPos = iAdj; + } + + return rc; +} + /* ** Implementation of snippet() function. */ @@ -267,9 +379,10 @@ static void fts5SnippetFunction( unsigned char *aSeen; /* Array of "seen instance" flags */ int iBestCol; /* Column containing best snippet */ int iBestStart = 0; /* First token of best snippet */ - int iBestLast; /* Last token of best snippet */ int nBestScore = 0; /* Score of best snippet */ int nColSize = 0; /* Total size of iBestCol in tokens */ + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ + int nCol; if( nVal!=5 ){ const char *zErr = "wrong number of arguments to function snippet()"; @@ -277,13 +390,13 @@ static void fts5SnippetFunction( return; } + nCol = pApi->xColumnCount(pFts); memset(&ctx, 0, sizeof(HighlightContext)); iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); zEllips = (const char*)sqlite3_value_text(apVal[3]); nToken = sqlite3_value_int(apVal[4]); - iBestLast = nToken-1; iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); @@ -291,65 +404,94 @@ static void fts5SnippetFunction( if( aSeen==0 ){ rc = SQLITE_NOMEM; } - if( rc==SQLITE_OK ){ rc = pApi->xInstCount(pFts, &nInst); } - for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &ip, &iSnippetCol, &iStart); - if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ - int nScore = 1000; - int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); - int j; - aSeen[ip] = 1; - for(j=i+1; rc==SQLITE_OK && jxInst(pFts, j, &ip, &ic, &io); - iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; - if( rc==SQLITE_OK && ic==iSnippetCol && iLastiLast ) iLast = iFinal; + memset(&sFinder, 0, sizeof(Fts5SFinder)); + for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize(pFts, + sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + ); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnSize(pFts, i, &nDocsize); + if( rc!=SQLITE_OK ) break; + + for(ii=0; rc==SQLITE_OK && iixInst(pFts, ii, &ip, &ic, &io); + if( ic!=i || rc!=SQLITE_OK ) continue; + memset(aSeen, 0, nPhrase); + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, + io, nToken, &nScore, &iAdj + ); + if( rc==SQLITE_OK && nScore>nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = iAdj; + nColSize = nDocsize; } - } - if( rc==SQLITE_OK && nScore>nBestScore ){ - iBestCol = iSnippetCol; - iBestStart = iStart; - iBestLast = iLast; - nBestScore = nScore; + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ + for(jj=0; jj<(sFinder.nFirst-1); jj++){ + if( sFinder.aFirst[jj+1]>io ) break; + } + + if( sFinder.aFirst[jj]nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = sFinder.aFirst[jj]; + nColSize = nDocsize; + } + } + } } } } - if( rc==SQLITE_OK ){ - rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); - } if( rc==SQLITE_OK ){ rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); } + if( rc==SQLITE_OK && nColSize==0 ){ + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); + } if( ctx.zIn ){ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } - if( (iBestStart+nToken-1)>iBestLast ){ - iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; - } - if( iBestStart+nToken>nColSize ){ - iBestStart = nColSize - nToken; - } - if( iBestStart<0 ) iBestStart = 0; - ctx.iRangeStart = iBestStart; ctx.iRangeEnd = iBestStart + nToken - 1; if( iBestStart>0 ){ fts5HighlightAppend(&rc, &ctx, zEllips, -1); } + + /* Advance iterator ctx.iter so that it points to the first coalesced + ** phrase instance at or following position iBestStart. */ + while( ctx.iter.iStart>=0 && ctx.iter.iStartxTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } @@ -358,15 +500,15 @@ static void fts5SnippetFunction( }else{ fts5HighlightAppend(&rc, &ctx, zEllips, -1); } - - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - sqlite3_free(ctx.zOut); } + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + sqlite3_free(ctx.zOut); sqlite3_free(aSeen); + sqlite3_free(sFinder.aFirst); } /************************************************************************/ diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index f49cede1..17fc43e0 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -18,6 +18,7 @@ #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 +#define FTS5_DEFAULT_USERMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) @@ -441,7 +442,9 @@ static const char *fts5ConfigGobbleWord( *pbQuoted = 1; }else{ zRet = fts5ConfigSkipBareword(zIn); - zOut[zRet-zIn] = '\0'; + if( zRet ){ + zOut[zRet-zIn] = '\0'; + } } } @@ -857,6 +860,18 @@ int sqlite3Fts5ConfigSetValue( } } + else if( 0==sqlite3_stricmp(zKey, "usermerge") ){ + int nUsermerge = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nUsermerge = sqlite3_value_int(pVal); + } + if( nUsermerge<2 || nUsermerge>16 ){ + *pbBadkey = 1; + }else{ + pConfig->nUsermerge = nUsermerge; + } + } + else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){ int nCrisisMerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ @@ -903,6 +918,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; + pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index d3f801b0..bde9eaed 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -167,6 +167,7 @@ static int fts5ExprGetToken( case ',': tok = FTS5_COMMA; break; case '+': tok = FTS5_PLUS; break; case '*': tok = FTS5_STAR; break; + case '-': tok = FTS5_MINUS; break; case '\0': tok = FTS5_EOF; break; case '"': { @@ -258,6 +259,8 @@ int sqlite3Fts5ExprNew( pNew->nPhrase = sParse.nPhrase; sParse.apPhrase = 0; } + }else{ + sqlite3Fts5ParseNodeFree(sParse.pExpr); } sqlite3_free(sParse.apPhrase); @@ -751,6 +754,7 @@ static int fts5ExprNearInitAll( Fts5ExprNearset *pNear = pNode->pNear; int i, j; int rc = SQLITE_OK; + int bEof = 1; assert( pNode->bNomatch==0 ); for(i=0; rc==SQLITE_OK && inPhrase; i++){ @@ -758,7 +762,6 @@ static int fts5ExprNearInitAll( for(j=0; jnTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; Fts5ExprTerm *p; - int bEof = 1; for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ if( p->pIter ){ @@ -778,13 +781,12 @@ static int fts5ExprNearInitAll( } } - if( bEof ){ - pNode->bEof = 1; - return rc; - } + if( bEof ) break; } + if( bEof ) break; } + pNode->bEof = bEof; return rc; } @@ -918,7 +920,7 @@ static int fts5ExprNodeTest_STRING( } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - if( pIter->iRowid==iLast ) continue; + if( pIter->iRowid==iLast || pIter->bEof ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; @@ -1268,6 +1270,8 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ if( Fts5NodeIsString(pNode) ){ /* Initialize all term iterators in the NEAR object. */ rc = fts5ExprNearInitAll(pExpr, pNode); + }else if( pNode->xNext==0 ){ + pNode->bEof = 1; }else{ int i; int nEof = 0; @@ -1319,23 +1323,22 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ */ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ Fts5ExprNode *pRoot = p->pRoot; - int rc = SQLITE_OK; - if( pRoot->xNext ){ - p->pIndex = pIdx; - p->bDesc = bDesc; - rc = fts5ExprNodeFirst(p, pRoot); + int rc; /* Return code */ - /* If not at EOF but the current rowid occurs earlier than iFirst in - ** the iteration order, move to document iFirst or later. */ - if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ - rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); - } + p->pIndex = pIdx; + p->bDesc = bDesc; + rc = fts5ExprNodeFirst(p, pRoot); - /* If the iterator is not at a real match, skip forward until it is. */ - while( pRoot->bNomatch ){ - assert( pRoot->bEof==0 && rc==SQLITE_OK ); - rc = fts5ExprNodeNext(p, pRoot, 0, 0); - } + /* If not at EOF but the current rowid occurs earlier than iFirst in + ** the iteration order, move to document iFirst or later. */ + if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ + rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); + } + + /* If the iterator is not at a real match, skip forward until it is. */ + while( pRoot->bNomatch ){ + assert( pRoot->bEof==0 && rc==SQLITE_OK ); + rc = fts5ExprNodeNext(p, pRoot, 0, 0); } return rc; } @@ -1444,6 +1447,21 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( sqlite3Fts5ParseNearsetFree(pNear); sqlite3Fts5ParsePhraseFree(pPhrase); }else{ + if( pRet->nPhrase>0 ){ + Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; + assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); + if( pPhrase->nTerm==0 ){ + fts5ExprPhraseFree(pPhrase); + pRet->nPhrase--; + pParse->nPhrase--; + pPhrase = pLast; + }else if( pLast->nTerm==0 ){ + fts5ExprPhraseFree(pLast); + pParse->apPhrase[pParse->nPhrase-2] = pPhrase; + pParse->nPhrase--; + pRet->nPhrase--; + } + } pRet->apPhrase[pRet->nPhrase++] = pPhrase; } return pRet; @@ -1475,9 +1493,9 @@ static int fts5ParseTokenize( /* If an error has already occurred, this is a no-op */ if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; - assert( pPhrase==0 || pPhrase->nTerm>0 ); - if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ + if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); @@ -1578,7 +1596,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( pParse->rc = rc; fts5ExprPhraseFree(sCtx.pPhrase); sCtx.pPhrase = 0; - }else if( sCtx.pPhrase ){ + }else{ if( pAppend==0 ){ if( (pParse->nPhrase % 8)==0 ){ @@ -1595,9 +1613,14 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( pParse->nPhrase++; } + if( sCtx.pPhrase==0 ){ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + }else if( sCtx.pPhrase->nTerm ){ + sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; + } pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; - assert( sCtx.pPhrase->nTerm>0 ); - sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; } return sCtx.pPhrase; @@ -1614,7 +1637,6 @@ int sqlite3Fts5ExprClonePhrase( ){ int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - int i; /* Used to iterate through phrase terms */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ @@ -1632,19 +1654,37 @@ int sqlite3Fts5ExprClonePhrase( pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); } + if( rc==SQLITE_OK ){ + Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; + if( pColsetOrig ){ + int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); + if( pColset ){ + memcpy(pColset, pColsetOrig, nByte); + } + pNew->pRoot->pNear->pColset = pColset; + } + } - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), - 0, 0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), + 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } if( rc==SQLITE_OK ){ @@ -1693,23 +1733,25 @@ void sqlite3Fts5ParseSetDistance( Fts5ExprNearset *pNear, Fts5Token *p ){ - int nNear = 0; - int i; - if( p->n ){ - for(i=0; in; i++){ - char c = (char)p->p[i]; - if( c<'0' || c>'9' ){ - sqlite3Fts5ParseError( - pParse, "expected integer, got \"%.*s\"", p->n, p->p - ); - return; + if( pNear ){ + int nNear = 0; + int i; + if( p->n ){ + for(i=0; in; i++){ + char c = (char)p->p[i]; + if( c<'0' || c>'9' ){ + sqlite3Fts5ParseError( + pParse, "expected integer, got \"%.*s\"", p->n, p->p + ); + return; + } + nNear = nNear * 10 + (p->p[i] - '0'); } - nNear = nNear * 10 + (p->p[i] - '0'); + }else{ + nNear = FTS5_DEFAULT_NEARDIST; } - }else{ - nNear = FTS5_DEFAULT_NEARDIST; + pNear->nNear = nNear; } - pNear->nNear = nNear; } /* @@ -1757,6 +1799,34 @@ static Fts5Colset *fts5ParseColset( return pNew; } +/* +** Allocate and return an Fts5Colset object specifying the inverse of +** the colset passed as the second argument. Free the colset passed +** as the second argument before returning. +*/ +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ + Fts5Colset *pRet; + int nCol = pParse->pConfig->nCol; + + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, + sizeof(Fts5Colset) + sizeof(int)*nCol + ); + if( pRet ){ + int i; + int iOld = 0; + for(i=0; i=p->nCol || p->aiCol[iOld]!=i ){ + pRet->aiCol[pRet->nCol++] = i; + }else{ + iOld++; + } + } + } + + sqlite3_free(p); + return pRet; +} + Fts5Colset *sqlite3Fts5ParseColset( Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ Fts5Colset *pColset, /* Existing colset object */ @@ -1896,10 +1966,14 @@ Fts5ExprNode *sqlite3Fts5ParseNode( int iPhrase; for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; + if( pNear->apPhrase[iPhrase]->nTerm==0 ){ + pRet->xNext = 0; + pRet->eType = FTS5_EOF; + } } if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL - && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm!=1) + && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1) ){ assert( pParse->rc==SQLITE_OK ); pParse->rc = SQLITE_ERROR; @@ -1928,6 +2002,70 @@ Fts5ExprNode *sqlite3Fts5ParseNode( return pRet; } +Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( + Fts5Parse *pParse, /* Parse context */ + Fts5ExprNode *pLeft, /* Left hand child expression */ + Fts5ExprNode *pRight /* Right hand child expression */ +){ + Fts5ExprNode *pRet = 0; + Fts5ExprNode *pPrev; + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pLeft); + sqlite3Fts5ParseNodeFree(pRight); + }else{ + + assert( pLeft->eType==FTS5_STRING + || pLeft->eType==FTS5_TERM + || pLeft->eType==FTS5_EOF + || pLeft->eType==FTS5_AND + ); + assert( pRight->eType==FTS5_STRING + || pRight->eType==FTS5_TERM + || pRight->eType==FTS5_EOF + ); + + if( pLeft->eType==FTS5_AND ){ + pPrev = pLeft->apChild[pLeft->nChild-1]; + }else{ + pPrev = pLeft; + } + assert( pPrev->eType==FTS5_STRING + || pPrev->eType==FTS5_TERM + || pPrev->eType==FTS5_EOF + ); + + if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); + sqlite3Fts5ParseNodeFree(pRight); + pRet = pLeft; + pParse->nPhrase--; + } + else if( pPrev->eType==FTS5_EOF ){ + Fts5ExprPhrase **ap; + + if( pPrev==pLeft ){ + pRet = pRight; + }else{ + pLeft->apChild[pLeft->nChild-1] = pRight; + pRet = pLeft; + } + + ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase]; + assert( ap[0]==pPrev->pNear->apPhrase[0] ); + memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase); + pParse->nPhrase--; + + sqlite3Fts5ParseNodeFree(pPrev); + } + else{ + pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0); + } + } + + return pRet; +} + static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ int nByte = 0; Fts5ExprTerm *p; @@ -2062,6 +2200,9 @@ static char *fts5ExprPrintTcl( static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ char *zRet = 0; + if( pExpr->eType==0 ){ + return sqlite3_mprintf("\"\""); + }else if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ Fts5ExprNearset *pNear = pExpr->pNear; int i; @@ -2122,7 +2263,7 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ zRet = 0; }else{ int e = pExpr->apChild[i]->eType; - int b = (e!=FTS5_STRING && e!=FTS5_TERM); + int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); zRet = fts5PrintfAppend(zRet, "%s%s%z%s", (i==0 ? "" : zOp), (b?"(":""), z, (b?")":"") @@ -2400,12 +2541,13 @@ static int fts5ExprPopulatePoslistsCb( UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ Fts5ExprTerm *pTerm; if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = strlen(pTerm->zTerm); + int nTerm = (int)strlen(pTerm->zTerm); if( (nTerm==nToken || (nTermbPrefix)) && memcmp(pTerm->zTerm, pToken, nTerm)==0 ){ @@ -2509,17 +2651,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); } -static void fts5ExprClearEof(Fts5ExprNode *pNode){ - int i; - for(i=0; inChild; i++){ - fts5ExprClearEof(pNode->apChild[i]); - } - pNode->bEof = 0; -} -void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ - fts5ExprClearEof(pExpr->pRoot); -} - /* ** This function is only called for detail=columns tables. */ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index ada8bb16..afa2a307 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -345,11 +345,11 @@ int sqlite3Fts5HashWrite( if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); - p->iCol = iCol; + p->iCol = (i16)iCol; p->iPos = 0; }else{ bNew = 1; - p->iCol = iPos = iCol; + p->iCol = (i16)(iPos = iCol); } } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index c40ffbcb..46517e1a 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -304,6 +304,10 @@ struct Fts5Index { sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ + + sqlite3_stmt *pDataVersion; + i64 iStructVersion; /* data_version when pStruct read */ + Fts5Structure *pStruct; /* Current db structure (or NULL) */ }; struct Fts5DoclistIter { @@ -706,6 +710,18 @@ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = fts5DataRead(p, iRowid); + if( pRet ){ + if( pRet->szLeaf>pRet->nn ){ + p->rc = FTS5_CORRUPT; + fts5DataRelease(pRet); + pRet = 0; + } + } + return pRet; +} + static int fts5IndexPrepareStmt( Fts5Index *p, sqlite3_stmt **ppStmt, @@ -865,7 +881,7 @@ static int fts5StructureDecode( for(iLvl=0; rc==SQLITE_OK && iLvlaLevel[iLvl]; - int nTotal; + int nTotal = 0; int iSeg; if( i>=nData ){ @@ -958,6 +974,50 @@ static void fts5StructureExtendLevel( } } +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ + Fts5Structure *pRet = 0; + Fts5Config *pConfig = p->pConfig; + int iCookie; /* Configuration cookie */ + Fts5Data *pData; + + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc==SQLITE_OK ){ + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); + if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + fts5DataRelease(pData); + if( p->rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + return pRet; +} + +static i64 fts5IndexDataVersion(Fts5Index *p){ + i64 iVersion = 0; + + if( p->rc==SQLITE_OK ){ + if( p->pDataVersion==0 ){ + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, + sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) + ); + if( p->rc ) return 0; + } + + if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ + iVersion = sqlite3_column_int64(p->pDataVersion, 0); + } + p->rc = sqlite3_reset(p->pDataVersion); + } + + return iVersion; +} + /* ** Read, deserialize and return the structure record. ** @@ -970,26 +1030,49 @@ static void fts5StructureExtendLevel( ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p){ - Fts5Config *pConfig = p->pConfig; - Fts5Structure *pRet = 0; /* Object to return */ - int iCookie; /* Configuration cookie */ - Fts5Data *pData; - pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); - if( p->rc ) return 0; - /* TODO: Do we need this if the leaf-index is appended? Probably... */ - memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); - p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->pStruct==0 ){ + p->iStructVersion = fts5IndexDataVersion(p); + if( p->rc==SQLITE_OK ){ + p->pStruct = fts5StructureReadUncached(p); + } } - fts5DataRelease(pData); - if( p->rc!=SQLITE_OK ){ - fts5StructureRelease(pRet); - pRet = 0; +#if 0 + else{ + Fts5Structure *pTest = fts5StructureReadUncached(p); + if( pTest ){ + int i, j; + assert_nc( p->pStruct->nSegment==pTest->nSegment ); + assert_nc( p->pStruct->nLevel==pTest->nLevel ); + for(i=0; inLevel; i++){ + assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); + assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); + for(j=0; jaLevel[i].nSeg; j++){ + Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; + Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; + assert_nc( p1->iSegid==p2->iSegid ); + assert_nc( p1->pgnoFirst==p2->pgnoFirst ); + assert_nc( p1->pgnoLast==p2->pgnoLast ); + } + } + fts5StructureRelease(pTest); + } + } +#endif + + if( p->rc!=SQLITE_OK ) return 0; + assert( p->iStructVersion!=0 ); + assert( p->pStruct!=0 ); + fts5StructureRef(p->pStruct); + return p->pStruct; +} + +static void fts5StructureInvalidate(Fts5Index *p){ + if( p->pStruct ){ + fts5StructureRelease(p->pStruct); + p->pStruct = 0; } - return pRet; } /* @@ -1447,7 +1530,7 @@ static void fts5SegIterNextPage( pIter->pLeaf = pIter->pNextLeaf; pIter->pNextLeaf = 0; }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ - pIter->pLeaf = fts5DataRead(p, + pIter->pLeaf = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) ); }else{ @@ -1950,9 +2033,8 @@ static void fts5SegIterNext( if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist - ); + ); } - } else if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( @@ -2154,6 +2236,10 @@ static void fts5LeafSeek( iPgidx = szLeaf; iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; + if( iOff>n ){ + p->rc = FTS5_CORRUPT; + return; + } while( 1 ){ @@ -2193,6 +2279,11 @@ static void fts5LeafSeek( iTermOff += nKeep; iOff = iTermOff; + if( iOff>=n ){ + p->rc = FTS5_CORRUPT; + return; + } + /* Read the nKeep field of the next term. */ fts5FastGetVarint32(a, iOff, nKeep); } @@ -2245,6 +2336,18 @@ static void fts5LeafSeek( fts5SegIterLoadNPos(p, pIter); } +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + } + return p->pIdxSelect; +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. @@ -2262,6 +2365,7 @@ static void fts5SegIterSeekInit( int iPg = 1; int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ + sqlite3_stmt *pIdxSelect = 0; assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); @@ -2270,23 +2374,16 @@ static void fts5SegIterSeekInit( /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ - if( p->pIdxSelect==0 ){ - Fts5Config *pConfig = p->pConfig; - fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( - "SELECT pgno FROM '%q'.'%q_idx' WHERE " - "segid=? AND term<=? ORDER BY term DESC LIMIT 1", - pConfig->zDb, pConfig->zName - )); - } + pIdxSelect = fts5IdxSelectStmt(p); if( p->rc ) return; - sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); - sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ - i64 val = sqlite3_column_int(p->pIdxSelect, 0); + sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ + i64 val = sqlite3_column_int(pIdxSelect, 0); iPg = (int)(val>>1); bDlidx = (val & 0x0001); } - p->rc = sqlite3_reset(p->pIdxSelect); + p->rc = sqlite3_reset(pIdxSelect); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; @@ -2743,6 +2840,7 @@ static void fts5MultiIterNext( i64 iFrom /* Advance at least as far as this */ ){ int bUseFrom = bFrom; + assert( pIter->base.bEof==0 ); while( p->rc==SQLITE_OK ){ int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; @@ -3113,6 +3211,15 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ } } +/* +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match +** against no columns at all). +*/ +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + UNUSED_PARAM(pSeg); + pIter->base.nData = 0; +} + /* ** xSetOutputs callback used by detail=col when there is a column filter ** and there are 100 or more columns. Also called as a fallback from @@ -3162,7 +3269,7 @@ static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ if( aiCol==aiColEnd ) goto setoutputs_col_out; } if( *aiCol==iPrev ){ - *aOut++ = (iPrev - iPrevOut) + 2; + *aOut++ = (u8)((iPrev - iPrevOut) + 2); iPrevOut = iPrev; } } @@ -3218,6 +3325,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } + else if( pIter->pColset->nCol==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; + } + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; } @@ -3448,18 +3559,46 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ - while( iSegid==0 ){ - int iLvl, iSeg; - sqlite3_randomness(sizeof(u32), (void*)&iSegid); - iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ - iSegid = 0; - } + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT ){ + aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); } } } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & (1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); + + { + sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc==SQLITE_OK ){ + u8 aBlob[2] = {0xff, 0xff}; + sqlite3_bind_int(pIdxSelect, 1, iSegid); + sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); + assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); + p->rc = sqlite3_reset(pIdxSelect); + } + } +#endif } } @@ -3705,6 +3844,9 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; +static int nCall = 0; +nCall++; + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ @@ -3904,7 +4046,9 @@ static void fts5WriteFinish( fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; - fts5WriteFlushBtree(p, pWriter); + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); @@ -4174,13 +4318,17 @@ static void fts5IndexMergeLevel( /* ** Do up to nPg pages of automerge work on the index. +** +** Return true if any changes were actually made, or false otherwise. */ -static void fts5IndexMerge( +static int fts5IndexMerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ - int nPg /* Pages of work to do */ + int nPg, /* Pages of work to do */ + int nMin /* Minimum number of segments to merge */ ){ int nRem = nPg; + int bRet = 0; Fts5Structure *pStruct = *ppStruct; while( nRem>0 && p->rc==SQLITE_OK ){ int iLvl; /* To iterate through levels */ @@ -4211,17 +4359,17 @@ static void fts5IndexMerge( } #endif - if( nBestpConfig->nAutomerge - && pStruct->aLevel[iBestLvl].nMerge==0 - ){ + if( nBestaLevel[iBestLvl].nMerge==0 ){ break; } + bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } } *ppStruct = pStruct; + return bRet; } /* @@ -4249,7 +4397,7 @@ static void fts5IndexAutomerge( pStruct->nWriteCounter += nLeaf; nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); - fts5IndexMerge(p, ppStruct, nRem); + fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); } } @@ -4319,6 +4467,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); + fts5StructureInvalidate(p); if( iSegid ){ const int pgsz = p->pConfig->pgsz; @@ -4469,28 +4618,41 @@ static void fts5IndexFlush(Fts5Index *p){ } } - -int sqlite3Fts5IndexOptimize(Fts5Index *p){ - Fts5Structure *pStruct; +static Fts5Structure *fts5IndexOptimizeStruct( + Fts5Index *p, + Fts5Structure *pStruct +){ Fts5Structure *pNew = 0; - int nSeg = 0; + int nByte = sizeof(Fts5Structure); + int nSeg = pStruct->nSegment; + int i; - assert( p->rc==SQLITE_OK ); - fts5IndexFlush(p); - pStruct = fts5StructureRead(p); - - if( pStruct ){ - assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); - nSeg = pStruct->nSegment; - if( nSeg>1 ){ - int nByte = sizeof(Fts5Structure); - nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); - pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); + /* Figure out if this structure requires optimization. A structure does + ** not require optimization if either: + ** + ** + it consists of fewer than two segments, or + ** + all segments are on the same level, or + ** + all segments except one are currently inputs to a merge operation. + ** + ** In the first case, return NULL. In the second, increment the ref-count + ** on *pStruct and return a copy of the pointer to it. + */ + if( nSeg<2 ) return 0; + for(i=0; inLevel; i++){ + int nThis = pStruct->aLevel[i].nSeg; + if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + fts5StructureRef(pStruct); + return pStruct; } + assert( pStruct->aLevel[i].nMerge<=nThis ); } + + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pNew ){ Fts5StructureLevel *pLvl; - int nByte = nSeg * sizeof(Fts5StructureSegment); + nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; @@ -4499,7 +4661,10 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ if( pLvl->aSeg ){ int iLvl, iSeg; int iSegOut = 0; - for(iLvl=0; iLvlnLevel; iLvl++){ + /* Iterate through all segments, from oldest to newest. Add them to + ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest + ** segment in the data structure. */ + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; iSegOut++; @@ -4512,8 +4677,27 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ } } + return pNew; +} + +int sqlite3Fts5IndexOptimize(Fts5Index *p){ + Fts5Structure *pStruct; + Fts5Structure *pNew = 0; + + assert( p->rc==SQLITE_OK ); + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + fts5StructureInvalidate(p); + + if( pStruct ){ + pNew = fts5IndexOptimizeStruct(p, pStruct); + } + fts5StructureRelease(pStruct); + + assert( pNew==0 || pNew->nSegment>0 ); if( pNew ){ - int iLvl = pNew->nLevel-1; + int iLvl; + for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){} while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ int nRem = FTS5_OPT_WORK_UNIT; fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); @@ -4523,20 +4707,32 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ fts5StructureRelease(pNew); } - fts5StructureRelease(pStruct); return fts5IndexReturn(p); } +/* +** This is called to implement the special "VALUES('merge', $nMerge)" +** INSERT command. +*/ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct; - - pStruct = fts5StructureRead(p); - if( pStruct && pStruct->nLevel ){ - fts5IndexMerge(p, &pStruct, nMerge); - fts5StructureWrite(p, pStruct); + Fts5Structure *pStruct = fts5StructureRead(p); + if( pStruct ){ + int nMin = p->pConfig->nUsermerge; + fts5StructureInvalidate(p); + if( nMerge<0 ){ + Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); + fts5StructureRelease(pStruct); + pStruct = pNew; + nMin = 2; + nMerge = nMerge*-1; + } + if( pStruct && pStruct->nLevel ){ + if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ + fts5StructureWrite(p, pStruct); + } + } + fts5StructureRelease(pStruct); } - fts5StructureRelease(pStruct); - return fts5IndexReturn(p); } @@ -4950,6 +5146,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); + fts5StructureInvalidate(p); /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } @@ -4961,6 +5158,7 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; + fts5StructureInvalidate(p); memset(&s, 0, sizeof(Fts5Structure)); fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); @@ -5019,11 +5217,13 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); + fts5StructureInvalidate(p); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pDataVersion); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -6279,3 +6479,12 @@ int sqlite3Fts5IndexInit(sqlite3 *db){ } return rc; } + + +int sqlite3Fts5IndexReset(Fts5Index *p){ + assert( p->pStruct==0 || p->iStructVersion!=0 ); + if( fts5IndexDataVersion(p)!=p->iStructVersion ){ + fts5StructureInvalidate(p); + } + return fts5IndexReturn(p); +} diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index fe1d72e2..384d3dd8 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -597,27 +597,38 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_OK; } +static int fts5NewTransaction(Fts5Table *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; + } + return sqlite3Fts5StorageReset(pTab->pStorage); +} + /* ** Implementation of xOpen method. */ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; - Fts5Cursor *pCsr; /* New cursor object */ + Fts5Cursor *pCsr = 0; /* New cursor object */ int nByte; /* Bytes of space to allocate */ - int rc = SQLITE_OK; /* Return code */ + int rc; /* Return code */ - nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); - pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); - if( pCsr ){ - Fts5Global *pGlobal = pTab->pGlobal; - memset(pCsr, 0, nByte); - pCsr->aColumnSize = (int*)&pCsr[1]; - pCsr->pNext = pGlobal->pCsr; - pGlobal->pCsr = pCsr; - pCsr->iCsrId = ++pGlobal->iNextId; - }else{ - rc = SQLITE_NOMEM; + rc = fts5NewTransaction(pTab); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); + pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); + if( pCsr ){ + Fts5Global *pGlobal = pTab->pGlobal; + memset(pCsr, 0, nByte); + pCsr->aColumnSize = (int*)&pCsr[1]; + pCsr->pNext = pGlobal->pCsr; + pGlobal->pCsr = pCsr; + pCsr->iCsrId = ++pGlobal->iNextId; + }else{ + rc = SQLITE_NOMEM; + } } *ppCsr = (sqlite3_vtab_cursor*)pCsr; return rc; @@ -1175,7 +1186,6 @@ static int fts5FilterMethod( pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); - sqlite3Fts5ExprClearEof(pCsr->pExpr); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; @@ -1511,13 +1521,13 @@ static int fts5UpdateMethod( rc = SQLITE_ERROR; } - /* Case 1: DELETE */ + /* DELETE */ else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); } - /* Case 2: INSERT */ + /* INSERT */ else if( eType0!=SQLITE_INTEGER ){ /* If this is a REPLACE, first remove the current entry (if any) */ if( eConflict==SQLITE_REPLACE @@ -1529,7 +1539,7 @@ static int fts5UpdateMethod( fts5StorageInsert(&rc, pTab, apVal, pRowid); } - /* Case 2: UPDATE */ + /* UPDATE */ else{ i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ @@ -1578,8 +1588,8 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); + fts5NewTransaction((Fts5Table*)pVtab); return SQLITE_OK; } @@ -2665,6 +2675,17 @@ static int fts5Init(sqlite3 *db){ ); } } + + /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file + ** fts5_test_mi.c is compiled and linked into the executable. And call + ** its entry point to enable the matchinfo() demo. */ +#ifdef SQLITE_FTS5_ENABLE_TEST_MI + if( rc==SQLITE_OK ){ + extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); + rc = sqlite3Fts5TestRegisterMatchinfo(db); + } +#endif + return rc; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index e4f5dd8f..a6958874 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -145,6 +145,7 @@ static int fts5StorageGetStmt( } *ppStmt = p->aStmt[eStmt]; + sqlite3_reset(*ppStmt); return rc; } @@ -246,7 +247,11 @@ int sqlite3Fts5CreateTable( char *zErr = 0; rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", - pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" + pConfig->zDb, pConfig->zName, zPost, zDefn, +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID + bWithout?" WITHOUT ROWID": +#endif + "" ); if( zErr ){ *pzErr = sqlite3_mprintf( @@ -368,6 +373,7 @@ static int fts5StorageInsertCallback( Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } @@ -639,6 +645,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ return sqlite3Fts5IndexMerge(p->pIndex, nMerge); } +int sqlite3Fts5StorageReset(Fts5Storage *p){ + return sqlite3Fts5IndexReset(p->pIndex); +} + /* ** Allocate a new rowid. This is used for "external content" tables when ** a NULL value is inserted into the rowid column. The new rowid is allocated @@ -810,6 +820,7 @@ static int fts5StorageIntegrityCallback( int iCol; UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; @@ -1121,5 +1132,3 @@ int sqlite3Fts5StorageConfigValue( } return rc; } - - diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 72db6577..5fe690f6 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -14,7 +14,14 @@ #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif #ifdef SQLITE_ENABLE_FTS5 @@ -78,7 +85,7 @@ static int f5tResultToErrorCode(const char *zRes){ return SQLITE_ERROR; } -static int f5tDbAndApi( +static int SQLITE_TCLAPI f5tDbAndApi( Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb, @@ -164,7 +171,7 @@ static int xTokenizeCb( return rc; } -static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); +static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); static int xQueryPhraseCb( const Fts5ExtensionApi *pApi, @@ -209,7 +216,7 @@ static void xSetAuxdataDestructor(void *p){ ** ** Description... */ -static int xF5tApi( +static int SQLITE_TCLAPI xF5tApi( void * clientData, Tcl_Interp *interp, int objc, @@ -602,7 +609,7 @@ static void xF5tDestroy(void *pCtx){ ** ** Description... */ -static int f5tCreateFunction( +static int SQLITE_TCLAPI f5tCreateFunction( void * clientData, Tcl_Interp *interp, int objc, @@ -672,7 +679,7 @@ static int xTokenizeCb2( ** ** Description... */ -static int f5tTokenize( +static int SQLITE_TCLAPI f5tTokenize( void * clientData, Tcl_Interp *interp, int objc, @@ -878,7 +885,7 @@ static int f5tTokenizerTokenize( /* ** sqlite3_fts5_token ?-colocated? TEXT START END */ -static int f5tTokenizerReturn( +static int SQLITE_TCLAPI f5tTokenizerReturn( void * clientData, Tcl_Interp *interp, int objc, @@ -949,7 +956,7 @@ static void f5tDelTokenizer(void *pCtx){ ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each ** token within the tokenized text. */ -static int f5tCreateTokenizer( +static int SQLITE_TCLAPI f5tCreateTokenizer( ClientData clientData, Tcl_Interp *interp, int objc, @@ -992,7 +999,7 @@ static int f5tCreateTokenizer( return TCL_OK; } -static void xF5tFree(ClientData clientData){ +static void SQLITE_TCLAPI xF5tFree(ClientData clientData){ ckfree(clientData); } @@ -1001,7 +1008,7 @@ static void xF5tFree(ClientData clientData){ ** ** Set or clear the global "may-be-corrupt" flag. Return the old value. */ -static int f5tMayBeCorrupt( +static int SQLITE_TCLAPI f5tMayBeCorrupt( void * clientData, Tcl_Interp *interp, int objc, @@ -1033,7 +1040,7 @@ static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){ return (h % nSlot); } -static int f5tTokenHash( +static int SQLITE_TCLAPI f5tTokenHash( void * clientData, Tcl_Interp *interp, int objc, @@ -1058,7 +1065,7 @@ static int f5tTokenHash( return TCL_OK; } -static int f5tRegisterMatchinfo( +static int SQLITE_TCLAPI f5tRegisterMatchinfo( void * clientData, Tcl_Interp *interp, int objc, @@ -1083,7 +1090,7 @@ static int f5tRegisterMatchinfo( return TCL_OK; } -static int f5tRegisterTok( +static int SQLITE_TCLAPI f5tRegisterTok( void * clientData, Tcl_Interp *interp, int objc, diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 28331773..a905b85b 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -41,16 +41,17 @@ */ -#ifdef SQLITE_TEST #ifdef SQLITE_ENABLE_FTS5 #include "fts5.h" -#include #include #include typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx; + +#ifndef SQLITE_AMALGAMATION typedef unsigned int u32; +#endif struct Fts5MatchinfoCtx { int nCol; /* Number of cols in FTS5 table */ @@ -67,18 +68,22 @@ struct Fts5MatchinfoCtx { ** If an error occurs, return NULL and leave an error in the database ** handle (accessible using sqlite3_errcode()/errmsg()). */ -static fts5_api *fts5_api_from_db(sqlite3 *db){ - fts5_api *pRet = 0; +static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){ sqlite3_stmt *pStmt = 0; + int rc; - if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0) - && SQLITE_ROW==sqlite3_step(pStmt) - && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0) - ){ - memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet)); + *ppApi = 0; + rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) + && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0) + ){ + memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*)); + } + rc = sqlite3_finalize(pStmt); } - sqlite3_finalize(pStmt); - return pRet; + + return rc; } @@ -244,11 +249,7 @@ static int fts5MatchinfoLocalCb( iOff>=0; pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) ){ - if( f=='b' ){ - aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); - }else{ - aOut[nMul * (iCol + iPhrase * p->nCol)]++; - } + aOut[nMul * (iCol + iPhrase * p->nCol)]++; } } @@ -402,14 +403,15 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ /* Extract the FTS5 API pointer from the database handle. The ** fts5_api_from_db() function above is copied verbatim from the ** FTS5 documentation. Refer there for details. */ - pApi = fts5_api_from_db(db); + rc = fts5_api_from_db(db, &pApi); + if( rc!=SQLITE_OK ) return rc; /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** ** Also check that the fts5_api object is version 2 or newer. */ - if( pApi==0 || pApi->iVersion<1 ){ + if( pApi==0 || pApi->iVersion<2 ){ return SQLITE_ERROR; } @@ -420,5 +422,4 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ } #endif /* SQLITE_ENABLE_FTS5 */ -#endif /* SQLITE_TEST */ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index f3a2381c..82c7dc90 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -279,8 +279,19 @@ static int fts5VocabBestIndexMethod( } } - pInfo->idxNum = idxNum; + /* This virtual table always delivers results in ascending order of + ** the "term" column (column 0). So if the user has requested this + ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the + ** sqlite3_index_info.orderByConsumed flag to tell the core the results + ** are already in sorted order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + pInfo->idxNum = idxNum; return SQLITE_OK; } diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y index 43936767..1cc4b88e 100644 --- a/ext/fts5/fts5parse.y +++ b/ext/fts5/fts5parse.y @@ -34,7 +34,6 @@ ); } %stack_overflow { - UNUSED_PARAM(yypMinor); /* Silence a compiler warning */ sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow"); } @@ -105,7 +104,7 @@ expr(A) ::= exprlist(X). {A = X;} exprlist(A) ::= cnearset(X). {A = X;} exprlist(A) ::= exprlist(X) cnearset(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0); + A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y); } cnearset(A) ::= nearset(X). { @@ -121,10 +120,17 @@ cnearset(A) ::= colset(X) COLON nearset(Y). { %type colsetlist {Fts5Colset*} %destructor colsetlist { sqlite3_free($$); } +colset(A) ::= MINUS LCP colsetlist(X) RCP. { + A = sqlite3Fts5ParseColsetInvert(pParse, X); +} colset(A) ::= LCP colsetlist(X) RCP. { A = X; } colset(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); } +colset(A) ::= MINUS STRING(X). { + A = sqlite3Fts5ParseColset(pParse, 0, &X); + A = sqlite3Fts5ParseColsetInvert(pParse, A); +} colsetlist(A) ::= colsetlist(Y) STRING(X). { A = sqlite3Fts5ParseColset(pParse, Y, &X); } @@ -132,7 +138,6 @@ colsetlist(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); } - %type nearset {Fts5ExprNearset*} %type nearphrases {Fts5ExprNearset*} %destructor nearset { sqlite3Fts5ParseNearsetFree($$); } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index cf688dcc..0f371dcf 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -16,8 +16,13 @@ if {![info exists testdir]} { source $testdir/tester.tcl ifcapable !fts5 { - finish_test + proc return_if_no_fts5 {} { + finish_test + return -code return + } return +} else { + proc return_if_no_fts5 {} {} } catch { @@ -25,12 +30,6 @@ catch { reset_db } -# If SQLITE_ENABLE_FTS5 is not defined, skip this test. -ifcapable !fts5 { - finish_test - return -} - proc fts5_test_poslist {cmd} { set res [list] for {set i 0} {$i < [$cmd xInstCount]} {incr i} { @@ -160,6 +159,12 @@ proc fts5_aux_test_functions {db} { } } +proc fts5_segcount {tbl} { + set N 0 + foreach n [fts5_level_segs $tbl] { incr N $n } + set N +} + proc fts5_level_segs {tbl} { set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10" set ret [list] diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index fdcf0839..428ca6c1 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -432,9 +432,16 @@ proc funk {} { } db func funk funk +# This test case corrupts the structure record within the first invocation +# of function funk(). Which used to cause the bm25() function to throw an +# exception. But since bm25() can now used the cached structure record, +# it never sees the corruption introduced by funk() and so the following +# statement no longer fails. +# do_catchsql_test 16.2 { SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' -} {1 {SQL logic error or missing database}} +} {0 {{} -1e-06 {}}} +# {1 {SQL logic error or missing database}} #------------------------------------------------------------------------- # diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index d6b22415..6aab55a5 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -72,42 +72,56 @@ foreach {tn doc res} { 2.2 {o X o o o o o o} {o [X] o o o o o...} 2.3 {o o X o o o o o} {o o [X] o o o o...} 2.4 {o o o X o o o o} {o o o [X] o o o...} - 2.5 {o o o o X o o o} {...o o o [X] o o o} - 2.6 {o o o o o X o o} {...o o o o [X] o o} - 2.7 {o o o o o o X o} {...o o o o o [X] o} + 2.5 {o o o o X o o o} {o o o o [X] o o...} + 2.6 {o o o o o X o o} {o o o o o [X] o...} + 2.7 {o o o o o o X o} {o o o o o o [X]...} 2.8 {o o o o o o o X} {...o o o o o o [X]} + 2.9 {o o o o o o o X o} {...o o o o o [X] o} + 2.10 {o o o o o o o X o o} {...o o o o [X] o o} + 2.11 {o o o o o o o X o o o} {...o o o [X] o o o} + 2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...} + + 3.1 {X o o o o o o o o} {[X] o o o o o o...} 3.2 {o X o o o o o o o} {o [X] o o o o o...} 3.3 {o o X o o o o o o} {o o [X] o o o o...} 3.4 {o o o X o o o o o} {o o o [X] o o o...} - 3.5 {o o o o X o o o o} {...o o o [X] o o o...} - 3.6 {o o o o o X o o o} {...o o o [X] o o o} - 3.7 {o o o o o o X o o} {...o o o o [X] o o} - 3.8 {o o o o o o o X o} {...o o o o o [X] o} - 3.9 {o o o o o o o o X} {...o o o o o o [X]} + + 3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...} + 3.6 {o o o o o o o o X o o o} {...o o o [X] o o o} + 3.7 {o o o o o o o o o X o o} {...o o o o [X] o o} + 3.8 {o o o o o o o o o o X o} {...o o o o o [X] o} + 3.9 {o o o o o o o o o o o X} {...o o o o o o [X]} 4.1 {X o o o o o X o o} {[X] o o o o o [X]...} - 4.2 {o X o o o o o X o} {...[X] o o o o o [X]...} - 4.3 {o o X o o o o o X} {...[X] o o o o o [X]} + 4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...} + 4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]} 5.1 {X o o o o X o o o} {[X] o o o o [X] o...} - 5.2 {o X o o o o X o o} {...[X] o o o o [X] o...} - 5.3 {o o X o o o o X o} {...[X] o o o o [X] o} - 5.4 {o o o X o o o o X} {...o [X] o o o o [X]} + 5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...} + 5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o} + 5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]} 6.1 {X o o o X o o o} {[X] o o o [X] o o...} 6.2 {o X o o o X o o o} {o [X] o o o [X] o...} - 6.3 {o o X o o o X o o} {...o [X] o o o [X] o...} - 6.4 {o o o X o o o X o} {...o [X] o o o [X] o} - 6.5 {o o o o X o o o X} {...o o [X] o o o [X]} + 6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...} + 6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o} + 6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]} 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} - 7.3 {o o X o o X o o o} {...o [X] o o [X] o o...} - 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} - 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} - 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} + 7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...} + 7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o} + 7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o} + 7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]} + + 8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o} + {...o o [X] [X] [X] o o...} + 8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} + {...o o [X] o o o o...} + 8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} + {o o o o [X] o o...} } { do_snippet_test 1.$tn $doc X $res } @@ -124,24 +138,43 @@ if {[detail_is_full]} { 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 2.3 {o o X Y o o o o} {o o [X Y] o o o...} - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} + 2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o} + 2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o} + 2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o} + 2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]} 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} + 3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...} + 3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o} + 3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o} + 3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o} + 3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]} } { do_snippet_test 2.$tn $doc "X + Y" $res } } +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13'); + SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz'); +} { + {1 2 3 4 5...} +} + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE p1 USING fts5(a, b); + INSERT INTO p1 VALUES( + 'x a a a a a a a a a a', + 'a a a a a a a a a a a a a a a a a a a x' + ); +} +do_execsql_test 5.1 { + SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); +} {{[x] a a a a a...}} + } ;# foreach_detail_mode finish_test diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test index 995fe857..fa3167a4 100644 --- a/ext/fts5/test/fts5aux.test +++ b/ext/fts5/test/fts5aux.test @@ -246,5 +246,37 @@ foreach {tn lRow res} { } $res } +#------------------------------------------------------------------------- +# Test the built-in bm25() demo. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('a', NULL); -- 1 + INSERT INTO t1 VALUES('a', NULL); -- 2 + INSERT INTO t1 VALUES('a', NULL); -- 3 + INSERT INTO t1 VALUES('a', NULL); -- 4 + INSERT INTO t1 VALUES('a', NULL); -- 5 + INSERT INTO t1 VALUES('a', NULL); -- 6 + INSERT INTO t1 VALUES('a', NULL); -- 7 + INSERT INTO t1 VALUES('a', NULL); -- 8 + INSERT INTO t1 VALUES(NULL, 'a a b'); -- 9 + INSERT INTO t1 VALUES(NULL, 'b b a'); -- 10 +} + +do_execsql_test 9.2 { + SELECT rowid FROM t1('a AND b') ORDER BY rank; +} { + 10 9 +} + +do_execsql_test 9.3 { + SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank; +} { + 9 10 +} + + + finish_test diff --git a/ext/fts5/test/fts5bigtok.test b/ext/fts5/test/fts5bigtok.test index 2267be50..9ccaf6c8 100644 --- a/ext/fts5/test/fts5bigtok.test +++ b/ext/fts5/test/fts5bigtok.test @@ -14,6 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5bigtok +return_if_no_fts5 proc rndterm {} { set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z] diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test new file mode 100644 index 00000000..e3333241 --- /dev/null +++ b/ext/fts5/test/fts5colset.test @@ -0,0 +1,59 @@ +# 2016 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5colset + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $::testprefix { + if {[detail_is_none]} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 + } + + foreach {tn q res} { + 1 "a" {1 2 3 4} + 2 "{a} : a" {1} + 3 "-{a} : a" {2 3 4} + 4 "- {a c} : a" {2 4} + 5 " - {d d c} : a" {1 2} + 6 "- {d c b a} : a" {} + 7 "-{\"a\"} : b" {1 2 3} + 8 "- c : a" {1 2 4} + 9 "-c : a" {1 2 4} + 10 "-\"c\" : a" {1 2 4} + } { + breakpoint + do_execsql_test 1.$tn { + SELECT rowid FROM t1($q) + } $res + } + + +} + + +finish_test + + diff --git a/ext/fts5/test/fts5config.test b/ext/fts5/test/fts5config.test index c30a5972..386d112e 100644 --- a/ext/fts5/test/fts5config.test +++ b/ext/fts5/test/fts5config.test @@ -247,5 +247,21 @@ do_catchsql_test 12.1 { INSERT INTO t1(t1, rank) VALUES('rank', NULL);; } {1 {SQL logic error or missing database}} +#------------------------------------------------------------------------- +# errors in the 'usermerge' option +# +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE tt USING fts5(ttt); +} +foreach {tn val} { + 1 -1 + 2 4.2 + 3 17 + 4 1 +} { + set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)" + do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}} +} + finish_test diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 3a4fcfaa..c10017a2 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -37,7 +37,7 @@ do_execsql_test 1.0 { } set mask [expr 31 << 31] -if 1 { +if 0 { # Test 1: # @@ -84,6 +84,8 @@ foreach {tno stmt} { } } +} + # Using the same database as the 1.* tests. # # Run N-1 tests, where N is the number of bytes in the rightmost leaf page @@ -212,8 +214,6 @@ foreach {tn nCut} { # do_test 4.$tn.x { expr $nCorrupt>0 } 1 } -} - set doc [string repeat "A B C " 1000] do_execsql_test 5.0 { CREATE VIRTUAL TABLE x5 USING fts5(tt); diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index 7a8cb5c4..9653bcaa 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -179,6 +179,10 @@ for {set i 1} {1} {incr i} { if {$end<=$i} break lset var end [expr $end - $i] set struct [binary format c* $var] + + db close + sqlite3 db test.db + db eval { BEGIN; UPDATE t1_data SET block = $struct WHERE id=10; diff --git a/ext/fts5/test/fts5determin.test b/ext/fts5/test/fts5determin.test new file mode 100644 index 00000000..e368c470 --- /dev/null +++ b/ext/fts5/test/fts5determin.test @@ -0,0 +1,67 @@ +# 2016 March 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, that the fts5 module is deterministic. At one point, when +# segment ids were allocated using sqlite3_randomness(), this was not the +# case. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aa +return_if_no_fts5 + +proc do_determin_test {tn} { + uplevel [list + do_execsql_test $tn { + SELECT (SELECT md5sum(id, block) FROM t1_data)== + (SELECT md5sum(id, block) FROM t2_data), + (SELECT md5sum(id, block) FROM t1_data)== + (SELECT md5sum(id, block) FROM t3_data) + } {1 1} + ] +} + +foreach_detail_mode $::testprefix { + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + } + + do_test 1.1 { + foreach t {t1 t2 t3} { + execsql [string map [list TBL $t] { + INSERT INTO TBL VALUES('a b c', 'd e f'); + INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3'); + INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz'); + }] + } + } {} + + do_determin_test 1.2 + + do_test 1.3 { + foreach t {t1 t2 t3} { + execsql [string map [list TBL $t] { + INSERT INTO TBL(TBL) VALUES('optimize'); + }] + } + } {} + + do_determin_test 1.4 +} + + +finish_test + + diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 232b5021..655beb98 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -178,7 +178,7 @@ do_execsql_test 3.2 { ORDER BY rowid DESC; } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} -do_execsql_test 3.2 { +do_execsql_test 3.3 { INSERT INTO abc(abc) VALUES('integrity-check'); INSERT INTO abc(abc) VALUES('optimize'); INSERT INTO abc(abc) VALUES('integrity-check'); @@ -187,7 +187,7 @@ do_execsql_test 3.2 { set v [lindex $vocab 0] set i 0 foreach v $vocab { - do_execsql_test 3.3.[incr i] { + do_execsql_test 3.4.[incr i] { SELECT rowid FROM abc WHERE abc MATCH $v } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} } diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 82053960..510a0d79 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -33,12 +33,12 @@ foreach {tn expr res} { 1 {abc} {"abc"} 2 {abc ""} {"abc"} 3 {""} {} - 4 {abc OR ""} {"abc"} - 5 {abc NOT ""} {"abc"} - 6 {abc AND ""} {"abc"} - 7 {"" OR abc} {"abc"} - 8 {"" NOT abc} {"abc"} - 9 {"" AND abc} {"abc"} + 4 {abc OR ""} {"abc" OR ""} + 5 {abc NOT ""} {"abc" NOT ""} + 6 {abc AND ""} {"abc" AND ""} + 7 {"" OR abc} {"" OR "abc"} + 8 {"" NOT abc} {"" NOT "abc"} + 9 {"" AND abc} {"" AND "abc"} 10 {abc + "" + def} {"abc" + "def"} 11 {abc "" def} {"abc" AND "def"} 12 {r+e OR w} {"r" + "e" OR "w"} @@ -63,6 +63,23 @@ do_catchsql_test 2.1 { SELECT fts5_expr_tcl() } {1 {wrong number of arguments to function fts5_expr_tcl}} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61'); + INSERT INTO e1 VALUES ("just a few words with a / inside"); +} +do_execsql_test 3.1 { + SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; +} {1 -1e-06} +do_execsql_test 3.2 { + SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"' +} 1 +do_execsql_test 3.3 { + SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; +} {1 -1e-06} + + + finish_test diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index acc43ebf..bfa54a5b 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -86,7 +86,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] do_faultsim_test 4 -faults oom-* -body { db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} } -test { - faultsim_test_result {0 {0 {} 4}} + faultsim_test_result {0 {0 {} 3}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test index ae584949..c613490e 100644 --- a/ext/fts5/test/fts5fault8.test +++ b/ext/fts5/test/fts5fault8.test @@ -54,7 +54,32 @@ foreach_detail_mode $testprefix { faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM} } } + } ;# foreach_detail_mode... + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x2 USING fts5(a); + INSERT INTO x2(x2, rank) VALUES('crisismerge', 2); + INSERT INTO x2(x2, rank) VALUES('pgsz', 32); + INSERT INTO x2 VALUES('a b c d'); + INSERT INTO x2 VALUES('e f g h'); + INSERT INTO x2 VALUES('i j k l'); + INSERT INTO x2 VALUES('m n o p'); + INSERT INTO x2 VALUES('q r s t'); + INSERT INTO x2 VALUES('u v w x'); + INSERT INTO x2 VALUES('y z a b'); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO x2(x2) VALUES('optimize') } +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} +} + + finish_test diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test new file mode 100644 index 00000000..40df8b60 --- /dev/null +++ b/ext/fts5/test/fts5faultB.test @@ -0,0 +1,83 @@ +# 2016 February 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultB + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + + +#------------------------------------------------------------------------- +# Errors while registering the matchinfo() demo function. +# +do_faultsim_test 1 -faults oom* -prep { + sqlite3 db test.db +} -body { + sqlite3_fts5_register_matchinfo db +} -test { + faultsim_test_result {0 {}} {1 SQLITE_ERROR} {1 SQLITE_NOMEM} +} + + +#------------------------------------------------------------------------- +# Errors while executing the matchinfo() demo function. +# +reset_db +sqlite3_fts5_register_matchinfo db +db func mit mit +do_execsql_test 2 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('x y z', '1 2 3'); + INSERT INTO t1 VALUES('x', '1 2 3 4 5 6 7'); +} + +do_faultsim_test 2.1 -faults oom* -body { + execsql { SELECT mit(matchinfo(t1, 'a')) FROM t1('x') } +} -test { + faultsim_test_result {0 {{2 5} {2 5}}} +} + +do_faultsim_test 2.2 -faults oom* -body { + execsql { SELECT mit(matchinfo(t1, 'l')) FROM t1('x') } +} -test { + faultsim_test_result {0 {{3 3} {1 7}}} +} + +do_execsql_test 2.3 { + INSERT INTO t1 VALUES('a b c d e f', 'a b d e f c'); + INSERT INTO t1 VALUES('l m b c a', 'n o a b c z'); +} + +do_faultsim_test 2.4 -faults oom* -body { + execsql { SELECT mit(matchinfo(t1, 's')) FROM t1('a b c') } +} -test { + faultsim_test_result {0 {{3 2} {2 3}}} +} + + +finish_test + diff --git a/ext/fts5/test/fts5fuzz1.test b/ext/fts5/test/fts5fuzz1.test new file mode 100644 index 00000000..599d7bcc --- /dev/null +++ b/ext/fts5/test/fts5fuzz1.test @@ -0,0 +1,93 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set testprefix fts5fuzz1 + + +#------------------------------------------------------------------------- +reset_db +do_catchsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a b); +} {/1 {parse error in.*}/} + + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE f1 USING fts5(a, b); + INSERT INTO f1 VALUES('a b', 'c d'); + INSERT INTO f1 VALUES('e f', 'a b'); +} + +do_execsql_test 2.2.1 { + SELECT rowid FROM f1('""'); +} {} + +do_execsql_test 2.2.2 { + SELECT rowid FROM f1('"" AND a'); +} {} + + +do_execsql_test 2.2.3 { + SELECT rowid FROM f1('"" a'); +} {1 2} + +do_execsql_test 2.2.4 { + SELECT rowid FROM f1('"" OR a'); +} {1 2} + +do_execsql_test 2.3 { + SELECT a, b FROM f1('NEAR("")'); +} {} + +do_execsql_test 2.4 { + SELECT a, b FROM f1('NEAR("", 5)'); +} {} + +do_execsql_test 2.5 { + SELECT a, b FROM f1('NEAR("" c, 5)'); +} {{a b} {c d}} + +do_execsql_test 2.6 { + SELECT a, b FROM f1('NEAR("" c d, 5)'); +} {{a b} {c d}} + +do_execsql_test 2.7 { + SELECT a, b FROM f1('NEAR(c d, 5)'); +} {{a b} {c d}} + +do_execsql_test 2.8 { + SELECT rowid FROM f1('NEAR("a" "b", 5)'); +} {1 2} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.2 { + CREATE VIRTUAL TABLE f2 USING fts5(o, t, tokenize="ascii separators abc"); + SELECT * FROM f2('a+4'); +} {} + + + +#------------------------------------------------------------------------- +reset_db +do_catchsql_test 4.1 { + CREATE VIRTUAL TABLE f2 USING fts5(o, t); + SELECT * FROM f2('(8 AND 9)`AND 10'); +} {1 {fts5: syntax error near "`"}} + +finish_test + diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 06f4550b..99b07677 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -467,5 +467,29 @@ do_execsql_test 12.1 { } ;# foreach_detail_mode +#------------------------------------------------------------------------- +# Test that a bad fts5() return is detected +# +reset_db +proc xyz {} {} +db func fts5 -argcount 0 xyz +do_test 13.1 { + list [catch { sqlite3_fts5_register_matchinfo db } msg] $msg +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +# Test that an invalid matchinfo() flag is detected +# +reset_db +sqlite3_fts5_register_matchinfo db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE x1 USING fts5(z); + INSERT INTO x1 VALUES('a b c a b c a b c'); +} {} + +do_catchsql_test 14.2 { + SELECT matchinfo(x1, 'd') FROM x1('a b c'); +} {1 {unrecognized matchinfo flag: d}} + finish_test diff --git a/ext/fts5/test/fts5merge.test b/ext/fts5/test/fts5merge.test index 9dd1ecd0..73e006a7 100644 --- a/ext/fts5/test/fts5merge.test +++ b/ext/fts5/test/fts5merge.test @@ -45,7 +45,7 @@ proc do_merge1_test {testname nRowPerSeg} { WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg) INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii; - INSERT INTO x8(x8, rank) VALUES('automerge', 2); + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); } for {set tn 1} {[lindex [fts5_level_segs x8] 0]>0} {incr tn} { @@ -84,9 +84,9 @@ proc do_merge2_test {testname nRow} { execsql { INSERT INTO x8 VALUES( rnddoc(($i%16) + 5) ) } while {[not_merged x8]} { execsql { - INSERT INTO x8(x8, rank) VALUES('automerge', 2); + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); INSERT INTO x8(x8, rank) VALUES('merge', 1); - INSERT INTO x8(x8, rank) VALUES('automerge', 16); + INSERT INTO x8(x8, rank) VALUES('usermerge', 16); INSERT INTO x8(x8) VALUES('integrity-check'); } } @@ -104,9 +104,9 @@ do_merge2_test 2.2 10 do_merge2_test 2.3 20 #------------------------------------------------------------------------- -# Test that an auto-merge will complete any merge that has already been +# Test that a merge will complete any merge that has already been # started, even if the number of input segments is less than the current -# value of the 'automerge' configuration parameter. +# value of the 'usermerge' configuration parameter. # db func rnddoc fts5_rnddoc @@ -119,7 +119,7 @@ do_execsql_test 3.1 { } do_test 3.2 { execsql { - INSERT INTO x8(x8, rank) VALUES('automerge', 4); + INSERT INTO x8(x8, rank) VALUES('usermerge', 4); INSERT INTO x8(x8, rank) VALUES('merge', 1); } fts5_level_segs x8 @@ -127,14 +127,14 @@ do_test 3.2 { do_test 3.3 { execsql { - INSERT INTO x8(x8, rank) VALUES('automerge', 2); + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); INSERT INTO x8(x8, rank) VALUES('merge', 1); } fts5_level_segs x8 } {2 1} do_test 3.4 { - execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) } + execsql { INSERT INTO x8(x8, rank) VALUES('usermerge', 4) } while {[not_merged x8]} { execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) } } @@ -176,7 +176,7 @@ foreach {tn pgsz} { INSERT INTO x8 SELECT mydoc() FROM ii; WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) INSERT INTO x8 SELECT mydoc() FROM ii; - INSERT INTO x8(x8, rank) VALUES('automerge', 2); + INSERT INTO x8(x8, rank) VALUES('usermerge', 2); } set expect [mycount] @@ -190,5 +190,55 @@ foreach {tn pgsz} { # db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r } } +#------------------------------------------------------------------------- +# Test that the 'merge' command does not modify the database if there is +# no work to do. + +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x9 USING fts5(one, two); + INSERT INTO x9(x9, rank) VALUES('pgsz', 32); + INSERT INTO x9(x9, rank) VALUES('automerge', 2); + INSERT INTO x9(x9, rank) VALUES('usermerge', 2); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); + INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100)); +} + +do_test 5.2 { + while 1 { + set nChange [db total_changes] + execsql { INSERT INTO x9(x9, rank) VALUES('merge', 1); } + set nChange [expr [db total_changes] - $nChange] + #puts $nChange + if {$nChange<2} break + } +} {} + + +#-------------------------------------------------------------------------- +# Test that running 'merge' on an empty database does not cause a +# problem. +# +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE g1 USING fts5(a, b); +} +do_execsql_test 6.1 { + INSERT INTO g1(g1, rank) VALUES('merge', 10); +} +do_execsql_test 6.2 { + INSERT INTO g1(g1, rank) VALUES('merge', -10); +} +do_execsql_test 6.3 { + INSERT INTO g1(g1) VALUES('integrity-check'); +} + + + finish_test diff --git a/ext/fts5/test/fts5merge2.test b/ext/fts5/test/fts5merge2.test index 87dbf143..0d3fad8b 100644 --- a/ext/fts5/test/fts5merge2.test +++ b/ext/fts5/test/fts5merge2.test @@ -13,7 +13,8 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5merge +set testprefix fts5merge2 +return_if_no_fts5 proc dump_structure {} { db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} { @@ -26,8 +27,6 @@ proc dump_structure {} { foreach_detail_mode $testprefix { -if {[detail_is_none]==0} continue - do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); diff --git a/ext/fts5/test/fts5multiclient.test b/ext/fts5/test/fts5multiclient.test new file mode 100644 index 00000000..a1e9aa0d --- /dev/null +++ b/ext/fts5/test/fts5multiclient.test @@ -0,0 +1,48 @@ +# 2016 March 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/lock_common.tcl + +set testprefix fts5multiclient +return_if_no_fts5 + +foreach_detail_mode $testprefix { + +do_multiclient_test tn { + + do_test 1.$tn.1 { + sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) } + sql1 { INSERT INTO t1 VALUES('a b c') } + sql2 { SELECT rowid FROM t1('b') } + } {1} + + do_test 1.$tn.2 { + sql2 { INSERT INTO t1 VALUES('a b c') } + sql1 { SELECT rowid FROM t1('b') } + } {1 2} + + do_test 1.$tn.3 { + sql2 { INSERT INTO t1 VALUES('a b c') } + sql1 { SELECT rowid FROM t1('b') } + } {1 2 3} + + do_test 1.$tn.4 { + sql2 { INSERT INTO t1 VALUES('a b c') } + sql1 { INSERT INTO t1 VALUES('a b c') } + sql3 { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} + +};# do_multiclient_test +};# foreach_detail_mode +finish_test + diff --git a/ext/fts5/test/fts5optimize.test b/ext/fts5/test/fts5optimize.test index 984af8c5..3ef6d8a1 100644 --- a/ext/fts5/test/fts5optimize.test +++ b/ext/fts5/test/fts5optimize.test @@ -20,6 +20,12 @@ ifcapable !fts5 { return } +# +# 1.* - Warm body tests for index optimization using ('optimize') +# +# 2.* - Warm body tests for index optimization using ('merge', -1) +# + proc rnddoc {nWord} { set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z} set nVocab [llength $vocab] @@ -30,14 +36,12 @@ proc rnddoc {nWord} { return $ret } - foreach {tn nStep} { 1 2 2 10 3 50 4 500 } { -if {$tn!=4} continue reset_db db func rnddoc rnddoc do_execsql_test 1.$tn.1 { @@ -60,7 +64,46 @@ if {$tn!=4} continue do_execsql_test 1.$tn.5 { INSERT INTO t1(t1) VALUES('integrity-check'); } + + do_test 1.$tn.6 { fts5_segcount t1 } 1 } +foreach {tn nStep} { + 1 2 + 2 10 + 3 50 + 4 500 +} { + reset_db + db func rnddoc rnddoc + do_execsql_test 1.$tn.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, y); + } + do_test 2.$tn.2 { + for {set i 0} {$i < $nStep} {incr i} { + execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) } + } + } {} + + do_execsql_test 2.$tn.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + do_test 2.$tn.4 { + execsql { INSERT INTO t1(t1, rank) VALUES('merge', -1) } + while 1 { + set c [db total_changes] + execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) } + set c [expr [db total_changes]-$c] + if {$c<2} break + } + } {} + + do_execsql_test 2.$tn.5 { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + do_test 2.$tn.6 { fts5_segcount t1 } 1 +} finish_test diff --git a/ext/fts5/test/fts5rank.test b/ext/fts5/test/fts5rank.test index 4961b426..a70c5d68 100644 --- a/ext/fts5/test/fts5rank.test +++ b/ext/fts5/test/fts5rank.test @@ -91,7 +91,61 @@ do_test 2.7 { } {1 3 2} +#-------------------------------------------------------------------------- +# At one point there was a problem with queries such as: +# +# ... MATCH 'x OR y' ORDER BY rank; +# +# if there were zero occurrences of token 'y' in the dataset. The +# following tests verify that that problem has been addressed. +# +foreach_detail_mode $::testprefix { + do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%); + INSERT INTO y1 VALUES('test xyz'); + INSERT INTO y1 VALUES('test test xyz test'); + INSERT INTO y1 VALUES('test test xyz'); + } + do_execsql_test 3.1.1 { + SELECT rowid FROM y1('test OR tset'); + } {1 2 3} + + do_execsql_test 3.1.2 { + SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1) + } {2 3 1} + + do_execsql_test 3.1.3 { + SELECT rowid FROM y1('test OR tset') ORDER BY +rank + } {2 3 1} + + do_execsql_test 3.1.4 { + SELECT rowid FROM y1('test OR tset') ORDER BY rank + } {2 3 1} + + do_execsql_test 3.1.5 { + SELECT rowid FROM y1('test OR xyz') ORDER BY rank + } {3 2 1} + + + do_execsql_test 3.2.1 { + CREATE VIRTUAL TABLE z1 USING fts5(a, detail=%DETAIL%); + INSERT INTO z1 VALUES('wrinkle in time'); + SELECT * FROM z1 WHERE z1 MATCH 'wrinkle in time OR a wrinkle in time'; + } {{wrinkle in time}} +} + +do_execsql_test 4.1 { + DROP TABLE IF EXISTS VTest; + CREATE virtual TABLE VTest USING FTS5( + Title, AUthor, tokenize ='porter unicode61 remove_diacritics 1', + columnsize='1', detail=full + ); + INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith'); + + SELECT * FROM VTest WHERE + VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank; +} {{wrinkle in time} {Bill Smith}} diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index a095a162..5ac413c8 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -265,7 +265,7 @@ do_test 11.0 { execsql " INSERT INTO t4 VALUES('a b c \x1A'); INSERT INTO t4 VALUES('a b c d\x1A'); - INSERT INTO t4 VALUES('a b c \x1Ad'); + INSERT INTO t4 VALUES('a b c \x1Ag'); INSERT INTO t4 VALUES('a b c d'); " } {} @@ -340,7 +340,7 @@ do_test 14.2 { #------------------------------------------------------------------------- db func rnddoc fts5_rnddoc -do_execsql_test 4.0 { +do_execsql_test 14.3 { CREATE VIRTUAL TABLE x1 USING fts5(x); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); @@ -348,9 +348,9 @@ do_execsql_test 4.0 { INSERT INTO x1 SELECT rnddoc(5) FROM ii; } -do_execsql_test 4.1 { +do_execsql_test 14.4 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' -} {0 {} 4} +} {0 {} 3} #------------------------------------------------------------------------- reset_db @@ -409,5 +409,63 @@ do_catchsql_test 19.2 { SELECT * FROM x1 WHERE x1 MATCH 'c0 AND (c1 AND (c2 AND (c3 AND (c4 AND (c5 AND (c6 AND (c7 AND (c8 AND (c9 AND (c10 AND (c11 AND (c12 AND (c13 AND (c14 AND (c15 AND (c16 AND (c17 AND (c18 AND (c19 AND (c20 AND (c21 AND (c22 AND (c23 AND (c24 AND (c25 AND (c26 AND (c27 AND (c28 AND (c29 AND (c30 AND (c31 AND (c32 AND (c33 AND (c34 AND (c35 AND (c36 AND (c37 AND (c38 AND (c39 AND (c40 AND (c41 AND (c42 AND (c43 AND (c44 AND (c45 AND (c46 AND (c47 AND (c48 AND (c49 AND (c50 AND (c51 AND (c52 AND (c53 AND (c54 AND (c55 AND (c56 AND (c57 AND (c58 AND (c59 AND (c60 AND (c61 AND (c62 AND (c63 AND (c64 AND (c65 AND (c66 AND (c67 AND (c68 AND (c69 AND (c70 AND (c71 AND (c72 AND (c73 AND (c74 AND (c75 AND (c76 AND (c77 AND (c78 AND (c79 AND (c80 AND (c81 AND (c82 AND (c83 AND (c84 AND (c85 AND (c86 AND (c87 AND (c88 AND (c89 AND (c90 AND (c91 AND (c92 AND (c93 AND (c94 AND (c95 AND (c96 AND (c97 AND (c98 AND (c99 AND (c100 AND (c101 AND (c102 AND (c103 AND (c104 AND (c105 AND (c106 AND (c107 AND (c108 AND (c109 AND (c110 AND (c111 AND (c112 AND (c113 AND (c114 AND (c115 AND (c116 AND (c117 AND (c118 AND (c119 AND (c120 AND (c121 AND (c122 AND (c123 AND (c124 AND (c125 AND (c126 AND (c127 AND (c128 AND (c129 AND (c130 AND (c131 AND (c132 AND (c133 AND (c134 AND (c135 AND (c136 AND (c137 AND (c138 AND (c139 AND (c140 AND (c141 AND (c142 AND (c143 AND (c144 AND (c145 AND (c146 AND (c147 AND (c148 AND (c149 AND (c150 AND (c151 AND (c152 AND (c153 AND (c154 AND (c155 AND (c156 AND (c157 AND (c158 AND (c159 AND (c160 AND (c161 AND (c162 AND (c163 AND (c164 AND (c165 AND (c166 AND (c167 AND (c168 AND (c169 AND (c170 AND (c171 AND (c172 AND (c173 AND (c174 AND (c175 AND (c176 AND (c177 AND (c178 AND (c179 AND (c180 AND (c181 AND (c182 AND (c183 AND (c184 AND (c185 AND (c186 AND (c187 AND (c188 AND (c189 AND (c190 AND (c191 AND (c192 AND (c193 AND (c194 AND (c195 AND (c196 AND (c197 AND (c198 AND (c199 AND c200)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))'; } {1 {fts5: parser stack overflow}} -finish_test +#------------------------------------------------------------------------- +reset_db +breakpoint +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree'); +} +do_test 20.1 { + for {set i 1} {$i <= 200} {incr i} { + execsql { INSERT INTO x1(rowid, x) VALUES($i, 'one two three'); } + } + execsql { INSERT INTO x1(x1) VALUES('optimize'); } + execsql { DELETE FROM x1 WHERE rowid = 4; } +} {} +do_execsql_test 20.2 { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x1(x1) VALUES('integrity-check'); +} {} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 32); + INSERT INTO x1(rowid, x) VALUES(11111, 'onetwothree'); +} +do_test 20.1 { + for {set i 1} {$i <= 200} {incr i} { + execsql { INSERT INTO x1(rowid, x) VALUES($i, 'one two three'); } + } + execsql { INSERT INTO x1(x1) VALUES('optimize'); } + execsql { DELETE FROM x1 WHERE rowid = 4; } +} {} +do_execsql_test 20.2 { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x1(x1) VALUES('integrity-check'); +} {} + +#------------------------------------------------------------------------- +reset_db +set doc "a b [string repeat x 100000]" +do_execsql_test 21.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(rowid, x) VALUES(11111, $doc); + INSERT INTO x1(rowid, x) VALUES(11112, $doc); +} +do_execsql_test 21.1 { + INSERT INTO x1(x1) VALUES('integrity-check'); +} +do_execsql_test 21.2 { + SELECT rowid FROM x1($doc); +} {11111 11112} +do_execsql_test 21.3 { + DELETE FROM x1 WHERE rowid=11111; + INSERT INTO x1(x1) VALUES('integrity-check'); + SELECT rowid FROM x1($doc); +} {11112} + +finish_test diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test index b6922f84..c755ea05 100644 --- a/ext/fts5/test/fts5simple3.test +++ b/ext/fts5/test/fts5simple3.test @@ -80,6 +80,40 @@ do_execsql_test 3.0 { SELECT * FROM x3('x OR y OR z'); } +#------------------------------------------------------------------------- +# Test that a crash occuring when the second or subsequent tokens in a +# phrase matched zero rows has been fixed. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1 VALUES('ab'); + INSERT INTO t1 VALUES('cd'); + INSERT INTO t1 VALUES('ab cd'); + INSERT INTO t1 VALUES('ab cdXXX'); + INSERT INTO t1 VALUES('abXXX cd'); +} +do_execsql_test 4.1 { + SELECT * FROM t1('"ab cd" OR "ab cd" *'); +} {{ab cd} {ab cdXXX}} +do_execsql_test 4.2 { + SELECT * FROM t1('"xy zz" OR "ab cd" *'); +} {{ab cd} {ab cdXXX}} +do_execsql_test 4.3 { + SELECT * FROM t1('"xy zz" OR "xy zz" *'); +} +do_execsql_test 4.4 { + SELECT * FROM t1('"ab cd" OR "xy zz" *'); +} {{ab cd}} +do_execsql_test 4.5 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('ab'); + INSERT INTO t2 VALUES('cd'); + INSERT INTO t2 VALUES('ef'); +} +do_execsql_test 4.6 { + SELECT * FROM t2('ab + xyz'); +} + finish_test diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test index d3ff5128..8e5bb8e2 100644 --- a/ext/fts5/test/fts5unicode2.test +++ b/ext/fts5/test/fts5unicode2.test @@ -160,12 +160,12 @@ foreach {tn query snippet} { the maximum x value. } 4 "rollback" { - ...[ROLLBACK]. Instead, the pending statement - will return SQLITE_ABORT upon next access after the [ROLLBACK]. + Pending statements no longer block [ROLLBACK]. Instead, the pending + statement will return SQLITE_ABORT upon... } 5 "rOllback" { - ...[ROLLBACK]. Instead, the pending statement - will return SQLITE_ABORT upon next access after the [ROLLBACK]. + Pending statements no longer block [ROLLBACK]. Instead, the pending + statement will return SQLITE_ABORT upon... } 6 "lang*" { Added support for the FTS4 [languageid] option. diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index f7278dd5..5e0499fb 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -442,8 +442,44 @@ if {[detail_is_none]} { } sqlite3_fts5_may_be_corrupt 0 - } +#------------------------------------------------------------------------- +# Test that both "ORDER BY term" and "ORDER BY term DESC" work. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1 VALUES('def ABC ghi'); + INSERT INTO x1 VALUES('DEF abc GHI'); +} + +do_execsql_test 9.2 { + CREATE VIRTUAL TABLE rrr USING fts5vocab(x1, row); + SELECT * FROM rrr +} { + abc 2 2 def 2 2 ghi 2 2 +} +do_execsql_test 9.3 { + SELECT * FROM rrr ORDER BY term ASC +} { + abc 2 2 def 2 2 ghi 2 2 +} +do_execsql_test 9.4 { + SELECT * FROM rrr ORDER BY term DESC +} { + ghi 2 2 def 2 2 abc 2 2 +} +do_test 9.5 { + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term ASC }] + expr [lsearch $e2 SorterSort]<0 +} 1 +do_test 9.6 { + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }] + expr [lsearch $e2 SorterSort]<0 +} 0 + + + finish_test diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl index d5df971d..4766b00b 100644 --- a/ext/fts5/tool/fts5txt2db.tcl +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -17,6 +17,7 @@ proc process_cmdline {} { {detail "full" "Fts5 detail mode to use"} {repeat 1 "Load each file this many times"} {prefix "" "Fts prefix= option"} + {trans 1 "True to use a transaction"} database file... } { @@ -214,7 +215,7 @@ foreach c [lrange $cols 1 end] { } append sql ")" -db eval BEGIN +if {$A(trans)} { db eval BEGIN } while {$i < $N} { foreach c $cols s $A(colsize) { set R($c) [lrange $tokens $i [expr $i+$s-1]] @@ -222,7 +223,7 @@ db eval BEGIN } db eval $sql } -db eval COMMIT +if {$A(trans)} { db eval COMMIT } diff --git a/ext/icu/icu.c b/ext/icu/icu.c index a2ff4927..d2beaa33 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -60,6 +60,38 @@ static void xFree(void *p){ sqlite3_free(p); } +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + /* ** Compare two UTF-8 strings for equality where the first string is ** a "LIKE" expression. Return true (1) if they are the same and @@ -73,16 +105,14 @@ static int icuLikeCompare( static const int MATCH_ONE = (UChar32)'_'; static const int MATCH_ALL = (UChar32)'%'; - int iPattern = 0; /* Current byte index in zPattern */ - int iString = 0; /* Current byte index in zString */ - int prevEscape = 0; /* True if the previous character was uEsc */ - while( zPattern[iPattern]!=0 ){ + while( 1 ){ /* Read (and consume) the next character from the input pattern. */ UChar32 uPattern; - U8_NEXT_UNSAFE(zPattern, iPattern, uPattern); + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; /* There are now 4 possibilities: ** @@ -99,28 +129,28 @@ static int icuLikeCompare( ** MATCH_ALL. For each MATCH_ONE, skip one character in the ** test string. */ - while( (c=zPattern[iPattern]) == MATCH_ALL || c == MATCH_ONE ){ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ if( c==MATCH_ONE ){ - if( zString[iString]==0 ) return 0; - U8_FWD_1_UNSAFE(zString, iString); + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); } - iPattern++; + zPattern++; } - if( zPattern[iPattern]==0 ) return 1; + if( *zPattern==0 ) return 1; - while( zString[iString] ){ - if( icuLikeCompare(&zPattern[iPattern], &zString[iString], uEsc) ){ + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ return 1; } - U8_FWD_1_UNSAFE(zString, iString); + SQLITE_ICU_SKIP_UTF8(zString); } return 0; }else if( !prevEscape && uPattern==MATCH_ONE ){ /* Case 2. */ - if( zString[iString]==0 ) return 0; - U8_FWD_1_UNSAFE(zString, iString); + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); }else if( !prevEscape && uPattern==uEsc){ /* Case 3. */ @@ -129,7 +159,7 @@ static int icuLikeCompare( }else{ /* Case 4. */ UChar32 uString; - U8_NEXT_UNSAFE(zString, iString, uString); + SQLITE_ICU_READ_UTF8(zString, uString); uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); if( uString!=uPattern ){ @@ -139,7 +169,7 @@ static int icuLikeCompare( } } - return zString[iString]==0; + return *zString==0; } /* @@ -319,20 +349,22 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ** of upper() or lower(). ** ** lower('I', 'en_us') -> 'i' -** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** lower('I', 'tr_tr') -> '\u131' (small dotless i) ** ** http://www.icu-project.org/userguide/posix.html#case_mappings */ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - const UChar *zInput; - UChar *zOutput; - int nInput; - int nOutput; - - UErrorCode status = U_ZERO_ERROR; + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; const char *zLocale = 0; assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); if( nArg==2 ){ zLocale = (const char *)sqlite3_value_text(apArg[1]); } @@ -341,26 +373,38 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ if( !zInput ){ return; } - nInput = sqlite3_value_bytes16(apArg[0]); - - nOutput = nInput * 2 + 2; - zOutput = sqlite3_malloc(nOutput); - if( !zOutput ){ + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); return; } - if( sqlite3_user_data(p) ){ - u_strToUpper(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); - }else{ - u_strToLower(zOutput, nOutput/2, zInput, nInput/2, zLocale, &status); - } + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( bToUpper ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } - if( !U_SUCCESS(status) ){ - icuFunctionError(p, "u_strToLower()/u_strToUpper", status); + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } return; } - - sqlite3_result_text16(p, zOutput, -1, xFree); + assert( 0 ); /* Unreachable */ } /* @@ -456,20 +500,20 @@ int sqlite3IcuInit(sqlite3 *db){ void *pContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, + {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, - {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, (void*)1, icuCaseFunc16}, - {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, + {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, }; diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index 85249198..efe76566 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -625,10 +625,10 @@ static int amatchLoadOneRule( }else{ memset(pRule, 0, sizeof(*pRule)); pRule->zFrom = &pRule->zTo[nTo+1]; - pRule->nFrom = nFrom; + pRule->nFrom = (amatch_len)nFrom; memcpy(pRule->zFrom, zFrom, nFrom+1); memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; + pRule->nTo = (amatch_len)nTo; pRule->rCost = rCost; pRule->iLang = (int)iLang; } @@ -1081,7 +1081,7 @@ static void amatchAddWord( pWord->rCost = rCost; pWord->iSeq = pCur->nWord++; amatchWriteCost(pWord); - pWord->nMatch = nMatch; + pWord->nMatch = (short)nMatch; pWord->pNext = pCur->pAllWords; pCur->pAllWords = pWord; pWord->sCost.zKey = pWord->zCost; @@ -1162,7 +1162,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ #endif nWord = (int)strlen(pWord->zWord+2); if( nWord+20>nBuf ){ - nBuf = nWord+100; + nBuf = (char)(nWord+100); zBuf = sqlite3_realloc(zBuf, nBuf); if( zBuf==0 ) return SQLITE_NOMEM; } diff --git a/ext/misc/carray.c b/ext/misc/carray.c new file mode 100644 index 00000000..025eb5db --- /dev/null +++ b/ext/misc/carray.c @@ -0,0 +1,364 @@ +/* +** 2016-06-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function that +** returns the values in a C-language array. +** Examples: +** +** SELECT * FROM carray($ptr,5) +** +** The query above returns 5 integers contained in a C-language array +** at the address $ptr. $ptr is a pointer to the array of integers that +** has been cast to an integer. +** +** There is an optional third parameter to determine the datatype of +** the C-language array. Allowed values of the third parameter are +** 'int32', 'int64', 'double', 'char*'. Example: +** +** SELECT * FROM carray($ptr,10,'char*'); +** +** HOW IT WORKS +** +** The carray "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE carray( +** value, +** pointer HIDDEN, +** count HIDDEN, +** ctype TEXT HIDDEN +** ); +** +** If the hidden columns "pointer" and "count" are unconstrained, then +** the virtual table has no rows. Otherwise, the virtual table interprets +** the integer value of "pointer" as a pointer to the array and "count" +** as the number of elements in the array. The virtual table steps through +** the array, element by element. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Allowed datatypes +*/ +#define CARRAY_INT32 0 +#define CARRAY_INT64 1 +#define CARRAY_DOUBLE 2 +#define CARRAY_TEXT 3 + +/* +** Names of types +*/ +static const char *azType[] = { "int32", "int64", "double", "char*" }; + + +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct carray_cursor carray_cursor; +struct carray_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + sqlite3_int64 iPtr; /* Pointer to array of values */ + sqlite3_int64 iCnt; /* Number of integers in the array */ + unsigned char eType; /* One of the CARRAY_type values */ +}; + +/* +** The carrayConnect() method is invoked to create a new +** carray_vtab that describes the carray virtual table. +** +** Think of this routine as the constructor for carray_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the carray_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against carray will look like. +*/ +static int carrayConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define CARRAY_COLUMN_VALUE 0 +#define CARRAY_COLUMN_POINTER 1 +#define CARRAY_COLUMN_COUNT 2 +#define CARRAY_COLUMN_CTYPE 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for carray_cursor objects. +*/ +static int carrayDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new carray_cursor object. +*/ +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + carray_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a carray_cursor. +*/ +static int carrayClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a carray_cursor to its next row of output. +*/ +static int carrayNext(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the carray_cursor +** is currently pointing. +*/ +static int carrayColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + carray_cursor *pCur = (carray_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case CARRAY_COLUMN_POINTER: x = pCur->iPtr; break; + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; + case CARRAY_COLUMN_CTYPE: { + sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC); + return SQLITE_OK; + } + default: { + switch( pCur->eType ){ + case CARRAY_INT32: { + int *p = (int*)pCur->iPtr; + sqlite3_result_int(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_INT64: { + sqlite3_int64 *p = (sqlite3_int64*)pCur->iPtr; + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_DOUBLE: { + double *p = (double*)pCur->iPtr; + sqlite3_result_double(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_TEXT: { + const char **p = (const char**)pCur->iPtr; + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); + return SQLITE_OK; + } + } + } + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + carray_cursor *pCur = (carray_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int carrayEof(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + return pCur->iRowid>pCur->iCnt; +} + +/* +** This method is called to "rewind" the carray_cursor object back +** to the first row of output. +*/ +static int carrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + carray_cursor *pCur = (carray_cursor *)pVtabCursor; + if( idxNum ){ + pCur->iPtr = sqlite3_value_int64(argv[0]); + pCur->iCnt = sqlite3_value_int64(argv[1]); + if( idxNum<3 ){ + pCur->eType = CARRAY_INT32; + }else{ + unsigned char i; + const char *zType = (const char*)sqlite3_value_text(argv[2]); + for(i=0; i=sizeof(azType)/sizeof(azType[0]) ){ + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "unknown datatype: %Q", zType); + return SQLITE_ERROR; + }else{ + pCur->eType = i; + } + } + }else{ + pCur->iPtr = 0; + pCur->iCnt = 0; + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the carray virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** idxNum is 2 if the pointer= and count= constraints exist, +** 3 if the ctype= constraint also exists, and is 0 otherwise. +** If idxNum is 0, then carray becomes an empty table. +*/ +static int carrayBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case CARRAY_COLUMN_POINTER: + ptrIdx = i; + break; + case CARRAY_COLUMN_COUNT: + cntIdx = i; + break; + case CARRAY_COLUMN_CTYPE: + ctypeIdx = i; + break; + } + } + if( ptrIdx>=0 && cntIdx>=0 ){ + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 2; + if( ctypeIdx>=0 ){ + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + }else{ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + pIdxInfo->idxNum = 0; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** carray virtual table. +*/ +static sqlite3_module carrayModule = { + 0, /* iVersion */ + 0, /* xCreate */ + carrayConnect, /* xConnect */ + carrayBestIndex, /* xBestIndex */ + carrayDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + carrayOpen, /* xOpen - open a cursor */ + carrayClose, /* xClose - close a cursor */ + carrayFilter, /* xFilter - configure scan constraints */ + carrayNext, /* xNext - advance a cursor */ + carrayEof, /* xEof - check for end of scan */ + carrayColumn, /* xColumn - read data */ + carrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_carray_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "carray", &carrayModule, 0); +#endif + return rc; +} diff --git a/ext/misc/csv.c b/ext/misc/csv.c new file mode 100644 index 00000000..2a9463bd --- /dev/null +++ b/ext/misc/csv.c @@ -0,0 +1,869 @@ +/* +** 2016-05-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the implementation of an SQLite virtual table for +** reading CSV files. +** +** Usage: +** +** .load ./csv +** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME); +** SELECT * FROM csv; +** +** The columns are named "c1", "c2", "c3", ... by default. But the +** application can define its own CREATE TABLE statement as an additional +** parameter. For example: +** +** CREATE VIRTUAL TABLE temp.csv2 USING csv( +** filename = "../http.log", +** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)" +** ); +** +** Instead of specifying a file, the text of the CSV can be loaded using +** the data= parameter. +** +** If the columns=N parameter is supplied, then the CSV file is assumed to have +** N columns. If the columns parameter is omitted, the CSV file is opened +** as soon as the virtual table is constructed and the first row of the CSV +** is read in order to count the tables. +** +** Some extra debugging features (used for testing virtual tables) are available +** if this module is compiled with -DSQLITE_TEST. +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define CSV_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define CSV_NOINLINE __declspec(noinline) +#else +# define CSV_NOINLINE +#endif + + +/* Max size of the error message in a CsvReader */ +#define CSV_MXERR 200 + +/* Size of the CsvReader input buffer */ +#define CSV_INBUFSZ 1024 + +/* A context object used when read a CSV file. */ +typedef struct CsvReader CsvReader; +struct CsvReader { + FILE *in; /* Read the CSV text from this input stream */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + char cTerm; /* Character that terminated the most recent field */ + size_t iIn; /* Next unread character in the input buffer */ + size_t nIn; /* Number of characters in the input buffer */ + char *zIn; /* The input buffer */ + char zErr[CSV_MXERR]; /* Error message */ +}; + +/* Initialize a CsvReader object */ +static void csv_reader_init(CsvReader *p){ + p->in = 0; + p->z = 0; + p->n = 0; + p->nAlloc = 0; + p->nLine = 0; + p->nIn = 0; + p->zIn = 0; + p->zErr[0] = 0; +} + +/* Close and reset a CsvReader object */ +static void csv_reader_reset(CsvReader *p){ + if( p->in ){ + fclose(p->in); + sqlite3_free(p->zIn); + } + sqlite3_free(p->z); + csv_reader_init(p); +} + +/* Report an error on a CsvReader */ +static void csv_errmsg(CsvReader *p, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap); + va_end(ap); +} + +/* Open the file associated with a CsvReader +** Return the number of errors. +*/ +static int csv_reader_open( + CsvReader *p, /* The reader to open */ + const char *zFilename, /* Read from this filename */ + const char *zData /* ... or use this data */ +){ + if( zFilename ){ + p->zIn = sqlite3_malloc( CSV_INBUFSZ ); + if( p->zIn==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + p->in = fopen(zFilename, "rb"); + if( p->in==0 ){ + csv_reader_reset(p); + csv_errmsg(p, "cannot open '%s' for reading", zFilename); + return 1; + } + }else{ + assert( p->in==0 ); + p->zIn = (char*)zData; + p->nIn = strlen(zData); + } + return 0; +} + +/* The input buffer has overflowed. Refill the input buffer, then +** return the next character +*/ +static CSV_NOINLINE int csv_getc_refill(CsvReader *p){ + size_t got; + + assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */ + assert( p->in!=0 ); /* Only called if reading froma file */ + + got = fread(p->zIn, 1, CSV_INBUFSZ, p->in); + if( got==0 ) return EOF; + p->nIn = got; + p->iIn = 1; + return p->zIn[0]; +} + +/* Return the next character of input. Return EOF at end of input. */ +static int csv_getc(CsvReader *p){ + if( p->iIn >= p->nIn ){ + if( p->in!=0 ) return csv_getc_refill(p); + return EOF; + } + return p->zIn[p->iIn++]; +} + +/* Increase the size of p->z and append character c to the end. +** Return 0 on success and non-zero if there is an OOM error */ +static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ + char *zNew; + int nNew = p->nAlloc*2 + 100; + zNew = sqlite3_realloc64(p->z, nNew); + if( zNew ){ + p->z = zNew; + p->nAlloc = nNew; + p->z[p->n++] = c; + return 0; + }else{ + csv_errmsg(p, "out of memory"); + return 1; + } +} + +/* Append a single character to the CsvReader.z[] array. +** Return 0 on success and non-zero if there is an OOM error */ +static int csv_append(CsvReader *p, char c){ + if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c); + p->z[p->n++] = c; + return 0; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** +** Return "" at EOF. Return 0 on an OOM error. +*/ +static char *csv_read_one_field(CsvReader *p){ + int c; + p->n = 0; + c = csv_getc(p); + if( c==EOF ){ + p->cTerm = EOF; + return ""; + } + if( c=='"' ){ + int pc, ppc; + int startLine = p->nLine; + pc = ppc = 0; + while( 1 ){ + c = csv_getc(p); + if( c<='"' || pc=='"' ){ + if( c=='\n' ) p->nLine++; + if( c=='"' ){ + if( pc=='"' ){ + pc = 0; + continue; + } + } + if( (c==',' && pc=='"') + || (c=='\n' && pc=='"') + || (c=='\n' && pc=='\r' && ppc=='"') + || (c==EOF && pc=='"') + ){ + do{ p->n--; }while( p->z[p->n]!='"' ); + p->cTerm = (char)c; + break; + } + if( pc=='"' && c!='\r' ){ + csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"'); + break; + } + if( c==EOF ){ + csv_errmsg(p, "line %d: unterminated %c-quoted field\n", + startLine, '"'); + p->cTerm = (char)c; + break; + } + } + if( csv_append(p, (char)c) ) return 0; + ppc = pc; + pc = c; + } + }else{ + while( c>',' || (c!=EOF && c!=',' && c!='\n') ){ + if( csv_append(p, (char)c) ) return 0; + c = csv_getc(p); + } + if( c=='\n' ){ + p->nLine++; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; + } + p->cTerm = (char)c; + } + if( p->z ) p->z[p->n] = 0; + return p->z; +} + + +/* Forward references to the various virtual table methods implemented +** in this file. */ +static int csvtabCreate(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabConnect(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*); +static int csvtabDisconnect(sqlite3_vtab*); +static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); +static int csvtabClose(sqlite3_vtab_cursor*); +static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); +static int csvtabNext(sqlite3_vtab_cursor*); +static int csvtabEof(sqlite3_vtab_cursor*); +static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); +static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); + +/* An instance of the CSV virtual table */ +typedef struct CsvTable { + sqlite3_vtab base; /* Base class. Must be first */ + char *zFilename; /* Name of the CSV file */ + char *zData; /* Raw CSV data in lieu of zFilename */ + long iStart; /* Offset to start of data in zFilename */ + int nCol; /* Number of columns in the CSV file */ + unsigned int tstFlags; /* Bit values used for testing */ +} CsvTable; + +/* Allowed values for tstFlags */ +#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/ + +/* A cursor for the CSV virtual table */ +typedef struct CsvCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + CsvReader rdr; /* The CsvReader object */ + char **azVal; /* Value of the current row */ + int *aLen; /* Length of each entry */ + sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */ +} CsvCursor; + +/* Transfer error message text from a reader into a CsvTable */ +static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){ + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr); +} + +/* +** This method is the destructor fo a CsvTable object. +*/ +static int csvtabDisconnect(sqlite3_vtab *pVtab){ + CsvTable *p = (CsvTable*)pVtab; + sqlite3_free(p->zFilename); + sqlite3_free(p->zData); + sqlite3_free(p); + return SQLITE_OK; +} + +/* Skip leading whitespace. Return a pointer to the first non-whitespace +** character, or to the zero terminator if the string has only whitespace */ +static const char *csv_skip_whitespace(const char *z){ + while( isspace((unsigned char)z[0]) ) z++; + return z; +} + +/* Remove trailing whitespace from the end of string z[] */ +static void csv_trim_whitespace(char *z){ + size_t n = strlen(z); + while( n>0 && isspace((unsigned char)z[n]) ) n--; + z[n] = 0; +} + +/* Dequote the string */ +static void csv_dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; izErr. If there are no errors, p->zErr[0]==0. +*/ +static int csv_string_parameter( + CsvReader *p, /* Leave the error message here, if there is one */ + const char *zParam, /* Parameter we are checking for */ + const char *zArg, /* Raw text of the virtual table argment */ + char **pzVal /* Write the dequoted string value here */ +){ + const char *zValue; + zValue = csv_parameter(zParam,(int)strlen(zParam),zArg); + if( zValue==0 ) return 0; + p->zErr[0] = 0; + if( *pzVal ){ + csv_errmsg(p, "more than one '%s' parameter", zParam); + return 1; + } + *pzVal = sqlite3_mprintf("%s", zValue); + if( *pzVal==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + csv_trim_whitespace(*pzVal); + csv_dequote(*pzVal); + return 1; +} + + +/* Return 0 if the argument is false and 1 if it is true. Return -1 if +** we cannot really tell. +*/ +static int csv_boolean(const char *z){ + if( sqlite3_stricmp("yes",z)==0 + || sqlite3_stricmp("on",z)==0 + || sqlite3_stricmp("true",z)==0 + || (z[0]=='1' && z[0]==0) + ){ + return 1; + } + if( sqlite3_stricmp("no",z)==0 + || sqlite3_stricmp("off",z)==0 + || sqlite3_stricmp("false",z)==0 + || (z[0]=='0' && z[1]==0) + ){ + return 0; + } + return -1; +} + + +/* +** Parameters: +** filename=FILENAME Name of file containing CSV content +** data=TEXT Direct CSV content. +** schema=SCHEMA Alternative CSV schema. +** header=YES|NO First row of CSV defines the names of +** columns if "yes". Default "no". +** columns=N Assume the CSV file contains N columns. +** +** Only available if compiled with SQLITE_TEST: +** +** testflags=N Bitmask of test flags. Optional +** +** If schema= is omitted, then the columns are named "c0", "c1", "c2", +** and so forth. If columns=N is omitted, then the file is opened and +** the number of columns in the first row is counted to determine the +** column count. If header=YES, then the first row is skipped. +*/ +static int csvtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + CsvTable *pNew = 0; /* The CsvTable object to construct */ + int bHeader = -1; /* header= flags. -1 means not seen yet */ + int rc = SQLITE_OK; /* Result code from this routine */ + int i, j; /* Loop counters */ +#ifdef SQLITE_TEST + int tstFlags = 0; /* Value for testflags=N parameter */ +#endif + int nCol = -99; /* Value of the columns= parameter */ + CsvReader sRdr; /* A CSV file reader used to store an error + ** message and/or to count the number of columns */ + static const char *azParam[] = { + "filename", "data", "schema", + }; + char *azPValue[3]; /* Parameter values */ +# define CSV_FILENAME (azPValue[0]) +# define CSV_DATA (azPValue[1]) +# define CSV_SCHEMA (azPValue[2]) + + + assert( sizeof(azPValue)==sizeof(azParam) ); + memset(&sRdr, 0, sizeof(sRdr)); + memset(azPValue, 0, sizeof(azPValue)); + for(i=3; i=0 ){ + csv_errmsg(&sRdr, "more than one 'header' parameter"); + goto csvtab_connect_error; + } + x = csv_boolean(zValue); + if( x==1 ){ + bHeader = 1; + }else if( x==0 ){ + bHeader = 0; + }else{ + csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue); + goto csvtab_connect_error; + } + }else +#ifdef SQLITE_TEST + if( (zValue = csv_parameter("testflags",9,z))!=0 ){ + tstFlags = (unsigned int)atoi(zValue); + }else +#endif + if( (zValue = csv_parameter("columns",7,z))!=0 ){ + if( nCol>0 ){ + csv_errmsg(&sRdr, "more than one 'columns' parameter"); + goto csvtab_connect_error; + } + nCol = atoi(zValue); + if( nCol<=0 ){ + csv_errmsg(&sRdr, "must have at least one column"); + goto csvtab_connect_error; + } + }else + { + csv_errmsg(&sRdr, "unrecognized parameter '%s'", z); + goto csvtab_connect_error; + } + } + if( (CSV_FILENAME==0)==(CSV_DATA==0) ){ + csv_errmsg(&sRdr, "must either filename= or data= but not both"); + goto csvtab_connect_error; + } + if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){ + goto csvtab_connect_error; + } + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) goto csvtab_connect_oom; + memset(pNew, 0, sizeof(*pNew)); + if( nCol>0 ){ + pNew->nCol = nCol; + }else{ + do{ + const char *z = csv_read_one_field(&sRdr); + if( z==0 ) goto csvtab_connect_oom; + pNew->nCol++; + }while( sRdr.cTerm==',' ); + } + pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0; + pNew->zData = CSV_DATA; CSV_DATA = 0; +#ifdef SQLITE_TEST + pNew->tstFlags = tstFlags; +#endif + pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0; + csv_reader_reset(&sRdr); + if( CSV_SCHEMA==0 ){ + char *zSep = ""; + CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x("); + if( CSV_SCHEMA==0 ) goto csvtab_connect_oom; + for(i=0; inCol; i++){ + CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i); + zSep = ","; + } + CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA); + } + rc = sqlite3_declare_vtab(db, CSV_SCHEMA); + if( rc ) goto csvtab_connect_error; + for(i=0; ibase); + for(i=0; ibase.pVtab; + int i; + for(i=0; inCol; i++){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + } +} + +/* +** The xConnect and xCreate methods do the same thing, but they must be +** different so that the virtual table is not an eponymous virtual table. +*/ +static int csvtabCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Destructor for a CsvCursor. +*/ +static int csvtabClose(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + csvtabCursorRowReset(pCur); + csv_reader_reset(&pCur->rdr); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Constructor for a new CsvTable cursor object. +*/ +static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + CsvTable *pTab = (CsvTable*)p; + CsvCursor *pCur; + size_t nByte; + nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol; + pCur = sqlite3_malloc64( nByte ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, nByte); + pCur->azVal = (char**)&pCur[1]; + pCur->aLen = (int*)&pCur->azVal[pTab->nCol]; + *ppCursor = &pCur->base; + if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){ + csv_xfer_error(pTab, &pCur->rdr); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + + +/* +** Advance a CsvCursor to its next row of input. +** Set the EOF marker if we reach the end of input. +*/ +static int csvtabNext(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + int i = 0; + char *z; + do{ + z = csv_read_one_field(&pCur->rdr); + if( z==0 ){ + csv_xfer_error(pTab, &pCur->rdr); + break; + } + if( inCol ){ + if( pCur->aLen[i] < pCur->rdr.n+1 ){ + char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1); + if( zNew==0 ){ + csv_errmsg(&pCur->rdr, "out of memory"); + csv_xfer_error(pTab, &pCur->rdr); + break; + } + pCur->azVal[i] = zNew; + pCur->aLen[i] = pCur->rdr.n+1; + } + memcpy(pCur->azVal[i], z, pCur->rdr.n+1); + i++; + } + }while( pCur->rdr.cTerm==',' ); + while( inCol ){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + i++; + } + if( z==0 || pCur->rdr.cTerm==EOF ){ + pCur->iRowid = -1; + }else{ + pCur->iRowid++; + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the CsvCursor +** is currently pointing. +*/ +static int csvtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + if( i>=0 && inCol && pCur->azVal[i]!=0 ){ + sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. +*/ +static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + CsvCursor *pCur = (CsvCursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int csvtabEof(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + return pCur->iRowid<0; +} + +/* +** Only a full table scan is supported. So xFilter simply rewinds to +** the beginning. +*/ +static int csvtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + CsvCursor *pCur = (CsvCursor*)pVtabCursor; + CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab; + pCur->iRowid = 0; + if( pCur->rdr.in==0 ){ + assert( pCur->rdr.zIn==pTab->zData ); + assert( pTab->iStart>=0 ); + assert( (size_t)pTab->iStart<=pCur->rdr.nIn ); + pCur->rdr.iIn = pTab->iStart; + }else{ + fseek(pCur->rdr.in, pTab->iStart, SEEK_SET); + pCur->rdr.iIn = 0; + pCur->rdr.nIn = 0; + } + return csvtabNext(pVtabCursor); +} + +/* +** Only a forward full table scan is supported. xBestIndex is mostly +** a no-op. If CSVTEST_FIDX is set, then the presence of equality +** constraints lowers the estimated cost, which is fiction, but is useful +** for testing certain kinds of virtual table behavior. +*/ +static int csvtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + pIdxInfo->estimatedCost = 1000000; +#ifdef SQLITE_TEST + if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){ + /* The usual (and sensible) case is to always do a full table scan. + ** The code in this branch only runs when testflags=1. This code + ** generates an artifical and unrealistic plan which is useful + ** for testing virtual table logic but is not helpful to real applications. + ** + ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual + ** table (even though it is not) and the cost of running the virtual table + ** is reduced from 1 million to just 10. The constraints are *not* marked + ** as omittable, however, so the query planner should still generate a + ** plan that gives a correct answer, even if they plan is not optimal. + */ + int i; + int nConst = 0; + for(i=0; inConstraint; i++){ + unsigned char op; + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + op = pIdxInfo->aConstraint[i].op; + if( op==SQLITE_INDEX_CONSTRAINT_EQ + || op==SQLITE_INDEX_CONSTRAINT_LIKE + || op==SQLITE_INDEX_CONSTRAINT_GLOB + ){ + pIdxInfo->estimatedCost = 10; + pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1; + nConst++; + } + } + } +#endif + return SQLITE_OK; +} + + +static sqlite3_module CsvModule = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +#ifdef SQLITE_TEST +/* +** For virtual table testing, make a version of the CSV virtual table +** available that has an xUpdate function. But the xUpdate always returns +** SQLITE_READONLY since the CSV file is not really writable. +*/ +static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){ + return SQLITE_READONLY; +} +static sqlite3_module CsvModuleFauxWrite = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + csvtabUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; +#endif /* SQLITE_TEST */ + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. The new +** CSV virtual table module is registered with the calling database +** connection. +*/ +int sqlite3_csv_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "csv", &CsvModule, 0); +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0); + } +#endif + return rc; +#else + return SQLITE_OK; +#endif +} diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 3ed4b0a9..77db56d4 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -344,10 +344,10 @@ static int fuzzerLoadOneRule( memset(pRule, 0, sizeof(*pRule)); pRule->zFrom = pRule->zTo; pRule->zFrom += nTo + 1; - pRule->nFrom = nFrom; + pRule->nFrom = (fuzzer_len)nFrom; memcpy(pRule->zFrom, zFrom, nFrom+1); memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; + pRule->nTo = (fuzzer_len)nTo; pRule->rCost = nCost; pRule->iRuleset = (int)iRuleset; } diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 99d299c2..89dd1325 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -49,13 +49,15 @@ SQLITE_EXTENSION_INIT1 #ifdef sqlite3Isdigit /* Use the SQLite core versions if this routine is part of the ** SQLite amalgamation */ -# define safe_isdigit(x) sqlite3Isdigit(x) -# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) #else /* Use the standard library for separate compilation */ #include /* amalgamator: keep */ -# define safe_isdigit(x) isdigit((unsigned char)(x)) -# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) #endif /* @@ -702,6 +704,15 @@ static int jsonParseAddNode( return pParse->nNode++; } +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + int i; + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; + return 1; +} + /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the ** index of the first character past the end of the value parsed. @@ -776,8 +787,13 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c==0 ) return -1; if( c=='\\' ){ c = pParse->zJson[++j]; - if( c==0 ) return -1; - jnFlags = JNODE_ESCAPE; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(pParse->zJson+j+1)) ){ + jnFlags = JNODE_ESCAPE; + }else{ + return -1; + } }else if( c=='"' ){ break; } @@ -1211,6 +1227,26 @@ static void jsonTest1Func( ** Scalar SQL function implementations ****************************************************************************/ +/* +** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** corresponding to the SQL value input. Mostly this means putting +** double-quotes around strings and returning the unquoted string "null" +** when given a NULL input. +*/ +static void jsonQuoteFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString jx; + UNUSED_PARAM(argc); + + jsonInit(&jx, ctx); + jsonAppendValue(&jx, argv[0]); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + /* ** Implementation of the json_array(VALUE,...) function. Return a JSON ** array that contains all values given in arguments. Or if any argument @@ -1625,7 +1661,7 @@ static void jsonObjectFinal(sqlite3_context *ctx){ if( pStr ){ jsonAppendChar(pStr, '}'); if( pStr->bErr ){ - if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx); + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); }else{ sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, @@ -1903,9 +1939,9 @@ static int jsonEachColumn( /* For json_each() path and root are the same so fall through ** into the root case */ } - case JEACH_ROOT: { + default: { const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; + if( zRoot==0 ) zRoot = "$"; sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); break; } @@ -2124,6 +2160,7 @@ int sqlite3Json1Init(sqlite3 *db){ { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, { "json_object", -1, 0, jsonObjectFunc }, + { "json_quote", 1, 0, jsonQuoteFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, { "json_replace", -1, 0, jsonReplaceFunc }, { "json_set", -1, 1, jsonSetFunc }, diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c new file mode 100644 index 00000000..62a8a033 --- /dev/null +++ b/ext/misc/memvfs.c @@ -0,0 +1,491 @@ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an in-memory read-only VFS implementation. The application +** supplies a block of memory which is the database file, and this VFS +** uses that block of memory. +** +** Because there is no place to store journals and no good way to lock +** the "file", this VFS is read-only. +** +** USAGE: +** +** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db, +** SQLITE_OPEN_READONLY | SQLITE_OPEN_URI, +** "memvfs"); +** +** The ptr= and sz= query parameters are required or the open will fail. +** The ptr= parameter gives the memory address of the buffer holding the +** read-only database and sz= gives the size of the database. The parameter +** values may be in hexadecimal or decimal. The filename is ignored. +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + unsigned char *aData; /* content of the file */ +}; + +/* +** Methods for MemFile +*/ +static int memClose(sqlite3_file*); +static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memTruncate(sqlite3_file*, sqlite3_int64 size); +static int memSync(sqlite3_file*, int flags); +static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memLock(sqlite3_file*, int); +static int memUnlock(sqlite3_file*, int); +static int memCheckReservedLock(sqlite3_file*, int *pResOut); +static int memFileControl(sqlite3_file*, int op, void *pArg); +static int memSectorSize(sqlite3_file*); +static int memDeviceCharacteristics(sqlite3_file*); +static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int memShmLock(sqlite3_file*, int offset, int n, int flags); +static void memShmBarrier(sqlite3_file*); +static int memShmUnmap(sqlite3_file*, int deleteFlag); +static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memDlOpen(sqlite3_vfs*, const char *zFilename); +static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memDlClose(sqlite3_vfs*, void*); +static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memSleep(sqlite3_vfs*, int microseconds); +static int memCurrentTime(sqlite3_vfs*, double*); +static int memGetLastError(sqlite3_vfs*, int, char *); +static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs mem_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memvfs", /* zName */ + 0, /* pAppData (set when registered) */ + memOpen, /* xOpen */ + memDelete, /* xDelete */ + memAccess, /* xAccess */ + memFullPathname, /* xFullPathname */ + memDlOpen, /* xDlOpen */ + memDlError, /* xDlError */ + memDlSym, /* xDlSym */ + memDlClose, /* xDlClose */ + memRandomness, /* xRandomness */ + memSleep, /* xSleep */ + memCurrentTime, /* xCurrentTime */ + memGetLastError, /* xGetLastError */ + memCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods mem_io_methods = { + 3, /* iVersion */ + memClose, /* xClose */ + memRead, /* xRead */ + memWrite, /* xWrite */ + memTruncate, /* xTruncate */ + memSync, /* xSync */ + memFileSize, /* xFileSize */ + memLock, /* xLock */ + memUnlock, /* xUnlock */ + memCheckReservedLock, /* xCheckReservedLock */ + memFileControl, /* xFileControl */ + memSectorSize, /* xSectorSize */ + memDeviceCharacteristics, /* xDeviceCharacteristics */ + memShmMap, /* xShmMap */ + memShmLock, /* xShmLock */ + memShmBarrier, /* xShmBarrier */ + memShmUnmap, /* xShmUnmap */ + memFetch, /* xFetch */ + memUnfetch /* xUnfetch */ +}; + + + +/* +** Close an mem-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. +*/ +static int memClose(sqlite3_file *pFile){ + return SQLITE_OK; +} + +/* +** Read data from an mem-file. +*/ +static int memRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + memcpy(zBuf, p->aData+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Write data to an mem-file. +*/ +static int memWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + return SQLITE_READONLY; +} + +/* +** Truncate an mem-file. +*/ +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ + return SQLITE_READONLY; +} + +/* +** Sync an mem-file. +*/ +static int memSync(sqlite3_file *pFile, int flags){ + return SQLITE_READONLY; +} + +/* +** Return the current file-size of an mem-file. +*/ +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an mem-file. +*/ +static int memLock(sqlite3_file *pFile, int eLock){ + return SQLITE_READONLY; +} + +/* +** Unlock an mem-file. +*/ +static int memUnlock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an mem-file. +*/ +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an mem-file. +*/ +static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return the sector-size in bytes for an mem-file. +*/ +static int memSectorSize(sqlite3_file *pFile){ + return 1024; +} + +/* +** Return the device characteristic flags supported by an mem-file. +*/ +static int memDeviceCharacteristics(sqlite3_file *pFile){ + return SQLITE_IOCAP_IMMUTABLE; +} + +/* Create a shared memory file mapping */ +static int memShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + return SQLITE_READONLY; +} + +/* Perform locking on a shared-memory segment */ +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + return SQLITE_READONLY; +} + +/* Memory barrier operation on shared memory */ +static void memShmBarrier(sqlite3_file *pFile){ + return; +} + +/* Unmap a shared memory segment */ +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int memFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemFile *p = (MemFile *)pFile; + *pp = (void*)(p->aData + iOfst); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + MemFile *p = (MemFile*)pFile; + memset(p, 0, sizeof(*p)); + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; + p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); + if( p->aData==0 ) return SQLITE_CANTOPEN; + p->sz = sqlite3_uri_int64(zName,"sz",0); + if( p->sz<0 ) return SQLITE_CANTOPEN; + pFile->pMethods = &mem_io_methods; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_READONLY; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int memAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + /* The spec says there are three possible values for flags. But only + ** two of them are actually used */ + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + if( flags==SQLITE_ACCESS_READWRITE ){ + *pResOut = 0; + }else{ + *pResOut = 1; + } + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} + +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +#ifdef MEMVFS_TEST +/* +** memload(FILENAME) +** +** This an SQL function used to help in testing the memvfs VFS. The +** function reads the content of a file into memory and then returns +** a string that gives the locate and size of the in-memory buffer. +*/ +#include +static void memvfsMemloadFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *p; + sqlite3_int64 sz; + FILE *in; + const char *zFilename = (const char*)sqlite3_value_text(argv[0]); + char zReturn[100]; + + if( zFilename==0 ) return; + in = fopen(zFilename, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + sz = ftell(in); + rewind(in); + p = sqlite3_malloc( sz ); + if( p==0 ){ + fclose(in); + sqlite3_result_error_nomem(context); + return; + } + fread(p, sz, 1, in); + fclose(in); + sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld", + (sqlite3_int64)p, sz); + sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT); +} +/* Called for each new database connection */ +static int memvfsRegister( + sqlite3 *db, + const char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0, + memvfsMemloadFunc, 0, 0); +} +#endif /* MEMVFS_TEST */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_memvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + mem_vfs.pAppData = sqlite3_vfs_find(0); + mem_vfs.szOsFile = sizeof(MemFile); + rc = sqlite3_vfs_register(&mem_vfs, 1); +#ifdef MEMVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index 74a4c9d1..a5d7e849 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -167,7 +167,7 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ /* ** Compare to doubles for sorting using qsort() */ -static int doubleCmp(const void *pA, const void *pB){ +static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){ double a = *(double*)pA; double b = *(double*)pB; if( a==b ) return 0; diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 7244d529..b4a8ab5c 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -136,7 +136,7 @@ struct ReCompiled { static void re_add_state(ReStateSet *pSet, int newState){ unsigned i; for(i=0; inState; i++) if( pSet->aState[i]==newState ) return; - pSet->aState[pSet->nState++] = newState; + pSet->aState[pSet->nState++] = (ReStateNumber)newState; } /* Extract the next unicode character from *pzIn and return it. Advance @@ -358,7 +358,7 @@ static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ p->aArg[i] = p->aArg[i-1]; } p->nState++; - p->aOp[iBefore] = op; + p->aOp[iBefore] = (char)op; p->aArg[iBefore] = arg; return iBefore; } @@ -677,12 +677,12 @@ const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ for(j=0, i=1; jzInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ unsigned x = pRe->aArg[i]; if( x<=127 ){ - pRe->zInit[j++] = x; + pRe->zInit[j++] = (unsigned char)x; }else if( x<=0xfff ){ - pRe->zInit[j++] = 0xc0 | (x>>6); + pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); pRe->zInit[j++] = 0x80 | (x&0x3f); }else if( x<=0xffff ){ - pRe->zInit[j++] = 0xd0 | (x>>12); + pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12)); pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); pRe->zInit[j++] = 0x80 | (x&0x3f); }else{ diff --git a/ext/misc/scrub.c b/ext/misc/scrub.c new file mode 100644 index 00000000..92718e23 --- /dev/null +++ b/ext/misc/scrub.c @@ -0,0 +1,610 @@ +/* +** 2016-05-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a utility function (and a utility program) that +** makes a copy of an SQLite database while simultaneously zeroing out all +** deleted content. +** +** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite +** deletes content, it does not overwrite the deleted content but rather marks +** the region of the file that held that content as being reusable. This can +** cause deleted content to recoverable from the database file. This stale +** content is removed by the VACUUM command, but VACUUM can be expensive for +** large databases. When in PRAGMA secure_delete=ON mode, the deleted content +** is zeroed, but secure_delete=ON has overhead as well. +** +** This utility attempts to make a copy of a complete SQLite database where +** all of the deleted content is zeroed out in the copy, and it attempts to +** do so while being faster than running VACUUM. +** +** Usage: +** +** int sqlite3_scrub_backup( +** const char *zSourceFile, // Source database filename +** const char *zDestFile, // Destination database filename +** char **pzErrMsg // Write error message here +** ); +** +** Simply call the API above specifying the filename of the source database +** and the name of the backup copy. The source database must already exist +** and can be in active use. (A read lock is held during the backup.) The +** destination file should not previously exist. If the pzErrMsg parameter +** is non-NULL and if an error occurs, then an error message might be written +** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to +** that error message. But if the error is an OOM, the error might not be +** reported. The routine always returns non-zero if there is an error. +** +** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and +** this file becomes a standalone program that can be run as follows: +** +** ./sqlite3scrub SOURCE DEST +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include + +typedef struct ScrubState ScrubState; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; + + +/* State information for a scrub-and-backup operation */ +struct ScrubState { + const char *zSrcFile; /* Name of the source file */ + const char *zDestFile; /* Name of the destination file */ + int rcErr; /* Error code */ + char *zErr; /* Error message text */ + sqlite3 *dbSrc; /* Source database connection */ + sqlite3_file *pSrc; /* Source file handle */ + sqlite3 *dbDest; /* Destination database connection */ + sqlite3_file *pDest; /* Destination file handle */ + u32 szPage; /* Page size */ + u32 szUsable; /* Usable bytes on each page */ + u32 nPage; /* Number of pages */ + u32 iLastPage; /* Page number of last page written so far*/ + u8 *page1; /* Content of page 1 */ +}; + +/* Store an error message */ +static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){ + va_list ap; + sqlite3_free(p->zErr); + va_start(ap, zFormat); + p->zErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR; +} + +/* Allocate memory to hold a single page of content */ +static u8 *scrubBackupAllocPage(ScrubState *p){ + u8 *pPage; + if( p->rcErr ) return 0; + pPage = sqlite3_malloc( p->szPage ); + if( pPage==0 ) p->rcErr = SQLITE_NOMEM; + return pPage; +} + +/* Read a page from the source database into memory. Use the memory +** provided by pBuf if not NULL or allocate a new page if pBuf==NULL. +*/ +static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){ + int rc; + sqlite3_int64 iOff; + u8 *pOut = pBuf; + if( p->rcErr ) return 0; + if( pOut==0 ){ + pOut = scrubBackupAllocPage(p); + if( pOut==0 ) return 0; + } + iOff = (pgno-1)*(sqlite3_int64)p->szPage; + rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff); + if( rc!=SQLITE_OK ){ + if( pBuf==0 ) sqlite3_free(pOut); + pOut = 0; + scrubBackupErr(p, "read failed for page %d", pgno); + p->rcErr = SQLITE_IOERR; + } + return pOut; +} + +/* Write a page to the destination database */ +static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){ + int rc; + sqlite3_int64 iOff; + if( p->rcErr ) return; + iOff = (pgno-1)*(sqlite3_int64)p->szPage; + rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff); + if( rc!=SQLITE_OK ){ + scrubBackupErr(p, "write failed for page %d", pgno); + p->rcErr = SQLITE_IOERR; + } + if( pgno>p->iLastPage ) p->iLastPage = pgno; +} + +/* Prepare a statement against the "db" database. */ +static sqlite3_stmt *scrubBackupPrepare( + ScrubState *p, /* Backup context */ + sqlite3 *db, /* Database to prepare against */ + const char *zSql /* SQL statement */ +){ + sqlite3_stmt *pStmt; + if( p->rcErr ) return 0; + p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( p->rcErr ){ + scrubBackupErr(p, "SQL error \"%s\" on \"%s\"", + sqlite3_errmsg(db), zSql); + sqlite3_finalize(pStmt); + return 0; + } + return pStmt; +} + + +/* Open the source database file */ +static void scrubBackupOpenSrc(ScrubState *p){ + sqlite3_stmt *pStmt; + int rc; + /* Open the source database file */ + p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc, + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); + if( p->rcErr ){ + scrubBackupErr(p, "cannot open source database: %s", + sqlite3_errmsg(p->dbSrc)); + return; + } + p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;", + 0, 0, 0); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot start a read transaction on the source database: %s", + sqlite3_errmsg(p->dbSrc)); + return; + } + rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL, + 0, 0); + if( rc ){ + scrubBackupErr(p, "cannot checkpoint the source database"); + return; + } + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->szPage = sqlite3_column_int(pStmt, 0); + }else{ + scrubBackupErr(p, "unable to determine the page size"); + } + sqlite3_finalize(pStmt); + if( p->rcErr ) return; + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nPage = sqlite3_column_int(pStmt, 0); + }else{ + scrubBackupErr(p, "unable to determine the size of the source database"); + } + sqlite3_finalize(pStmt); + sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc); + if( p->pSrc==0 || p->pSrc->pMethods==0 ){ + scrubBackupErr(p, "cannot get the source file handle"); + p->rcErr = SQLITE_ERROR; + } +} + +/* Create and open the destination file */ +static void scrubBackupOpenDest(ScrubState *p){ + sqlite3_stmt *pStmt; + int rc; + char *zSql; + if( p->rcErr ) return; + p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); + if( p->rcErr ){ + scrubBackupErr(p, "cannot open destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage); + if( zSql==0 ){ + p->rcErr = SQLITE_NOMEM; + return; + } + p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot set the page size on the destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0); + p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot start a write transaction on the destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + scrubBackupErr(p, "cannot measure the size of the destination"); + }else if( sqlite3_column_int(pStmt, 0)>1 ){ + scrubBackupErr(p, "destination database is not empty - holds %d pages", + sqlite3_column_int(pStmt, 0)); + } + sqlite3_finalize(pStmt); + sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest); + if( p->pDest==0 || p->pDest->pMethods==0 ){ + scrubBackupErr(p, "cannot get the destination file handle"); + p->rcErr = SQLITE_ERROR; + } +} + +/* Read a 32-bit big-endian integer */ +static u32 scrubBackupInt32(const u8 *a){ + u32 v = a[3]; + v += ((u32)a[2])<<8; + v += ((u32)a[1])<<16; + v += ((u32)a[0])<<24; + return v; +} + +/* Read a 16-bit big-endian integer */ +static u32 scrubBackupInt16(const u8 *a){ + return (a[0]<<8) + a[1]; +} + +/* +** Read a varint. Put the value in *pVal and return the number of bytes. +*/ +static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* +** Return the number of bytes in a varint. +*/ +static int scrubBackupVarintSize(const u8 *z){ + int i; + for(i=0; i<8; i++){ + if( (z[i]&0x80)==0 ){ return i+1; } + } + return 9; +} + +/* +** Copy the freelist trunk page given, and all its descendents, +** zeroing out as much as possible in the process. +*/ +static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){ + u8 *a, *aBuf; + u32 n, mx; + + if( p->rcErr ) return; + aBuf = scrubBackupAllocPage(p); + if( aBuf==0 ) return; + + while( pgno && nFree){ + a = scrubBackupRead(p, pgno, aBuf); + if( a==0 ) break; + n = scrubBackupInt32(&a[4]); + mx = p->szUsable/4 - 2; + if( nszPage); + } + scrubBackupWrite(p, iLeaf, aZero); + nFree--; + } +#endif + } + sqlite3_free(aBuf); +} + +/* +** Copy an overflow chain from source to destination. Zero out any +** unused tail at the end of the overflow chain. +*/ +static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){ + u8 *a, *aBuf; + + aBuf = scrubBackupAllocPage(p); + if( aBuf==0 ) return; + while( nByte>0 && pgno!=0 ){ + a = scrubBackupRead(p, pgno, aBuf); + if( a==0 ) break; + if( nByte >= (p->szUsable)-4 ){ + nByte -= (p->szUsable) - 4; + }else{ + u32 x = (p->szUsable - 4) - nByte; + u32 i = p->szUsable - x; + memset(&a[i], 0, x); + nByte = 0; + } + scrubBackupWrite(p, pgno, a); + pgno = scrubBackupInt32(a); + } + sqlite3_free(aBuf); +} + + +/* +** Copy B-Tree page pgno, and all of its children, from source to destination. +** Zero out deleted content during the copy. +*/ +static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){ + u8 *a; + u32 i, n, pc; + u32 nCell; + u32 nPrefix; + u32 szHdr; + u32 iChild; + u8 *aTop; + u8 *aCell; + u32 x, y; + int ln = 0; + + + if( p->rcErr ) return; + if( iDepth>50 ){ + scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno); + return; + } + if( pgno==1 ){ + a = p->page1; + }else{ + a = scrubBackupRead(p, pgno, 0); + if( a==0 ) return; + } + nPrefix = pgno==1 ? 100 : 0; + aTop = &a[nPrefix]; + szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05); + aCell = aTop + szHdr; + nCell = scrubBackupInt16(&aTop[3]); + + /* Zero out the gap between the cell index and the start of the + ** cell content area */ + x = scrubBackupInt16(&aTop[5]); /* First byte of cell content area */ + if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; } + y = szHdr + nPrefix + nCell*2; + if( y>x ){ ln=__LINE__; goto btree_corrupt; } + if( y0 && pc(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; } + n = scrubBackupInt16(&a[pc+2]); + if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; } + if( n>4 ) memset(&a[pc+4], 0, n-4); + x = scrubBackupInt16(&a[pc]); + if( x0 ){ ln=__LINE__; goto btree_corrupt; } + pc = x; + } + + /* Write this one page */ + scrubBackupWrite(p, pgno, a); + + /* Walk the tree and process child pages */ + for(i=0; i p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; } + if( aTop[0]==0x05 || aTop[0]==0x02 ){ + if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; } + iChild = scrubBackupInt32(&a[pc]); + pc += 4; + scrubBackupBtree(p, iChild, iDepth+1); + if( aTop[0]==0x05 ) continue; + } + pc += scrubBackupVarint(&a[pc], &P); + if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; } + if( aTop[0]==0x0d ){ + X = p->szUsable - 35; + }else{ + X = ((p->szUsable - 12)*64/255) - 23; + } + if( P<=X ){ + /* All content is local. No overflow */ + continue; + } + M = ((p->szUsable - 12)*32/255)-23; + K = M + ((P-M)%(p->szUsable-4)); + if( aTop[0]==0x0d ){ + pc += scrubBackupVarintSize(&a[pc]); + if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; } + } + nLocal = K<=X ? K : M; + if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; } + iChild = scrubBackupInt32(&a[pc+nLocal]); + scrubBackupOverflow(p, iChild, P-nLocal); + } + + /* Walk the right-most tree */ + if( aTop[0]==0x05 || aTop[0]==0x02 ){ + iChild = scrubBackupInt32(&aTop[8]); + scrubBackupBtree(p, iChild, iDepth+1); + } + + /* All done */ + if( pgno>1 ) sqlite3_free(a); + return; + +btree_corrupt: + scrubBackupErr(p, "corruption on page %d of source database (errid=%d)", + pgno, ln); + if( pgno>1 ) sqlite3_free(a); +} + +/* +** Copy all ptrmap pages from source to destination. +** This routine is only called if the source database is in autovacuum +** or incremental vacuum mode. +*/ +static void scrubBackupPtrmap(ScrubState *p){ + u32 pgno = 2; + u32 J = p->szUsable/5; + u32 iLock = (1073742335/p->szPage)+1; + u8 *a, *pBuf; + if( p->rcErr ) return; + pBuf = scrubBackupAllocPage(p); + if( pBuf==0 ) return; + while( pgno<=p->nPage ){ + a = scrubBackupRead(p, pgno, pBuf); + if( a==0 ) break; + scrubBackupWrite(p, pgno, a); + pgno += J+1; + if( pgno==iLock ) pgno++; + } + sqlite3_free(pBuf); +} + +int sqlite3_scrub_backup( + const char *zSrcFile, /* Source file */ + const char *zDestFile, /* Destination file */ + char **pzErr /* Write error here if non-NULL */ +){ + ScrubState s; + u32 n, i; + sqlite3_stmt *pStmt; + + memset(&s, 0, sizeof(s)); + s.zSrcFile = zSrcFile; + s.zDestFile = zDestFile; + + /* Open both source and destination databases */ + scrubBackupOpenSrc(&s); + scrubBackupOpenDest(&s); + + /* Read in page 1 */ + s.page1 = scrubBackupRead(&s, 1, 0); + if( s.page1==0 ) goto scrub_abort; + s.szUsable = s.szPage - s.page1[20]; + + /* Copy the freelist */ + n = scrubBackupInt32(&s.page1[36]); + i = scrubBackupInt32(&s.page1[32]); + if( n ) scrubBackupFreelist(&s, i, n); + + /* Copy ptrmap pages */ + n = scrubBackupInt32(&s.page1[52]); + if( n ) scrubBackupPtrmap(&s); + + /* Copy all of the btrees */ + scrubBackupBtree(&s, 1, 0); + pStmt = scrubBackupPrepare(&s, s.dbSrc, + "SELECT rootpage FROM sqlite_master WHERE coalesce(rootpage,0)>0"); + if( pStmt==0 ) goto scrub_abort; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i = (u32)sqlite3_column_int(pStmt, 0); + scrubBackupBtree(&s, i, 0); + } + sqlite3_finalize(pStmt); + + /* If the last page of the input db file is a free-list leaf, then the + ** backup file on disk is still smaller than the size indicated within + ** the database header. In this case, write a page of zeroes to the + ** last page of the backup database so that SQLite does not mistakenly + ** think the db is corrupt. */ + if( s.iLastPage=0 ){ pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[startIdx].omit = 1; + pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY; } if( stopIdx>=0 ){ pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[stopIdx].omit = 1; + pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; } if( stepIdx>=0 ){ pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[stepIdx].omit = 1; + pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; } if( (idxNum & 3)==3 ){ /* Both start= and stop= boundaries are available. This is the ** the preferred case */ - pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); pIdxInfo->estimatedRows = 1000; if( pIdxInfo->nOrderBy==1 ){ if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8; diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index cac1edcf..1ac1712f 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -390,7 +390,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ /* Special processing if either string is empty */ if( nA==0 ){ - cBprev = dc; + cBprev = (char)dc; for(xB=res=0; (cB = zB[xB])!=0; xB++){ res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV; cBprev = cB; @@ -398,7 +398,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ return res; } if( nB==0 ){ - cAprev = dc; + cAprev = (char)dc; for(xA=res=0; (cA = zA[xA])!=0; xA++){ res += insertOrDeleteCost(cAprev, cA, zA[xA+1]); cAprev = cA; @@ -420,8 +420,8 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ /* Compute the Wagner edit distance */ m[0] = 0; - cx[0] = dc; - cBprev = dc; + cx[0] = (char)dc; + cBprev = (char)dc; for(xB=1; xB<=nB; xB++){ cBnext = zB[xB]; cB = zB[xB-1]; @@ -429,7 +429,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB, cBnext); cBprev = cB; } - cAprev = dc; + cAprev = (char)dc; for(xA=1; xA<=nA; xA++){ int lastA = (xA==nA); cA = zA[xA-1]; @@ -476,7 +476,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ d = m[xB]; dc = cx[xB]; m[xB] = totalCost; - cx[xB] = ncx; + cx[xB] = (char)ncx; cBprev = cB; } cAprev = cA; @@ -711,9 +711,9 @@ static int editDist3ConfigLoad( if( nExtra<0 ) nExtra = 0; pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra ); if( pCost==0 ){ rc = SQLITE_NOMEM; break; } - pCost->nFrom = nFrom; - pCost->nTo = nTo; - pCost->iCost = iCost; + pCost->nFrom = (u8)nFrom; + pCost->nTo = (u8)nTo; + pCost->iCost = (u16)iCost; memcpy(pCost->a, zFrom, nFrom); memcpy(pCost->a + nFrom, zTo, nTo); pCost->pNext = pLang->pCost; @@ -1616,7 +1616,7 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){ zIn += sz; nIn -= sz; if( c<=127 ){ - zOut[nOut++] = c; + zOut[nOut++] = (unsigned char)c; }else{ int xTop, xBtm, x; xTop = sizeof(translit)/sizeof(translit[0]) - 1; @@ -1734,6 +1734,7 @@ static void scriptCodeSqlFunc( int c, sz; int scriptMask = 0; int res; + int seenDigit = 0; # define SCRIPT_LATIN 0x0001 # define SCRIPT_CYRILLIC 0x0002 # define SCRIPT_GREEK 0x0004 @@ -1744,8 +1745,12 @@ static void scriptCodeSqlFunc( c = utf8Read(zIn, nIn, &sz); zIn += sz; nIn -= sz; - if( c<0x02af && (c>=0x80 || midClass[c&0x7f]=0x80 || midClass[c&0x7f]='0' && c<='9' ){ + seenDigit = 1; + } }else if( c>=0x0400 && c<=0x04ff ){ scriptMask |= SCRIPT_CYRILLIC; }else if( c>=0x0386 && c<=0x03ce ){ @@ -1756,6 +1761,7 @@ static void scriptCodeSqlFunc( scriptMask |= SCRIPT_ARABIC; } } + if( scriptMask==0 && seenDigit ) scriptMask = SCRIPT_LATIN; switch( scriptMask ){ case 0: res = 999; break; case SCRIPT_LATIN: res = 215; break; @@ -1775,7 +1781,7 @@ static void scriptCodeSqlFunc( */ /* Maximum length of a phonehash used for querying the shadow table */ -#define SPELLFIX_MX_HASH 8 +#define SPELLFIX_MX_HASH 32 /* Maximum number of hash strings to examine per query */ #define SPELLFIX_MX_RUN 1 @@ -2225,7 +2231,7 @@ static int spellfix1Score(int iDistance, int iRank){ ** Compare two spellfix1_row objects for sorting purposes in qsort() such ** that they sort in order of increasing distance. */ -static int spellfix1RowCompare(const void *A, const void *B){ +static int SQLITE_CDECL spellfix1RowCompare(const void *A, const void *B){ const struct spellfix1_row *a = (const struct spellfix1_row*)A; const struct spellfix1_row *b = (const struct spellfix1_row*)B; return a->iScore - b->iScore; diff --git a/ext/misc/vfsstat.c b/ext/misc/vfsstat.c new file mode 100644 index 00000000..14753d27 --- /dev/null +++ b/ext/misc/vfsstat.c @@ -0,0 +1,816 @@ +/* +** 2016-05-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the implementation of an SQLite vfs shim that +** tracks I/O. Access to the accumulated status counts is provided using +** an eponymous virtual table. +*/ +#include +SQLITE_EXTENSION_INIT1 + +/* +** This module contains code for a wrapper VFS that cause stats for +** most VFS calls to be recorded. +** +** To use this module, first compile it as a loadable extension. See +** https://www.sqlite.org/loadext.html#build for compilations instructions. +** +** After compliing, load this extension, then open database connections to be +** measured. Query usages status using the vfsstat virtual table: +** +** SELECT * FROM vfsstat; +** +** Reset counters using UPDATE statements against vfsstat: +** +** UPDATE vfsstat SET count=0; +** +** EXAMPLE SCRIPT: +** +** .load ./vfsstat +** .open test.db +** DROP TABLE IF EXISTS t1; +** CREATE TABLE t1(x,y); +** INSERT INTO t1 VALUES(123, randomblob(5000)); +** CREATE INDEX t1x ON t1(x); +** DROP TABLE t1; +** VACUUM; +** SELECT * FROM vfsstat WHERE count>0; +** +** LIMITATIONS: +** +** This module increments counters without using mutex protection. So if +** two or more threads try to use this module at the same time, race conditions +** may occur which mess up the counts. This is harmless, other than giving +** incorrect statistics. +*/ +#include +#include +#include + +/* +** File types +*/ +#define VFSSTAT_MAIN 0 /* Main database file */ +#define VFSSTAT_JOURNAL 1 /* Rollback journal */ +#define VFSSTAT_WAL 2 /* Write-ahead log file */ +#define VFSSTAT_MASTERJRNL 3 /* Master journal */ +#define VFSSTAT_SUBJRNL 4 /* Subjournal */ +#define VFSSTAT_TEMPDB 5 /* TEMP database */ +#define VFSSTAT_TEMPJRNL 6 /* Journal for TEMP database */ +#define VFSSTAT_TRANSIENT 7 /* Transient database */ +#define VFSSTAT_ANY 8 /* Unspecified file type */ +#define VFSSTAT_nFile 9 /* This many file types */ + +/* Names of the file types. These are allowed values for the +** first column of the vfsstat virtual table. +*/ +static const char *azFile[] = { + "database", "journal", "wal", "master-journal", "sub-journal", + "temp-database", "temp-journal", "transient-db", "*" +}; + +/* +** Stat types +*/ +#define VFSSTAT_BYTESIN 0 /* Bytes read in */ +#define VFSSTAT_BYTESOUT 1 /* Bytes written out */ +#define VFSSTAT_READ 2 /* Read requests */ +#define VFSSTAT_WRITE 3 /* Write requests */ +#define VFSSTAT_SYNC 4 /* Syncs */ +#define VFSSTAT_OPEN 5 /* File opens */ +#define VFSSTAT_LOCK 6 /* Lock requests */ +#define VFSSTAT_ACCESS 0 /* xAccess calls. filetype==ANY only */ +#define VFSSTAT_DELETE 1 /* xDelete calls. filetype==ANY only */ +#define VFSSTAT_FULLPATH 2 /* xFullPathname calls. ANY only */ +#define VFSSTAT_RANDOM 3 /* xRandomness calls. ANY only */ +#define VFSSTAT_SLEEP 4 /* xSleep calls. ANY only */ +#define VFSSTAT_CURTIME 5 /* xCurrentTime calls. ANY only */ +#define VFSSTAT_nStat 7 /* This many stat types */ + + +/* Names for the second column of the vfsstat virtual table for all +** cases except when the first column is "*" or VFSSTAT_ANY. */ +static const char *azStat[] = { + "bytes-in", "bytes-out", "read", "write", "sync", "open", "lock", +}; +static const char *azStatAny[] = { + "access", "delete", "fullpathname", "randomness", "sleep", "currenttimestamp", + "not-used" +}; + +/* Total number of counters */ +#define VFSSTAT_MXCNT (VFSSTAT_nStat*VFSSTAT_nFile) + +/* +** Performance stats are collected in an instance of the following +** global array. +*/ +static sqlite3_uint64 aVfsCnt[VFSSTAT_MXCNT]; + +/* +** Access to a specific counter +*/ +#define STATCNT(filetype,stat) (aVfsCnt[(filetype)*VFSSTAT_nStat+(stat)]) + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct VStatVfs VStatVfs; +typedef struct VStatFile VStatFile; + +/* An instance of the VFS */ +struct VStatVfs { + sqlite3_vfs base; /* VFS methods */ + sqlite3_vfs *pVfs; /* Parent VFS */ +}; + +/* An open file */ +struct VStatFile { + sqlite3_file base; /* IO methods */ + sqlite3_file *pReal; /* Underlying file handle */ + unsigned char eFiletype; /* What type of file is this */ +}; + +#define REALVFS(p) (((VStatVfs*)(p))->pVfs) + +/* +** Methods for VStatFile +*/ +static int vstatClose(sqlite3_file*); +static int vstatRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int vstatWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int vstatTruncate(sqlite3_file*, sqlite3_int64 size); +static int vstatSync(sqlite3_file*, int flags); +static int vstatFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int vstatLock(sqlite3_file*, int); +static int vstatUnlock(sqlite3_file*, int); +static int vstatCheckReservedLock(sqlite3_file*, int *pResOut); +static int vstatFileControl(sqlite3_file*, int op, void *pArg); +static int vstatSectorSize(sqlite3_file*); +static int vstatDeviceCharacteristics(sqlite3_file*); +static int vstatShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int vstatShmLock(sqlite3_file*, int offset, int n, int flags); +static void vstatShmBarrier(sqlite3_file*); +static int vstatShmUnmap(sqlite3_file*, int deleteFlag); +static int vstatFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int vstatUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for VStatVfs +*/ +static int vstatOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int vstatDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int vstatAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int vstatFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *vstatDlOpen(sqlite3_vfs*, const char *zFilename); +static void vstatDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void vstatDlClose(sqlite3_vfs*, void*); +static int vstatRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int vstatSleep(sqlite3_vfs*, int microseconds); +static int vstatCurrentTime(sqlite3_vfs*, double*); +static int vstatGetLastError(sqlite3_vfs*, int, char *); +static int vstatCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static VStatVfs vstat_vfs = { + { + 2, /* iVersion */ + 0, /* szOsFile (set by register_vstat()) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "vfslog", /* zName */ + 0, /* pAppData */ + vstatOpen, /* xOpen */ + vstatDelete, /* xDelete */ + vstatAccess, /* xAccess */ + vstatFullPathname, /* xFullPathname */ + vstatDlOpen, /* xDlOpen */ + vstatDlError, /* xDlError */ + vstatDlSym, /* xDlSym */ + vstatDlClose, /* xDlClose */ + vstatRandomness, /* xRandomness */ + vstatSleep, /* xSleep */ + vstatCurrentTime, /* xCurrentTime */ + vstatGetLastError, /* xGetLastError */ + vstatCurrentTimeInt64 /* xCurrentTimeInt64 */ + }, + 0 +}; + +static const sqlite3_io_methods vstat_io_methods = { + 3, /* iVersion */ + vstatClose, /* xClose */ + vstatRead, /* xRead */ + vstatWrite, /* xWrite */ + vstatTruncate, /* xTruncate */ + vstatSync, /* xSync */ + vstatFileSize, /* xFileSize */ + vstatLock, /* xLock */ + vstatUnlock, /* xUnlock */ + vstatCheckReservedLock, /* xCheckReservedLock */ + vstatFileControl, /* xFileControl */ + vstatSectorSize, /* xSectorSize */ + vstatDeviceCharacteristics, /* xDeviceCharacteristics */ + vstatShmMap, /* xShmMap */ + vstatShmLock, /* xShmLock */ + vstatShmBarrier, /* xShmBarrier */ + vstatShmUnmap, /* xShmUnmap */ + vstatFetch, /* xFetch */ + vstatUnfetch /* xUnfetch */ +}; + + + +/* +** Close an vstat-file. +*/ +static int vstatClose(sqlite3_file *pFile){ + VStatFile *p = (VStatFile *)pFile; + int rc = SQLITE_OK; + + if( p->pReal->pMethods ){ + rc = p->pReal->pMethods->xClose(p->pReal); + } + return rc; +} + + +/* +** Read data from an vstat-file. +*/ +static int vstatRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + VStatFile *p = (VStatFile *)pFile; + + rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + STATCNT(p->eFiletype,VFSSTAT_READ)++; + if( rc==SQLITE_OK ){ + STATCNT(p->eFiletype,VFSSTAT_BYTESIN) += iAmt; + } + return rc; +} + +/* +** Write data to an vstat-file. +*/ +static int vstatWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + VStatFile *p = (VStatFile *)pFile; + + rc = p->pReal->pMethods->xWrite(p->pReal, z, iAmt, iOfst); + STATCNT(p->eFiletype,VFSSTAT_WRITE)++; + if( rc==SQLITE_OK ){ + STATCNT(p->eFiletype,VFSSTAT_BYTESOUT) += iAmt; + } + return rc; +} + +/* +** Truncate an vstat-file. +*/ +static int vstatTruncate(sqlite3_file *pFile, sqlite_int64 size){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xTruncate(p->pReal, size); + return rc; +} + +/* +** Sync an vstat-file. +*/ +static int vstatSync(sqlite3_file *pFile, int flags){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xSync(p->pReal, flags); + STATCNT(p->eFiletype,VFSSTAT_SYNC)++; + return rc; +} + +/* +** Return the current file-size of an vstat-file. +*/ +static int vstatFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + return rc; +} + +/* +** Lock an vstat-file. +*/ +static int vstatLock(sqlite3_file *pFile, int eLock){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + STATCNT(p->eFiletype,VFSSTAT_LOCK)++; + return rc; +} + +/* +** Unlock an vstat-file. +*/ +static int vstatUnlock(sqlite3_file *pFile, int eLock){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xUnlock(p->pReal, eLock); + STATCNT(p->eFiletype,VFSSTAT_LOCK)++; + return rc; +} + +/* +** Check if another file-handle holds a RESERVED lock on an vstat-file. +*/ +static int vstatCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); + STATCNT(p->eFiletype,VFSSTAT_LOCK)++; + return rc; +} + +/* +** File control method. For custom operations on an vstat-file. +*/ +static int vstatFileControl(sqlite3_file *pFile, int op, void *pArg){ + VStatFile *p = (VStatFile *)pFile; + int rc; + rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("vstat/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an vstat-file. +*/ +static int vstatSectorSize(sqlite3_file *pFile){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xSectorSize(p->pReal); + return rc; +} + +/* +** Return the device characteristic flags supported by an vstat-file. +*/ +static int vstatDeviceCharacteristics(sqlite3_file *pFile){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal); + return rc; +} + +/* Create a shared memory file mapping */ +static int vstatShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xShmMap(p->pReal, iPg, pgsz, bExtend, pp); +} + +/* Perform locking on a shared-memory segment */ +static int vstatShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xShmLock(p->pReal, offset, n, flags); +} + +/* Memory barrier operation on shared memory */ +static void vstatShmBarrier(sqlite3_file *pFile){ + VStatFile *p = (VStatFile *)pFile; + p->pReal->pMethods->xShmBarrier(p->pReal); +} + +/* Unmap a shared memory segment */ +static int vstatShmUnmap(sqlite3_file *pFile, int deleteFlag){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xShmUnmap(p->pReal, deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int vstatFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xFetch(p->pReal, iOfst, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int vstatUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xUnfetch(p->pReal, iOfst, pPage); +} + +/* +** Open an vstat file handle. +*/ +static int vstatOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + int rc; + VStatFile *p = (VStatFile*)pFile; + + p->pReal = (sqlite3_file*)&p[1]; + rc = REALVFS(pVfs)->xOpen(REALVFS(pVfs), zName, p->pReal, flags, pOutFlags); + if( flags & SQLITE_OPEN_MAIN_DB ){ + p->eFiletype = VFSSTAT_MAIN; + }else if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + p->eFiletype = VFSSTAT_JOURNAL; + }else if( flags & SQLITE_OPEN_WAL ){ + p->eFiletype = VFSSTAT_WAL; + }else if( flags & SQLITE_OPEN_MASTER_JOURNAL ){ + p->eFiletype = VFSSTAT_MASTERJRNL; + }else if( flags & SQLITE_OPEN_SUBJOURNAL ){ + p->eFiletype = VFSSTAT_SUBJRNL; + }else if( flags & SQLITE_OPEN_TEMP_DB ){ + p->eFiletype = VFSSTAT_TEMPDB; + }else if( flags & SQLITE_OPEN_TEMP_JOURNAL ){ + p->eFiletype = VFSSTAT_TEMPJRNL; + }else{ + p->eFiletype = VFSSTAT_TRANSIENT; + } + STATCNT(p->eFiletype,VFSSTAT_OPEN)++; + pFile->pMethods = rc ? 0 : &vstat_io_methods; + return rc; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int vstatDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc; + rc = REALVFS(pVfs)->xDelete(REALVFS(pVfs), zPath, dirSync); + STATCNT(VFSSTAT_ANY,VFSSTAT_DELETE)++; + return rc; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int vstatAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + int rc; + rc = REALVFS(pVfs)->xAccess(REALVFS(pVfs), zPath, flags, pResOut); + STATCNT(VFSSTAT_ANY,VFSSTAT_ACCESS)++; + return rc; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int vstatFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + STATCNT(VFSSTAT_ANY,VFSSTAT_FULLPATH)++; + return REALVFS(pVfs)->xFullPathname(REALVFS(pVfs), zPath, nOut, zOut); +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *vstatDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return REALVFS(pVfs)->xDlOpen(REALVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void vstatDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + REALVFS(pVfs)->xDlError(REALVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return REALVFS(pVfs)->xDlSym(REALVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void vstatDlClose(sqlite3_vfs *pVfs, void *pHandle){ + REALVFS(pVfs)->xDlClose(REALVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int vstatRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + STATCNT(VFSSTAT_ANY,VFSSTAT_RANDOM)++; + return REALVFS(pVfs)->xRandomness(REALVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int vstatSleep(sqlite3_vfs *pVfs, int nMicro){ + STATCNT(VFSSTAT_ANY,VFSSTAT_SLEEP)++; + return REALVFS(pVfs)->xSleep(REALVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int vstatCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++; + return REALVFS(pVfs)->xCurrentTime(REALVFS(pVfs), pTimeOut); +} + +static int vstatGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return REALVFS(pVfs)->xGetLastError(REALVFS(pVfs), a, b); +} +static int vstatCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++; + return REALVFS(pVfs)->xCurrentTimeInt64(REALVFS(pVfs), p); +} + +/* +** A virtual table for accessing the stats collected by this VFS shim +*/ +static int vstattabConnect(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int vstattabBestIndex(sqlite3_vtab*,sqlite3_index_info*); +static int vstattabDisconnect(sqlite3_vtab*); +static int vstattabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); +static int vstattabClose(sqlite3_vtab_cursor*); +static int vstattabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); +static int vstattabNext(sqlite3_vtab_cursor*); +static int vstattabEof(sqlite3_vtab_cursor*); +static int vstattabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); +static int vstattabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); +static int vstattabUpdate(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); + +/* A cursor for the vfsstat virtual table */ +typedef struct VfsStatCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + int i; /* Pointing to this aVfsCnt[] value */ +} VfsStatCursor; + + +static int vstattabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define VSTAT_COLUMN_FILE 0 +#define VSTAT_COLUMN_STAT 1 +#define VSTAT_COLUMN_COUNT 2 + + rc = sqlite3_declare_vtab(db,"CREATE TABLE x(file,stat,count)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for vstat table object. +*/ +static int vstattabDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new vstat table cursor object. +*/ +static int vstattabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + VfsStatCursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + + +/* +** Destructor for a VfsStatCursor. +*/ +static int vstattabClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a VfsStatCursor to its next row of output. +*/ +static int vstattabNext(sqlite3_vtab_cursor *cur){ + ((VfsStatCursor*)cur)->i++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the VfsStatCursor +** is currently pointing. +*/ +static int vstattabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + VfsStatCursor *pCur = (VfsStatCursor*)cur; + switch( i ){ + case VSTAT_COLUMN_FILE: { + sqlite3_result_text(ctx, azFile[pCur->i/VFSSTAT_nStat], -1, SQLITE_STATIC); + break; + } + case VSTAT_COLUMN_STAT: { + const char **az; + az = (pCur->i/VFSSTAT_nStat)==VFSSTAT_ANY ? azStatAny : azStat; + sqlite3_result_text(ctx, az[pCur->i%VFSSTAT_nStat], -1, SQLITE_STATIC); + break; + } + case VSTAT_COLUMN_COUNT: { + sqlite3_result_int64(ctx, aVfsCnt[pCur->i]); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. +*/ +static int vstattabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + VfsStatCursor *pCur = (VfsStatCursor*)cur; + *pRowid = pCur->i; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int vstattabEof(sqlite3_vtab_cursor *cur){ + VfsStatCursor *pCur = (VfsStatCursor*)cur; + return pCur->i >= VFSSTAT_MXCNT; +} + +/* +** Only a full table scan is supported. So xFilter simply rewinds to +** the beginning. +*/ +static int vstattabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + VfsStatCursor *pCur = (VfsStatCursor*)pVtabCursor; + pCur->i = 0; + return SQLITE_OK; +} + +/* +** Only a forwards full table scan is supported. xBestIndex is a no-op. +*/ +static int vstattabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + return SQLITE_OK; +} + +/* +** Any VSTAT_COLUMN_COUNT can be changed to a positive integer. +** No deletions or insertions are allowed. No changes to other +** columns are allowed. +*/ +static int vstattabUpdate( + sqlite3_vtab *tab, + int argc, sqlite3_value **argv, + sqlite3_int64 *pRowid +){ + sqlite3_int64 iRowid, x; + if( argc==1 ) return SQLITE_ERROR; + if( sqlite3_value_type(argv[0])!=SQLITE_INTEGER ) return SQLITE_ERROR; + iRowid = sqlite3_value_int64(argv[0]); + if( iRowid!=sqlite3_value_int64(argv[1]) ) return SQLITE_ERROR; + if( iRowid<0 || iRowid>=VFSSTAT_MXCNT ) return SQLITE_ERROR; + if( sqlite3_value_type(argv[VSTAT_COLUMN_COUNT+2])!=SQLITE_INTEGER ){ + return SQLITE_ERROR; + } + x = sqlite3_value_int64(argv[VSTAT_COLUMN_COUNT+2]); + if( x<0 ) return SQLITE_ERROR; + aVfsCnt[iRowid] = x; + return SQLITE_OK; +} + +static sqlite3_module VfsStatModule = { + 0, /* iVersion */ + 0, /* xCreate */ + vstattabConnect, /* xConnect */ + vstattabBestIndex, /* xBestIndex */ + vstattabDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + vstattabOpen, /* xOpen - open a cursor */ + vstattabClose, /* xClose - close a cursor */ + vstattabFilter, /* xFilter - configure scan constraints */ + vstattabNext, /* xNext - advance a cursor */ + vstattabEof, /* xEof - check for end of scan */ + vstattabColumn, /* xColumn - read data */ + vstattabRowid, /* xRowid - read data */ + vstattabUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +/* +** This routine is an sqlite3_auto_extension() callback, invoked to register +** the vfsstat virtual table for all new database connections. +*/ +static int vstatRegister( + sqlite3 *db, + const char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + return sqlite3_create_module(db, "vfsstat", &VfsStatModule, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** +** Register the new VFS. Make arrangement to register the virtual table +** for each new database connection. +*/ +int sqlite3_vfsstat_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + vstat_vfs.pVfs = sqlite3_vfs_find(0); + vstat_vfs.base.szOsFile = sizeof(VStatFile) + vstat_vfs.pVfs->szOsFile; + rc = sqlite3_vfs_register(&vstat_vfs.base, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension(vstatRegister); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index 018296b8..a3efad0d 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -24,12 +24,22 @@ */ void usage(const char *zArgv0){ fprintf(stderr, -"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n" +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n" "\n" -" Argument RBU-DB must be an RBU database containing an update suitable for\n" -" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n" -" (the default value), this program attempts to apply the entire update to\n" -" the target database.\n" +"Where options are:\n" +"\n" +" -step NSTEP\n" +" -vacuum\n" +"\n" +" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n" +" database containing an update suitable for target database TARGET-DB.\n" +" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n" +" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n" +" API documentation for details).\n" +"\n" +" If NSTEP is set to less than or equal to zero (the default value), this \n" +" program attempts to perform the entire update or vacuum operation before\n" +" exiting\n" "\n" " If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" " to sqlite3rbu_step(). If the RBU update has not been completely applied\n" @@ -69,29 +79,43 @@ int main(int argc, char **argv){ char *zErrmsg; /* Error message, if any */ sqlite3rbu *pRbu; /* RBU handle */ int nStep = 0; /* Maximum number of step() calls */ + int bVacuum = 0; int rc; sqlite3_int64 nProgress = 0; + int nArg = argc-2; - /* Process command line arguments. Following this block local variables - ** zTarget, zRbu and nStep are all set. */ - if( argc==5 ){ - size_t nArg1 = strlen(argv[1]); - if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); - nStep = atoi(argv[2]); - }else if( argc!=3 ){ - usage(argv[0]); + if( argc<3 ) usage(argv[0]); + for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ + bVacuum = 1; + }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i0 && $state=="oal" && (!$oal_exists || $wal_exists)) + || ($state=="move" && (!$oal_exists || $wal_exists)) + || ($state=="checkpoint" && ($oal_exists || !$wal_exists)) + || ($state=="done" && ($oal_exists && $progress!=0)) + } { + error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } +} + +proc check_poststep_state {rc target state} { + if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} { + set oal_exists [file exists $target-oal] + set wal_exists [file exists $target-wal] + if {$state=="move" && ($oal_exists || !$wal_exists)} { + error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } + } +} + # Run the RBU in file $rbu on target database $target until completion. # proc run_rbu {target rbu} { sqlite3rbu rbu $target $rbu while 1 { + set state [rbu state] + + check_prestep_state $target $state set rc [rbu step] + check_poststep_state $rc $target $state + if {$rc!="SQLITE_OK"} break } rbu close @@ -29,10 +60,33 @@ proc run_rbu {target rbu} { proc step_rbu {target rbu} { while 1 { sqlite3rbu rbu $target $rbu + set state [rbu state] + check_prestep_state $target $state set rc [rbu step] + check_poststep_state $rc $target $state rbu close if {$rc != "SQLITE_OK"} break } set rc } +proc do_rbu_vacuum_test {tn step} { + uplevel [list do_test $tn.1 { + if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } + while 1 { + if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } + set state [rbu state] + check_prestep_state test.db $state + set rc [rbu step] + check_poststep_state $rc test.db $state + if {$rc!="SQLITE_OK"} break + if {$step==1} { rbu close } + } + rbu close + } {SQLITE_DONE}] + + uplevel [list do_execsql_test $tn.2 { + PRAGMA integrity_check + } ok] +} + diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test index 10be9e0d..6f6af828 100644 --- a/ext/rbu/rbudiff.test +++ b/ext/rbu/rbudiff.test @@ -18,22 +18,17 @@ if {![info exists testdir]} { source $testdir/tester.tcl set testprefix rbudiff -if {$tcl_platform(platform)=="windows"} { - set PROG "sqldiff.exe" -} else { - set PROG "./sqldiff" -} -if {![file exe $PROG]} { - puts "rbudiff.test cannot run because $PROG is not available" - finish_test - return -} +set PROG [test_find_sqldiff] db close proc get_rbudiff_sql {db1 db2} { exec $::PROG --rbu $db1 $db2 } +proc get_vtab_rbudiff_sql {db1 db2} { + exec $::PROG --vtab --rbu $db1 $db2 +} + proc step_rbu {target rbu} { while 1 { sqlite3rbu rbu $target $rbu @@ -45,6 +40,7 @@ proc step_rbu {target rbu} { } proc apply_rbudiff {sql target} { + test_rbucount $sql forcedelete rbu.db sqlite3 rbudb rbu.db rbudb eval $sql @@ -52,15 +48,43 @@ proc apply_rbudiff {sql target} { step_rbu $target rbu.db } +proc sqlesc {id} { + set ret "'[string map {' ''} $id]'" + set ret +} + +# The only argument is the output of an [sqldiff -rbu] run. This command +# tests that the contents of the rbu_count table is correct. An exception +# is thrown if it is not. +# +proc test_rbucount {sql} { + sqlite3 tmpdb "" + tmpdb eval $sql + tmpdb eval { + SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table' + } { + set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"] + set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}] + if {$a != $b} { + tmpdb close + error "rbu_count error - tbl = $name" + } + } + tmpdb close + return "" +} + proc rbudiff_cksum {db1} { set txt "" sqlite3 dbtmp $db1 foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] { set cols [list] - dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" } + dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { + lappend cols "quote( $name )" + } append txt [dbtmp eval \ - "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1" + "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1" ] } dbtmp close @@ -116,6 +140,24 @@ foreach {tn init mod} { ); } + 4 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO x1 VALUES('u', 'v', NULL); + INSERT INTO x1 VALUES('x', 'y', 'z'); + INSERT INTO x1 VALUES('a', NULL, 'b'); + } { + INSERT INTO x1 VALUES('a', 'b', 'c'); + } + + 5 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, X''); + } { + UPDATE t1 SET b = X'' WHERE a=1; + UPDATE t1 SET b = NULL WHERE a=2; + } + } { catch { db close } @@ -146,5 +188,116 @@ foreach {tn init mod} { do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] } +#------------------------------------------------------------------------- +# Test that if the --vtab switch is present, [sqldiff] handles virtual +# table types fts[345] and rtree correctly. +# +ifcapable fts3&&fts5&&rtree { + +foreach {tn init mod} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 2 { + CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2); + INSERT INTO "x y" VALUES(1, 2, 3); + INSERT INTO "x y" VALUES(2, 4, 6); + } { + DELETE FROM "x y" WHERE rowid = 1; + INSERT INTO "x y" VALUES(3, 6, 9); + } + + 3 { + CREATE VIRTUAL TABLE 'x''y' USING fts3; + INSERT INTO 'x''y' VALUES('one two three'); + INSERT INTO 'x''y' VALUES('four five six'); + } { + DELETE FROM 'x''y' WHERE rowid = 1; + INSERT INTO 'x''y' VALUES('one two three'); + } +} { + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db close + + do_test 2.$tn.1 { + set sql [get_vtab_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] +} + +} + +ifcapable fts5 { + foreach {tn init mod} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 2 { + CREATE VIRTUAL TABLE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 3 { + creAte virTUal +tablE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + } { + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db eval { INSERT INTO t1(t1) VALUES('optimize') } + db close + + do_test 3.$tn.1 { + set sql [get_vtab_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + + sqlite3 db test.db + sqlite3 db2 test.db2 + do_test 3.$tn.2 { + db2 eval { SELECT * FROM t1 ORDER BY rowid } + } [db eval { SELECT * FROM t1 ORDER BY rowid }] + + do_test 3.$tn.3 { + db2 eval { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} + + db close + db2 close + } +} + + finish_test diff --git a/ext/rbu/rbudor.test b/ext/rbu/rbudor.test new file mode 100644 index 00000000..dcee7ac3 --- /dev/null +++ b/ext/rbu/rbudor.test @@ -0,0 +1,59 @@ +# 2016 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test file focuses on interactions between RBU and the feature +# enabled by SQLITE_DIRECT_OVERFLOW_READ - Direct Overflow Read. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbudor + +set bigA [string repeat a 5000] +set bigB [string repeat b 5000] +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + INSERT INTO t1 VALUES(1, $bigA); +} {} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, $bigB, 0); + } + rbu close +} {} + +do_test 1.2 { + sqlite3rbu rbu test.db rbu.db + while {[rbu state]!="checkpoint"} { + rbu step + } + rbu step + db eval { SELECT * FROM t1 } +} [list 1 $bigA 2 $bigB] + +do_test 1.3 { + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} {SQLITE_DONE} + +do_execsql_test 1.4 { + SELECT * FROM t1 +} [list 1 $bigA 2 $bigB] + +finish_test + diff --git a/ext/rbu/rbufault3.test b/ext/rbu/rbufault3.test new file mode 100644 index 00000000..e72eb14f --- /dev/null +++ b/ext/rbu/rbufault3.test @@ -0,0 +1,98 @@ +# 2016 April 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests for RBU vacuum operations. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +source $testdir/malloc_common.tcl +set ::testprefix rbufault3 + +foreach {fault errlist} { + oom-* { + {1 SQLITE_NOMEM} + {1 SQLITE_IOERR_NOMEM} + {1 {SQLITE_NOMEM - out of memory}} + } + + ioerr-* { + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + {1 SQLITE_IOERR_WRITE} + {1 SQLITE_IOERR_FSYNC} + {1 SQLITE_IOERR_READ} + {1 {SQLITE_IOERR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - SQL logic error or missing database}} + } + + cantopen* { + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database file}} + {1 SQLITE_CANTOPEN} + } + +} { + + reset_db + do_execsql_test 0 { + CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y)); + INSERT INTO target VALUES(1, 2, 3); + INSERT INTO target VALUES(4, 5, 6); + INSERT INTO target VALUES(7, 8, 9); + CREATE INDEX i1 ON target(z); + } + faultsim_save_and_close + + do_faultsim_test 1 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist] + } + + do_faultsim_test 2 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + faultsim_save_and_close + + do_faultsim_test 3 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + +} + +finish_test + diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test new file mode 100644 index 00000000..af202829 --- /dev/null +++ b/ext/rbu/rbuprogress.test @@ -0,0 +1,417 @@ +# 2016 March 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuprogress + + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + create_db_file $filename { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + return $filename +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + rbu bp_progress +} {0 0} +do_test 1.2 { rbu step ; rbu bp_progress } {3333 0} +do_test 1.3 { rbu step ; rbu bp_progress } {6666 0} +do_test 1.4 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.5 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.6 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000} + +do_test 1.10 { + rbu close +} {SQLITE_DONE} + +#------------------------------------------------------------------------- +# +proc do_sp_test {tn bReopen target rbu reslist} { + uplevel [list do_test $tn [subst -nocommands { + if {$bReopen==0} { sqlite3rbu rbu $target $rbu } + set res [list] + while 1 { + if {$bReopen} { sqlite3rbu rbu $target $rbu } + set rc [rbu step] + if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" } + lappend res [lindex [rbu bp_progress] 0] + if {[lindex [set res] end]==10000} break + if {$bReopen} { rbu close } + } + if {[set res] != [list $reslist]} { + rbu close + error "1. reslist incorrect (expect=$reslist got=[set res])" + } + + # One step to clean up the temporary tables used to update the only + # target table in the rbu database. And one more to move the *-oal + # file to *-wal. After each of these steps, the progress remains + # at "10000 0". + # + if {[lindex [list $reslist] 0]!=-1} { + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "2. reslist incorrect (expect=10000 0 got=[set res])" + } + } + + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "3. reslist incorrect (expect=10000 0 got=[set res])" + } + + # Do the checkpoint. + while {[rbu step]=="SQLITE_OK"} { + foreach {a b} [rbu bp_progress] {} + if {[set a]!=10000 || [set b]<=0 || [set b]>10000} { + rbu close + error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])" + } + } + + set res [rbu bp_progress] + if {[set res] != [list 10000 10000]} { + rbu close + error "5. reslist is incorrect (expect=10000 10000 got=[set res])" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach {bReopen} { 0 1 } { + reset_db + do_test 2.$bReopen.1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(5, 5, 5, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 2); + } + } {} + do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000} + + reset_db + do_test 2.$bReopen.2.0 { + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + create_rbu1 rbu.db + } {rbu.db} + do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.3.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + } {} + do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000} + + reset_db + do_test 2.$bReopen.4.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.5.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000} + + reset_db + do_test 2.$bReopen.6.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + } + } {} + do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000} +} + +#------------------------------------------------------------------------- +# The following tests verify that the API works when resuming an update +# during the incremental checkpoint stage. +# +proc do_phase2_test {tn bReopen target rbu nStep} { + uplevel [list do_test $tn [subst -nocommands { + + # Build the OAL/WAL file: + sqlite3rbu rbu $target $rbu + while {[lindex [rbu bp_progress] 0]<10000} { + set rc [rbu step] + if {"SQLITE_OK" != [set rc]} { rbu close } + } + + # Clean up the temp tables and move the *-oal file to *-wal. + rbu step + rbu step + + for {set i 0} {[set i] < $nStep} {incr i} { + if {$bReopen} { + rbu close + sqlite3rbu rbu $target $rbu + } + rbu step + set res [rbu bp_progress] + set expect [expr (1 + [set i]) * 10000 / $nStep] + if {[lindex [set res] 1] != [set expect]} { + error "Have [set res], expected 10000 [set expect]" + } + } + + set rc [rbu step] + if {[set rc] != "SQLITE_DONE"} { + error "Have [set rc], expected SQLITE_DONE" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach bReopen {0 1} { + do_test 3.$bReopen.1.0 { + reset_db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 0); + INSERT INTO data_t2 VALUES(1, 2, 0); + INSERT INTO data_t3 VALUES(1, 2, 0); + INSERT INTO data_t4 VALUES(1, 2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + INSERT INTO rbu_count VALUES('data_t2', 1); + INSERT INTO rbu_count VALUES('data_t3', 1); + INSERT INTO rbu_count VALUES('data_t4', 1); + } + } {} + do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5 +} + + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + wr { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } + pk { CREATE TABLE t1(a INT PRIMARY KEY, b, c) } + } { + + foreach {tn2 rbusql r1 r3} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {1666 3333 5000 6666 8333 10000} + + 2 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 1); + } + {3333 6666 10000} + {2000 4000 6000 8000 10000} + + 3 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {1666 2500 3750 5000 6250 7500 8750 10000} + + } { + + reset_db ; execsql $tbl + do_test 4.$tn.$bReopen.$tn2.0 { + execsql { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(ipk) $r1 + set R(wr) $r1 + set R(pk) $r3 + do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + nopk { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1c ON t1(c); + } + vtab { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + } + } { + + if {$tn=="vtab"} { ifcapable !fts5 break } + + foreach {tn2 rbusql r1 r2} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {5000 10000} + + 2 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + + 3 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1); + INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x'); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + } { + + reset_db ; execsql $tbl + do_test 5.$tn.$bReopen.$tn2.0 { + execsql { + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(nopk) $r1 + set R(vtab) $r2 + do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + + +finish_test + diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test new file mode 100644 index 00000000..86f4aa77 --- /dev/null +++ b/ext/rbu/rbuvacuum.test @@ -0,0 +1,392 @@ +# 2016 April 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuvacuum + +foreach step {0 1} { + + set ::testprefix rbuvacuum-step=$step + reset_db + + # Simplest possible vacuum. + do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + PRAGMA integrity_check; + } {ok} + do_rbu_vacuum_test 1.1 $step + + # A vacuum that actually reclaims space. + do_execsql_test 1.2.1 { + INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {12} + do_execsql_test 1.2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; + PRAGMA page_count; + } {12} + do_rbu_vacuum_test 1.2.3 $step + do_execsql_test 1.2.4 { + PRAGMA page_count; + } {3} + + # Add an index to the table. + do_execsql_test 1.3.1 { + CREATE INDEX t1b ON t1(b); + INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {18} + do_execsql_test 1.3.2 { + DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; + PRAGMA page_count; + } {18} + do_rbu_vacuum_test 1.3.3 $step + do_execsql_test 1.3.4 { + PRAGMA page_count; + } {5} + + # WITHOUT ROWID table. + do_execsql_test 1.4.1 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 2 AND 7; + PRAGMA page_count; + } {20} + do_rbu_vacuum_test 1.4.2 $step + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {10} + + # WITHOUT ROWID table with an index. + do_execsql_test 1.4.1 { + CREATE INDEX t2c ON t2(c); + + INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 8 AND 12; + PRAGMA page_count; + } {35} + do_rbu_vacuum_test 1.4.2 $step + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {15} + do_execsql_test 1.4.4 { + VACUUM; + PRAGMA page_count; + } {15} + + do_execsql_test 1.5.1 { + CREATE TABLE t3(a, b, c); + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + INSERT INTO t3 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.5.2 $step + do_execsql_test 1.5.3 { + SELECT * FROM t3 + } {a b c d e f g h i} + do_execsql_test 1.5.4 { + CREATE INDEX t3a ON t3(a); + CREATE INDEX t3b ON t3(b); + CREATE INDEX t3c ON t3(c); + INSERT INTO t3 VALUES('j', 'k', 'l'); + DELETE FROM t3 WHERE a = 'g'; + } + do_rbu_vacuum_test 1.5.5 $step + do_execsql_test 1.5.6 { + SELECT rowid, * FROM t3 ORDER BY b + } {1 a b c 2 d e f 4 j k l} + + do_execsql_test 1.6.1 { + CREATE TABLE t4(a PRIMARY KEY, b, c); + INSERT INTO t4 VALUES('a', 'b', 'c'); + INSERT INTO t4 VALUES('d', 'e', 'f'); + INSERT INTO t4 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.6.2 $step + do_execsql_test 1.6.3 { + SELECT * FROM t4 + } {a b c d e f g h i} + do_execsql_test 1.6.4 { + CREATE INDEX t4a ON t4(a); + CREATE INDEX t4b ON t4(b); + CREATE INDEX t4c ON t4(c); + + INSERT INTO t4 VALUES('j', 'k', 'l'); + DELETE FROM t4 WHERE a='g'; + } + do_rbu_vacuum_test 1.6.5 $step + do_execsql_test 1.6.6 { + SELECT * FROM t4 ORDER BY b + } {a b c d e f j k l} + + reset_db + do_execsql_test 1.7.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + DELETE FROM t1 WHERE a=2; + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + DELETE FROM t1 WHERE a=6; + SELECT * FROM t1; + } {1 one 3 three 5 five} + do_rbu_vacuum_test 1.7.1 $step + do_execsql_test 1.7.2 { + INSERT INTO t1 VALUES(NULL, 'seven'); + SELECT * FROM t1; + } {1 one 3 three 5 five 7 seven} + + reset_db + do_execsql_test 1.8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + ANALYZE; + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + do_rbu_vacuum_test 1.8.1 $step + do_execsql_test 1.7.2 { + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + + reset_db + do_execsql_test 1.9.0 { + PRAGMA page_size = 8192; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 412; + PRAGMA application_id = 413; + + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + do_rbu_vacuum_test 1.9.1 $step + do_execsql_test 1.9.2 { + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + # Vacuum a database with a large sqlite_master table. + # + reset_db + do_test 1.10.1 { + for {set i 1} {$i < 50} {incr i} { + execsql "PRAGMA page_size = 1024" + execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));" + execsql " + INSERT INTO t$i VALUES(1, 2, 3); + INSERT INTO t$i VALUES(4, 5, 6); + " + } + } {} + do_rbu_vacuum_test 1.10.2 $step + + # Database with empty tables. + # + reset_db + do_execsql_test 1.11.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t4 VALUES(1, 2); + } + do_rbu_vacuum_test 1.11.2 $step + do_execsql_test 1.11.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} + reset_db + do_execsql_test 1.12.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + } + do_rbu_vacuum_test 1.12.2 $step + do_execsql_test 1.12.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} +} +set ::testprefix rbuvacuum + +#------------------------------------------------------------------------- +# Test some error cases: +# +# 2.1.* the db being vacuumed being in wal mode already. +# 2.2.* database modified mid vacuum. +# +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(9, 10); +} wal +do_test 2.1.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step +} {SQLITE_ERROR} +do_test 2.1.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}} + +reset_db +do_execsql_test 2.2.0 { + CREATE TABLE tx(a PRIMARY KEY, b BLOB); + INSERT INTO tx VALUES(1, randomblob(900)); + INSERT INTO tx SELECT a+1, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+2, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+4, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+8, randomblob(900) FROM tx; +} +db_save_and_close +for {set i 1} 1 {incr i} { + db_restore_and_reopen + + sqlite3rbu_vacuum rbu test.db state.db + for {set step 0} {$step<$i} {incr step} { rbu step } + rbu close + if {[file exists test.db-wal]} break + + execsql { INSERT INTO tx VALUES(20, 20) } + + do_test 2.2.$i.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step + } {SQLITE_BUSY} + do_test 2.2.$i.2 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database modified during rbu vacuum}} +} + +#------------------------------------------------------------------------- +# Test that a database that uses custom collation sequences can be RBU +# vacuumed. +# +reset_db +forcedelete state.db +proc noop {args} {} +proc length_cmp {x y} { + set n1 [string length $x] + set n2 [string length $y] + return [expr $n1 - $n2] +} +sqlite3_create_collation_v2 db length length_cmp noop + +do_execsql_test 3.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'i'); + INSERT INTO t1 VALUES(2, 'iiii'); + INSERT INTO t1 VALUES(3, 'ii'); + INSERT INTO t1 VALUES(4, 'iii'); + SELECT a FROM t1 ORDER BY b COLLATE length; +} {1 3 4 2} +do_execsql_test 3.1 { + CREATE INDEX i1 ON t1(b COLLATE length); +} + +do_test 3.2 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.3 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.4 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 1] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.5 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + set db2 [rbu db 1] + + sqlite3_create_collation_v2 $db1 length length_cmp noop + sqlite3_create_collation_v2 $db2 length length_cmp noop + + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {0 SQLITE_DONE} + +catch { db close } +finish_test + diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test new file mode 100644 index 00000000..bd386603 --- /dev/null +++ b/ext/rbu/rbuvacuum2.test @@ -0,0 +1,204 @@ +# 2016 June 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] + +foreach step {0 1} { + set ::testprefix rbuvacuum2-$step + + #------------------------------------------------------------------------- + # Test that a database that contains fts3 tables can be vacuumed. + # + ifcapable fts3 { + reset_db + do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts3(z, y); + INSERT INTO t1 VALUES('fix this issue', 'at some point'); + } + + do_rbu_vacuum_test 1.2 $step + + do_execsql_test 1.3 { + SELECT * FROM t1; + } {{fix this issue} {at some point}} + + do_execsql_test 1.4 { + SELECT rowid FROM t1 WHERE t1 MATCH 'fix'; + } {1} + + do_execsql_test 1.5 { + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('l h i', 'd e f'); + DELETE FROM t1 WHERE docid = 2; + INSERT INTO t1 VALUES('a b c', 'x y z'); + } + + do_rbu_vacuum_test 1.6 $step + do_execsql_test 1.7 { + INSERT INTO t1(t1) VALUES('integrity-check'); + SELECT * FROM t1; + } { + {fix this issue} {at some point} + {l h i} {d e f} + {a b c} {x y z} + } + } + + #------------------------------------------------------------------------- + # Test that a database that contains fts5 tables can be vacuumed. + # + ifcapable fts5 { + reset_db + do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t1 USING fts5(z, y); + INSERT INTO t1 VALUES('fix this issue', 'at some point'); + } + + do_rbu_vacuum_test 2.2 $step + + do_execsql_test 2.3 { + SELECT * FROM t1; + } {{fix this issue} {at some point}} + + do_execsql_test 2.4 { + SELECT rowid FROM t1 ('fix'); + } {1} + + do_execsql_test 2.5 { + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('l h i', 'd e f'); + DELETE FROM t1 WHERE rowid = 2; + INSERT INTO t1 VALUES('a b c', 'x y z'); + } + + do_rbu_vacuum_test 2.6 $step + do_execsql_test 2.7 { + INSERT INTO t1(t1) VALUES('integrity-check'); + SELECT * FROM t1; + } { + {fix this issue} {at some point} + {l h i} {d e f} + {a b c} {x y z} + } + } + + #------------------------------------------------------------------------- + # Test that a database that contains an rtree table can be vacuumed. + # + ifcapable rtree { + reset_db + do_execsql_test 3.1 { + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO rt VALUES(1, 45, 55); + INSERT INTO rt VALUES(2, 50, 60); + INSERT INTO rt VALUES(3, 55, 65); + } + + do_rbu_vacuum_test 3.2 $step + + do_execsql_test 3.3 { + SELECT * FROM rt; + } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0} + + do_execsql_test 3.4.1 { + SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 + } {1 2} + do_execsql_test 3.4.2 { + SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 + } {2 3} + + do_rbu_vacuum_test 3.5 $step + + do_execsql_test 3.6.1 { + SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 + } {1 2} + do_execsql_test 3.6.2 { + SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 + } {2 3} + } + + ifcapable trigger { + reset_db + do_execsql_test 4.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END; + } + + do_execsql_test 4.2 { + SELECT * FROM sqlite_master; + } { + table t1 t1 2 {CREATE TABLE t1(a, b, c)} + view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} + trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} + } + + do_rbu_vacuum_test 4.3 $step + do_execsql_test 4.4 { + SELECT * FROM sqlite_master; + } { + table t1 t1 2 {CREATE TABLE t1(a, b, c)} + view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} + trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} + } + } +} + +#------------------------------------------------------------------------- +# Test that passing a NULL value as the second argument to +# sqlite3rbu_vacuum() causes it to: +# +# * Use -vacuum as the state db, and +# * Set the state db permissions to the same as those on the db file. +# +db close +if {$::tcl_platform(platform)=="unix"} { + forcedelete test.db + + sqlite3 db test.db + do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + } + db close + + foreach {tn perm} { + 1 00755 + 2 00666 + 3 00644 + 4 00444 + } { + forcedelete test.db-vacuum + + do_test 5.$tn.1 { + file attributes test.db -permissions $perm + sqlite3rbu_vacuum rbu test.db + rbu step + } {SQLITE_OK} + + do_test 5.$tn.2 { file exists test.db-vacuum } 1 + do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm + rbu close + } +} + + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 145b446d..746469a8 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -147,14 +147,15 @@ ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. */ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 @@ -175,6 +176,7 @@ typedef struct RbuUpdateStmt RbuUpdateStmt; #if !defined(SQLITE_AMALGAMATION) typedef unsigned int u32; +typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; #endif @@ -188,6 +190,8 @@ typedef sqlite3_int64 i64; #define WAL_LOCK_CKPT 1 #define WAL_LOCK_READ0 3 +#define SQLITE_FCNTL_RBUCNT 5149216 + /* ** A structure to store values read from the rbu_state table in memory. */ @@ -200,6 +204,7 @@ struct RbuState { i64 nProgress; u32 iCookie; i64 iOalSz; + i64 nPhaseOneStep; }; struct RbuUpdateStmt { @@ -244,6 +249,7 @@ struct RbuObjIter { int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ @@ -280,10 +286,11 @@ struct RbuObjIter { */ #define RBU_INSERT 1 /* Insert on a main table b-tree */ #define RBU_DELETE 2 /* Delete a row from a main table b-tree */ -#define RBU_IDX_DELETE 3 /* Delete a row from an aux. index b-tree */ -#define RBU_IDX_INSERT 4 /* Insert on an aux. index b-tree */ -#define RBU_UPDATE 5 /* Update a row in a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ +#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ /* ** A single step of an incremental checkpoint - frame iWalFrame of the wal @@ -296,6 +303,43 @@ struct RbuFrame { /* ** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ struct sqlite3rbu { int eStage; /* Value of RBU_STATE_STAGE field */ @@ -313,6 +357,7 @@ struct sqlite3rbu { const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ i64 iOalSz; + i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -325,6 +370,10 @@ struct sqlite3rbu { int pgsz; u8 *aBuf; i64 iWalCksum; + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ }; /* @@ -350,6 +399,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -360,6 +410,11 @@ struct rbu_file { rbu_file *pMainNext; /* Next MAIN_DB file */ }; +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + /************************************************************************* ** The following three functions, found below: @@ -808,8 +863,11 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ /* ** The implementation of the rbu_target_name() SQL function. This function -** accepts one argument - the name of a table in the RBU database. If the -** table name matches the pattern: +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: ** ** data[0-9]_ ** @@ -820,21 +878,33 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ ** "data_t1" -> "t1" ** "data0123_t2" -> "t2" ** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). */ static void rbuTargetNameFunc( - sqlite3_context *context, + sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ + sqlite3rbu *p = sqlite3_user_data(pCtx); const char *zIn; - assert( argc==1 ); + assert( argc==1 || argc==2 ); zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + if( zIn ){ + if( rbuIsVacuum(p) ){ + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } } } } @@ -851,11 +921,14 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ int rc; memset(pIter, 0, sizeof(RbuObjIter)); - rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " "ORDER BY name" - ); + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); if( rc==SQLITE_OK ){ rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, @@ -1143,6 +1216,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ ); } + pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); sqlite3_stmt *pXInfo = 0; @@ -1156,6 +1230,12 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ } rbuFinalize(p, pXInfo); bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; } rbuFinalize(p, pList); @@ -1221,6 +1301,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ pStmt = 0; if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; @@ -1269,6 +1350,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ rbuFinalize(p, pStmt); rbuObjIterCacheIndexedCols(p, pIter); assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); } return p->rc; @@ -1359,6 +1441,8 @@ static char *rbuObjIterGetIndexCols( for(i=0; pIter->abTblPk[i]==0; i++); assert( inTblCol ); zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; }else{ zCol = "rbu_rowid"; } @@ -1822,6 +1906,14 @@ static void rbuTmpInsertFunc( int rc = SQLITE_OK; int i; + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); } @@ -1891,7 +1983,7 @@ static int rbuObjIterPrepareAll( } /* And to delete index entries */ - if( p->rc==SQLITE_OK ){ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError( p->dbMain, &pIter->pDelete, &p->zErrmsg, sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) @@ -1901,6 +1993,15 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; + if( rbuIsVacuum(p) ){ + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zCollist, zLimit + ); + }else + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", @@ -1909,13 +2010,13 @@ static int rbuObjIterPrepareAll( ); }else{ zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "UNION ALL " "SELECT %s, rbu_control FROM '%q' " "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " - "UNION ALL " - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " "ORDER BY %s%s", - zCollist, pIter->zDataTbl, zCollist, p->zStateDb, pIter->zDataTbl, + zCollist, pIter->zDataTbl, zCollist, zLimit ); } @@ -1927,7 +2028,9 @@ static int rbuObjIterPrepareAll( sqlite3_free(zWhere); sqlite3_free(zBind); }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); const char *zTbl = pIter->zTbl; /* Table this step applies to */ const char *zWrite; /* Imposter table name */ @@ -1954,8 +2057,10 @@ static int rbuObjIterPrepareAll( ); } - /* Create the DELETE statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, sqlite3_mprintf( "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere @@ -1963,7 +2068,7 @@ static int rbuObjIterPrepareAll( ); } - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ const char *zRbuRowid = ""; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; @@ -1981,17 +2086,17 @@ static int rbuObjIterPrepareAll( rbuMPrintfExec(p, p->dbMain, "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " "BEGIN " - " SELECT rbu_tmp_insert(2, %s);" + " SELECT rbu_tmp_insert(3, %s);" "END;" "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " "BEGIN " - " SELECT rbu_tmp_insert(2, %s);" + " SELECT rbu_tmp_insert(3, %s);" "END;" "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " "BEGIN " - " SELECT rbu_tmp_insert(3, %s);" + " SELECT rbu_tmp_insert(4, %s);" "END;", zWrite, zTbl, zOldlist, zWrite, zTbl, zOldlist, @@ -2013,10 +2118,16 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM '%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + "SELECT %s,%s rbu_control%s FROM '%q'%s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, pIter->zDataTbl, zLimit ) ); @@ -2111,11 +2222,15 @@ static int rbuGetUpdateStmt( return p->rc; } -static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ sqlite3 *db = 0; if( p->rc==SQLITE_OK ){ const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); sqlite3_close(db); @@ -2125,17 +2240,113 @@ static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ return db; } +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. */ static void rbuOpenDatabase(sqlite3rbu *p){ - assert( p->rc==SQLITE_OK ); - assert( p->dbMain==0 && p->dbRbu==0 ); + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); - p->eStage = 0; - p->dbMain = rbuOpenDbhandle(p, p->zTarget); - p->dbRbu = rbuOpenDbhandle(p, p->zRbu); + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + } + } /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ @@ -2146,6 +2357,96 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_master" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 @@ -2160,7 +2461,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } @@ -2209,9 +2510,9 @@ static void rbuFileSuffix3(const char *zBase, char *z){ #endif { int i, sz; - sz = sqlite3Strlen30(z); + sz = (int)strlen(z)&0xffffff; for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); } #endif } @@ -2419,9 +2720,15 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); + const char *zMove = zBase; + char *zOal; + char *zWal; - char *zWal = sqlite3_mprintf("%s-wal", zBase); - char *zOal = sqlite3_mprintf("%s-oal", zBase); + if( rbuIsVacuum(p) ){ + zMove = sqlite3_db_filename(p->dbRbu, "main"); + } + zOal = sqlite3_mprintf("%s-oal", zMove); + zWal = sqlite3_mprintf("%s-wal", zMove); assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2442,8 +2749,8 @@ static void rbuMoveOalFile(sqlite3rbu *p){ /* Re-open the databases. */ rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); p->dbMain = 0; p->dbRbu = 0; @@ -2509,14 +2816,12 @@ static int rbuStepType(sqlite3rbu *p, const char **pzMask){ switch( sqlite3_column_type(p->objiter.pSelect, iCol) ){ case SQLITE_INTEGER: { int iVal = sqlite3_column_int(p->objiter.pSelect, iCol); - if( iVal==0 ){ - res = RBU_INSERT; - }else if( iVal==1 ){ - res = RBU_DELETE; - }else if( iVal==2 ){ - res = RBU_IDX_DELETE; - }else if( iVal==3 ){ - res = RBU_IDX_INSERT; + switch( iVal ){ + case 0: res = RBU_INSERT; break; + case 1: res = RBU_DELETE; break; + case 2: res = RBU_REPLACE; break; + case 3: res = RBU_IDX_DELETE; break; + case 4: res = RBU_IDX_INSERT; break; } break; } @@ -2555,6 +2860,83 @@ static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){ # define assertColumnName(x,y,z) #endif +/* +** Argument eType must be one of RBU_INSERT, RBU_DELETE, RBU_IDX_INSERT or +** RBU_IDX_DELETE. This function performs the work of a single +** sqlite3rbu_step() call for the type of operation specified by eType. +*/ +static void rbuStepOneOp(sqlite3rbu *p, int eType){ + RbuObjIter *pIter = &p->objiter; + sqlite3_value *pVal; + sqlite3_stmt *pWriter; + int i; + + assert( p->rc==SQLITE_OK ); + assert( eType!=RBU_DELETE || pIter->zIdx==0 ); + assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE + || eType==RBU_INSERT || eType==RBU_IDX_INSERT + ); + + /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE + ** statement below does actually delete a row, nPhaseOneStep will be + ** incremented by the same amount when SQL function rbu_tmp_insert() + ** is invoked by the trigger. */ + if( eType==RBU_DELETE ){ + p->nPhaseOneStep -= p->objiter.nIndex; + } + + if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ + pWriter = pIter->pDelete; + }else{ + pWriter = pIter->pInsert; + } + + for(i=0; inCol; i++){ + /* If this is an INSERT into a table b-tree and the table has an + ** explicit INTEGER PRIMARY KEY, check that this is not an attempt + ** to write a NULL into the IPK column. That is not permitted. */ + if( eType==RBU_INSERT + && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i] + && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL + ){ + p->rc = SQLITE_MISMATCH; + p->zErrmsg = sqlite3_mprintf("datatype mismatch"); + return; + } + + if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){ + continue; + } + + pVal = sqlite3_column_value(pIter->pSelect, i); + p->rc = sqlite3_bind_value(pWriter, i+1, pVal); + if( p->rc ) return; + } + if( pIter->zIdx==0 ){ + if( pIter->eType==RBU_PK_VTAB + || pIter->eType==RBU_PK_NONE + || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: + ** + ** SELECT , rbu_control, rbu_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + assertColumnName(pIter->pSelect, pIter->nCol+1, + rbuIsVacuum(p) ? "rowid" : "rbu_rowid" + ); + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } + } + if( p->rc==SQLITE_OK ){ + sqlite3_step(pWriter); + p->rc = resetAndCollectError(pWriter, &p->zErrmsg); + } +} + /* ** This function does the work for an sqlite3rbu_step() call. ** @@ -2569,78 +2951,36 @@ static void assertColumnName(sqlite3_stmt *pStmt, int iCol, const char *zName){ static int rbuStep(sqlite3rbu *p){ RbuObjIter *pIter = &p->objiter; const char *zMask = 0; - int i; int eType = rbuStepType(p, &zMask); if( eType ){ + assert( eType==RBU_INSERT || eType==RBU_DELETE + || eType==RBU_REPLACE || eType==RBU_IDX_DELETE + || eType==RBU_IDX_INSERT || eType==RBU_UPDATE + ); assert( eType!=RBU_UPDATE || pIter->zIdx==0 ); - if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){ + if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){ rbuBadControlError(p); } - else if( - eType==RBU_INSERT - || eType==RBU_DELETE - || eType==RBU_IDX_DELETE - || eType==RBU_IDX_INSERT - ){ - sqlite3_value *pVal; - sqlite3_stmt *pWriter; - - assert( eType!=RBU_UPDATE ); - assert( eType!=RBU_DELETE || pIter->zIdx==0 ); - - if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ - pWriter = pIter->pDelete; - }else{ - pWriter = pIter->pInsert; + else if( eType==RBU_REPLACE ){ + if( pIter->zIdx==0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + rbuStepOneOp(p, RBU_DELETE); } - - for(i=0; inCol; i++){ - /* If this is an INSERT into a table b-tree and the table has an - ** explicit INTEGER PRIMARY KEY, check that this is not an attempt - ** to write a NULL into the IPK column. That is not permitted. */ - if( eType==RBU_INSERT - && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i] - && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL - ){ - p->rc = SQLITE_MISMATCH; - p->zErrmsg = sqlite3_mprintf("datatype mismatch"); - goto step_out; - } - - if( eType==RBU_DELETE && pIter->abTblPk[i]==0 ){ - continue; - } - - pVal = sqlite3_column_value(pIter->pSelect, i); - p->rc = sqlite3_bind_value(pWriter, i+1, pVal); - if( p->rc ) goto step_out; - } - if( pIter->zIdx==0 - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - /* For a virtual table, or a table with no primary key, the - ** SELECT statement is: - ** - ** SELECT , rbu_control, rbu_rowid FROM .... - ** - ** Hence column_value(pIter->nCol+1). - */ - assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); - pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); - p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); - } - if( p->rc==SQLITE_OK ){ - sqlite3_step(pWriter); - p->rc = resetAndCollectError(pWriter, &p->zErrmsg); - } - }else{ + if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT); + } + else if( eType!=RBU_UPDATE ){ + rbuStepOneOp(p, eType); + } + else{ sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; assert( eType==RBU_UPDATE ); + p->nPhaseOneStep -= p->objiter.nIndex; rbuGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ + int i; for(i=0; p->rc==SQLITE_OK && inCol; i++){ char c = zMask[pIter->aiSrcOrder[i]]; pVal = sqlite3_column_value(pIter->pSelect, i); @@ -2663,20 +3003,23 @@ static int rbuStep(sqlite3rbu *p){ } } } - - step_out: return p->rc; } /* ** Increment the schema cookie of the main database opened by p->dbMain. +** +** Or, if this is an RBU vacuum, set the schema cookie of the main db +** opened by p->dbMain to one more than the schema cookie of the main +** db opened by p->dbRbu. */ static void rbuIncrSchemaCookie(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ + sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); int iCookie = 1000000; sqlite3_stmt *pStmt; - p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ @@ -2704,6 +3047,7 @@ static void rbuIncrSchemaCookie(sqlite3rbu *p){ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); int rc; assert( p->zErrmsg==0 ); @@ -2717,6 +3061,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %d), " "(%d, %lld), " "(%d, %lld), " + "(%d, %lld), " "(%d, %lld) ", p->zStateDb, RBU_STATE_STAGE, eStage, @@ -2725,8 +3070,9 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, - RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, - RBU_STATE_OALSZ, p->iOalSz + RBU_STATE_COOKIE, (i64)pFd->iCookie, + RBU_STATE_OALSZ, p->iOalSz, + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -2740,6 +3086,92 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ } +/* +** The second argument passed to this function is the name of a PRAGMA +** setting - "page_size", "auto_vacuum", "user_version" or "application_id". +** This function executes the following on sqlite3rbu.dbRbu: +** +** "PRAGMA main.$zPragma" +** +** where $zPragma is the string passed as the second argument, then +** on sqlite3rbu.dbMain: +** +** "PRAGMA main.$zPragma = $val" +** +** where $val is the value returned by the first PRAGMA invocation. +** +** In short, it copies the value of the specified PRAGMA setting from +** dbRbu to dbMain. +*/ +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pPragma = 0; + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.%s", zPragma) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ + p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", + zPragma, sqlite3_column_int(pPragma, 0) + ); + } + rbuFinalize(p, pPragma); + } +} + +/* +** The RBU handle passed as the only argument has just been opened and +** the state database is empty. If this RBU handle was opened for an +** RBU vacuum operation, create the schema in the target db. +*/ +static void rbuCreateTargetSchema(sqlite3rbu *p){ + sqlite3_stmt *pSql = 0; + sqlite3_stmt *pInsert = 0; + + assert( rbuIsVacuum(p) ); + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " AND name!='sqlite_sequence' " + " ORDER BY type DESC" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); + } + rbuFinalize(p, pSql); + if( p->rc!=SQLITE_OK ) return; + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + int i; + for(i=0; i<5; i++){ + sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); + } + sqlite3_step(pInsert); + p->rc = sqlite3_reset(pInsert); + } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); + } + + rbuFinalize(p, pSql); + rbuFinalize(p, pInsert); +} + /* ** Step the RBU object. */ @@ -2748,13 +3180,22 @@ int sqlite3rbu_step(sqlite3rbu *p){ switch( p->eStage ){ case RBU_STAGE_OAL: { RbuObjIter *pIter = &p->objiter; + + /* If this is an RBU vacuum operation and the state table was empty + ** when this handle was opened, create the target database schema. */ + if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){ + rbuCreateTargetSchema(p); + rbuCopyPragma(p, "user_version"); + rbuCopyPragma(p, "application_id"); + } + while( p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); @@ -2841,90 +3282,6 @@ int sqlite3rbu_step(sqlite3rbu *p){ } } -/* -** Free an RbuState object allocated by rbuLoadState(). -*/ -static void rbuFreeState(RbuState *p){ - if( p ){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); - } -} - -/* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). -** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. -*/ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; - int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; - - default: - rc = SQLITE_CORRUPT; - break; - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - - p->rc = rc; - return pRet; -} - /* ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero ** otherwise. Either or both argument may be NULL. Two NULL values are @@ -3021,18 +3378,109 @@ static void rbuDeleteVfs(sqlite3rbu *p){ } /* -** Open and return a new RBU handle. +** This user-defined SQL function is invoked with a single argument - the +** name of a table expected to appear in the target database. It returns +** the number of auxilliary indexes on the table. */ -sqlite3rbu *sqlite3rbu_open( +static void rbuIndexCntFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx); + sqlite3_stmt *pStmt = 0; + char *zErrmsg = 0; + int rc; + + assert( nVal==1 ); + + rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + sqlite3_mprintf("SELECT count(*) FROM sqlite_master " + "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) + ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(pCtx, zErrmsg, -1); + }else{ + int nIndex = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + nIndex = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + sqlite3_result_int(pCtx, nIndex); + }else{ + sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + } + } + + sqlite3_free(zErrmsg); +} + +/* +** If the RBU database contains the rbu_count table, use it to initialize +** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table +** is assumed to contain the same columns as: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There should be one row in the table for each data_xxx table in the +** database. The 'tbl' column should contain the name of a data_xxx table, +** and the cnt column the number of rows it contains. +** +** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt +** for all rows in the rbu_count table, where nIndex is the number of +** indexes on the corresponding target database table. +*/ +static void rbuInitPhaseOneSteps(sqlite3rbu *p){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int bExists = 0; /* True if rbu_count exists */ + + p->nPhaseOneStep = -1; + + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 + ); + + /* Check for the rbu_count table. If it does not exist, or if an error + ** occurs, nPhaseOneStep will be left set to -1. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + ); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + bExists = 1; + } + p->rc = sqlite3_finalize(pStmt); + } + + if( p->rc==SQLITE_OK && bExists ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" + "FROM rbu_count" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + } + } +} + + +static sqlite3rbu *openRbuHandle( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; - size_t nTarget = strlen(zTarget); + size_t nTarget = zTarget ? strlen(zTarget) : 0; size_t nRbu = strlen(zRbu); - size_t nState = zState ? strlen(zState) : 0; - size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1; p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ @@ -3042,22 +3490,23 @@ sqlite3rbu *sqlite3rbu_open( memset(p, 0, sizeof(sqlite3rbu)); rbuCreateVfs(p); - /* Open the target database */ + /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zRbu = &p->zTarget[nTarget+1]; + char *pCsr = (char*)&p[1]; + if( zTarget ){ + p->zTarget = pCsr; + memcpy(p->zTarget, zTarget, nTarget+1); + pCsr += nTarget+1; + } + p->zRbu = pCsr; memcpy(p->zRbu, zRbu, nRbu+1); + pCsr += nRbu+1; if( zState ){ - p->zState = &p->zRbu[nRbu+1]; - memcpy(p->zState, zState, nState+1); + p->zState = rbuMPrintf(p, "%s", zState); } rbuOpenDatabase(p); } - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - if( p->rc==SQLITE_OK ){ pState = rbuLoadState(p); assert( pState || p->rc!=SQLITE_OK ); @@ -3065,9 +3514,11 @@ sqlite3rbu *sqlite3rbu_open( if( pState->eStage==0 ){ rbuDeleteOalFile(p); + rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; }else{ p->eStage = pState->eStage; + p->nPhaseOneStep = pState->nPhaseOneStep; } p->nProgress = pState->nProgress; p->iOalSz = pState->iOalSz; @@ -3085,38 +3536,27 @@ sqlite3rbu *sqlite3rbu_open( } } - if( p->rc==SQLITE_OK + if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) - && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie - ){ - /* At this point (pTargetFd->iCookie) contains the value of the - ** change-counter cookie (the thing that gets incremented when a - ** transaction is committed in rollback mode) currently stored on - ** page 1 of the database file. */ - p->rc = SQLITE_BUSY; - p->zErrmsg = sqlite3_mprintf("database modified during rbu update"); + && pState->eStage!=0 + ){ + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); + if( pFd->iCookie!=pState->iCookie ){ + /* At this point (pTargetFd->iCookie) contains the value of the + ** change-counter cookie (the thing that gets incremented when a + ** transaction is committed in rollback mode) currently stored on + ** page 1 of the database file. */ + p->rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", + (rbuIsVacuum(p) ? "vacuum" : "update") + ); + } } if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; - - /* Open transactions both databases. The *-oal file is opened or - ** created at this point. */ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); - } - - /* Check if the main database is a zipvfs db. If it is, set the upper - ** level pager to use "journal_mode=off". This prevents it from - ** generating a large journal using a temp file. */ - if( p->rc==SQLITE_OK ){ - int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); - if( frc==SQLITE_OK ){ - p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); - } - } + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ @@ -3127,12 +3567,34 @@ sqlite3rbu *sqlite3rbu_open( ** update finished. */ if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){ p->rc = SQLITE_DONE; - } + p->eStage = RBU_STAGE_DONE; + }else{ + if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } - if( p->rc==SQLITE_OK ){ - rbuSetupOal(p, pState); - } + /* Open transactions both databases. The *-oal file is opened or + ** created at this point. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } + /* Check if the main database is a zipvfs db. If it is, set the upper + ** level pager to use "journal_mode=off". This prevents it from + ** generating a large journal using a temp file. */ + if( p->rc==SQLITE_OK ){ + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); + if( frc==SQLITE_OK ){ + p->rc = sqlite3_exec( + db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); + } + } + + if( p->rc==SQLITE_OK ){ + rbuSetupOal(p, pState); + } + } }else if( p->eStage==RBU_STAGE_MOVE ){ /* no-op */ }else if( p->eStage==RBU_STAGE_CKPT ){ @@ -3150,6 +3612,44 @@ sqlite3rbu *sqlite3rbu_open( return p; } +/* +** Allocate and return an RBU handle with all fields zeroed except for the +** error code, which is set to SQLITE_MISUSE. +*/ +static sqlite3rbu *rbuMisuseError(void){ + sqlite3rbu *pRet; + pRet = sqlite3_malloc64(sizeof(sqlite3rbu)); + if( pRet ){ + memset(pRet, 0, sizeof(sqlite3rbu)); + pRet->rc = SQLITE_MISUSE; + } + return pRet; +} + +/* +** Open and return a new RBU handle. +*/ +sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +){ + if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); } + /* TODO: Check that zTarget and zRbu are non-NULL */ + return openRbuHandle(zTarget, zRbu, zState); +} + +/* +** Open a handle to begin or resume an RBU VACUUM operation. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +){ + if( zTarget==0 ){ return rbuMisuseError(); } + /* TODO: Check that both arguments are non-NULL */ + return openRbuHandle(0, zTarget, zState); +} /* ** Return the database handle used by pRbu. @@ -3170,7 +3670,7 @@ sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ */ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ - int i; + unsigned int i; size_t nErrmsg = strlen(p->zErrmsg); for(i=0; i<(nErrmsg-8); i++){ if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ @@ -3204,9 +3704,19 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close any open statement handles. */ rbuObjIterFinalize(&p->objiter); + /* If this is an RBU vacuum handle and the vacuum has either finished + ** successfully or encountered an error, delete the contents of the + ** state table. This causes the next call to sqlite3rbu_vacuum() + ** specifying the current target and state databases to start a new + ** vacuum from scratch. */ + if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){ + int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0); + if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2; + } + /* Close the open database handle and VFS object. */ - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); @@ -3214,6 +3724,7 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ rbuEditErrmsg(p); rc = p->rc; *pzErrmsg = p->zErrmsg; + sqlite3_free(p->zState); sqlite3_free(p); }else{ rc = SQLITE_NOMEM; @@ -3231,9 +3742,75 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } +/* +** Return permyriadage progress indications for the two main stages of +** an RBU update. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ + const int MAX_PROGRESS = 10000; + switch( p->eStage ){ + case RBU_STAGE_OAL: + if( p->nPhaseOneStep>0 ){ + *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep); + }else{ + *pnOne = -1; + } + *pnTwo = 0; + break; + + case RBU_STAGE_MOVE: + *pnOne = MAX_PROGRESS; + *pnTwo = 0; + break; + + case RBU_STAGE_CKPT: + *pnOne = MAX_PROGRESS; + *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame); + break; + + case RBU_STAGE_DONE: + *pnOne = MAX_PROGRESS; + *pnTwo = MAX_PROGRESS; + break; + + default: + assert( 0 ); + } +} + +/* +** Return the current state of the RBU vacuum or update operation. +*/ +int sqlite3rbu_state(sqlite3rbu *p){ + int aRes[] = { + 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE, + 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE + }; + + assert( RBU_STAGE_OAL==1 ); + assert( RBU_STAGE_MOVE==2 ); + assert( RBU_STAGE_CKPT==4 ); + assert( RBU_STAGE_DONE==5 ); + assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL ); + assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE ); + assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT ); + assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE ); + + if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){ + return SQLITE_RBU_STATE_ERROR; + }else{ + assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE ); + assert( p->eStage==RBU_STAGE_OAL + || p->eStage==RBU_STAGE_MOVE + || p->eStage==RBU_STAGE_CKPT + || p->eStage==RBU_STAGE_DONE + ); + return aRes[p->eStage]; + } +} + int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; - if( rc==SQLITE_DONE ) return SQLITE_OK; assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); @@ -3372,6 +3949,22 @@ static u32 rbuGetU32(u8 *aBuf){ + ((u32)aBuf[3]); } +/* +** Write an unsigned 32-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU32(u8 *aBuf, u32 iVal){ + aBuf[0] = (iVal >> 24) & 0xFF; + aBuf[1] = (iVal >> 16) & 0xFF; + aBuf[2] = (iVal >> 8) & 0xFF; + aBuf[3] = (iVal >> 0) & 0xFF; +} + +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3397,6 +3990,35 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); +#if 1 + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( pRbu && rbuIsVacuum(pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + && pRbu->rc==SQLITE_OK + ){ + sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + u8 *aBuf = (u8*)zBuf; + u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0; + rbuPutU32(&aBuf[52], iRoot); /* largest root page number */ + rbuPutU32(&aBuf[36], 0); /* number of free pages */ + rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ + + if( iAmt>100 ){ + memset(&aBuf[100], 0, iAmt-100); + rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + aBuf[100] = 0x0D; + } + } + } +#endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3471,7 +4093,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* @@ -3483,7 +4118,9 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){ + if( eLock==SQLITE_LOCK_EXCLUSIVE + && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) + ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; @@ -3546,6 +4183,12 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + else if( op==SQLITE_FCNTL_RBUCNT ){ + sqlite3rbu *pRbu = (sqlite3rbu*)pArg; + pRbu->nRbu++; + pRbu->pRbuFd = p; + p->bNolock = 1; + } rc = xControl(p->pReal, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ @@ -3704,11 +4347,38 @@ static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){ static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ rbu_file *pDb; sqlite3_mutex_enter(pRbuVfs->mutex); - for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext); + for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){} sqlite3_mutex_leave(pRbuVfs->mutex); return pDb; } +/* +** A main database named zName has just been opened. The following +** function returns a pointer to a buffer owned by SQLite that contains +** the name of the *-wal file this db connection will use. SQLite +** happens to pass a pointer to this buffer when using xAccess() +** or xOpen() to operate on the *-wal file. +*/ +static const char *rbuMainToWal(const char *zName, int flags){ + int n = (int)strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + return z; +} + /* ** Open an rbu file handle. */ @@ -3744,6 +4414,7 @@ static int rbuVfsOpen( rbu_file *pFd = (rbu_file *)pFile; int rc = SQLITE_OK; const char *zOpen = zName; + int oflags = flags; memset(pFd, 0, sizeof(rbu_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; @@ -3756,23 +4427,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = (int)strlen(zName); - const char *z = &zName[n]; - if( flags & SQLITE_OPEN_URI ){ - int odd = 0; - while( 1 ){ - if( z[0]==0 ){ - odd = 1 - odd; - if( odd && z[1]==0 ) break; - } - z++; - } - z += 2; - }else{ - while( *z==0 ) z++; - } - z += (n + 8 + 1); - pFd->zWal = z; + pFd->zWal = rbuMainToWal(zName, flags); } else if( flags & SQLITE_OPEN_WAL ){ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); @@ -3782,10 +4437,17 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - size_t nCopy = strlen(zName); - char *zCopy = sqlite3_malloc64(nCopy+2); + const char *zBase = zName; + size_t nCopy; + char *zCopy; + if( rbuIsVacuum(pDb->pRbu) ){ + zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); + zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI); + } + nCopy = strlen(zBase); + zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ - memcpy(zCopy, zName, nCopy); + memcpy(zCopy, zBase, nCopy); zCopy[nCopy-3] = 'o'; zCopy[nCopy] = '\0'; zCopy[nCopy+1] = '\0'; @@ -3800,8 +4462,17 @@ static int rbuVfsOpen( } } + if( oflags & SQLITE_OPEN_MAIN_DB + && sqlite3_uri_boolean(zName, "rbu_memory", 0) + ){ + assert( oflags & SQLITE_OPEN_MAIN_DB ); + oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + zOpen = 0; + } + if( rc==SQLITE_OK ){ - rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f1a0f3cd..12112c14 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -104,7 +104,7 @@ ** may also be named data_, where is any sequence ** of zero or more numeric characters (0-9). This can be significant because ** tables within the RBU database are always processed in order sorted by -** name. By judicious selection of the the portion of the names +** name. By judicious selection of the portion of the names ** of the RBU tables the user can therefore control the order in which they ** are processed. This can be useful, for example, to ensure that "external ** content" FTS4 tables are updated before their underlying content tables. @@ -314,6 +314,44 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ); +/* +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function identifies a database in which +** to store the state of the RBU vacuum operation if it is suspended. The +** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum +** operation, the state database should either not exist or be empty +** (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** If the second argument passed to this function is NULL, then the +** name of the state database is "-vacuum", where +** is the name of the target database file. In this case, on UNIX, if the +** state database is not already present in the file-system, it is created +** with the same permissions as the target db is made. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This @@ -400,6 +438,86 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: +** +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. +** +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). +** +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. +** +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. +** +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. +*/ +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 + +int sqlite3rbu_state(sqlite3rbu *pRbu); + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 3fa85b75..b1a22527 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -17,11 +17,19 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) #include "sqlite3rbu.h" -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif #include -/* From main.c (apparently...) */ +/* From main.c */ extern const char *sqlite3ErrName(int); +extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ Tcl_Interp *interp = (Tcl_Interp*)sqlite3_user_data(pCtx); @@ -48,7 +56,7 @@ void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ } -static int test_sqlite3rbu_cmd( +static int SQLITE_TCLAPI test_sqlite3rbu_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -66,6 +74,10 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"bp_progress", 2, ""}, /* 5 */ + {"db", 3, "RBU"}, /* 6 */ + {"state", 2, ""}, /* 7 */ + {"progress", 2, ""}, /* 8 */ {0,0,0} }; int iCmd; @@ -136,6 +148,46 @@ static int test_sqlite3rbu_cmd( break; } + case 5: /* bp_progress */ { + int one, two; + Tcl_Obj *pObj; + sqlite3rbu_bp_progress(pRbu, &one, &two); + + pObj = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one)); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two)); + Tcl_SetObjResult(interp, pObj); + break; + } + + case 6: /* db */ { + int bArg; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bArg) ){ + ret = TCL_ERROR; + }else{ + char zBuf[50]; + sqlite3 *db = sqlite3rbu_db(pRbu, bArg); + if( sqlite3TestMakePointerStr(interp, zBuf, (void*)db) ){ + ret = TCL_ERROR; + }else{ + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + } + } + break; + } + case 7: /* state */ { + const char *aRes[] = { 0, "oal", "move", "checkpoint", "done", "error" }; + int eState = sqlite3rbu_state(pRbu); + assert( eState>0 && eState<=5 ); + Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC); + break; + } + case 8: /* progress */ { + sqlite3_int64 nStep = sqlite3rbu_progress(pRbu); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep)); + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; @@ -147,7 +199,7 @@ static int test_sqlite3rbu_cmd( /* ** Tclcmd: sqlite3rbu CMD ?? */ -static int test_sqlite3rbu( +static int SQLITE_TCLAPI test_sqlite3rbu( ClientData clientData, Tcl_Interp *interp, int objc, @@ -174,10 +226,38 @@ static int test_sqlite3rbu( return TCL_OK; } +/* +** Tclcmd: sqlite3rbu_vacuum CMD +*/ +static int SQLITE_TCLAPI test_sqlite3rbu_vacuum( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3rbu *pRbu = 0; + const char *zCmd; + const char *zTarget; + const char *zStateDb = 0; + + if( objc!=3 && objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zTarget = Tcl_GetString(objv[2]); + if( objc==4 ) zStateDb = Tcl_GetString(objv[3]); + + pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + /* ** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT */ -static int test_sqlite3rbu_create_vfs( +static int SQLITE_TCLAPI test_sqlite3rbu_create_vfs( ClientData clientData, Tcl_Interp *interp, int objc, @@ -212,7 +292,7 @@ static int test_sqlite3rbu_create_vfs( /* ** Tclcmd: sqlite3rbu_destroy_vfs NAME */ -static int test_sqlite3rbu_destroy_vfs( +static int SQLITE_TCLAPI test_sqlite3rbu_destroy_vfs( ClientData clientData, Tcl_Interp *interp, int objc, @@ -233,7 +313,7 @@ static int test_sqlite3rbu_destroy_vfs( /* ** Tclcmd: sqlite3rbu_internal_test */ -static int test_sqlite3rbu_internal_test( +static int SQLITE_TCLAPI test_sqlite3rbu_internal_test( ClientData clientData, Tcl_Interp *interp, int objc, @@ -261,6 +341,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){ Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "sqlite3rbu", test_sqlite3rbu }, + { "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum }, { "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs }, { "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs }, { "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test }, @@ -273,7 +354,11 @@ int SqliteRbu_Init(Tcl_Interp *interp){ } #else -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif int SqliteRbu_Init(Tcl_Interp *interp){ return TCL_OK; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */ #endif /* defined(SQLITE_TEST) */ diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 4e473a22..c7de6ac1 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1542,7 +1542,7 @@ static int rtreeFilter( if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ - RtreeSearchPoint *p; /* Search point for the the leaf */ + RtreeSearchPoint *p; /* Search point for the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); i64 iNode = 0; rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); @@ -1741,7 +1741,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ return SQLITE_NOMEM; } - nRow = pRtree->nRowEst / (iIdx + 1); + nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; setEstimatedRows(pIdxInfo, nRow); @@ -2800,6 +2800,53 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){ } #endif /* !defined(SQLITE_RTREE_INT_ONLY) */ +/* +** A constraint has failed while inserting a row into an rtree table. +** Assuming no OOM error occurs, this function sets the error message +** (at pRtree->base.zErrMsg) to an appropriate value and returns +** SQLITE_CONSTRAINT. +** +** Parameter iCol is the index of the leftmost column involved in the +** constraint failure. If it is 0, then the constraint that failed is +** the unique constraint on the id column. Otherwise, it is the rtree +** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** +** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. +*/ +static int rtreeConstraintError(Rtree *pRtree, int iCol){ + sqlite3_stmt *pStmt = 0; + char *zSql; + int rc; + + assert( iCol==0 || iCol%2 ); + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + if( iCol==0 ){ + const char *zCol = sqlite3_column_name(pStmt, 0); + pRtree->base.zErrMsg = sqlite3_mprintf( + "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol + ); + }else{ + const char *zCol1 = sqlite3_column_name(pStmt, iCol); + const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); + pRtree->base.zErrMsg = sqlite3_mprintf( + "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 + ); + } + } + + sqlite3_finalize(pStmt); + return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); +} + + /* ** The xUpdate method for rtree module virtual tables. @@ -2850,7 +2897,7 @@ static int rtreeUpdate( cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]); cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]); if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } @@ -2861,7 +2908,7 @@ static int rtreeUpdate( cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]); cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]); if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } @@ -2882,7 +2929,7 @@ static int rtreeUpdate( if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ rc = rtreeDeleteRowid(pRtree, cell.iRowid); }else{ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, 0); goto constraint; } } @@ -2965,6 +3012,13 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ int rc; i64 nRow = 0; + rc = sqlite3_table_column_metadata( + db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + ); + if( rc!=SQLITE_OK ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + return rc==SQLITE_ERROR ? SQLITE_OK : rc; + } zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index c9192de1..9dc101a7 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -194,13 +194,13 @@ do_test rtree-2.1.3 { do_test rtree-2.2.1 { catchsql { INSERT INTO t1 VALUES(2, 1, 3, 2, 4) } -} {1 {constraint failed}} +} {1 {UNIQUE constraint failed: t1.ii}} do_test rtree-2.2.2 { catchsql { INSERT INTO t1 VALUES(4, 1, 3, 4, 2) } -} {1 {constraint failed}} +} {1 {rtree constraint failed: t1.(y1<=y2)}} do_test rtree-2.2.3 { catchsql { INSERT INTO t1 VALUES(4, 3, 1, 2, 4) } -} {1 {constraint failed}} +} {1 {rtree constraint failed: t1.(x1<=x2)}} do_test rtree-2.2.4 { execsql { SELECT ii FROM t1 ORDER BY ii } } {1 2 3} @@ -236,7 +236,7 @@ do_test rtree-3.1.3 { # Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)): do_test rtree-3.2.1 { catchsql { INSERT INTO t1 VALUES(7, 2, 6, 4, 3) } -} {1 {constraint failed}} +} {1 {rtree constraint failed: t1.(y1<=y2)}} do_test rtree-3.2.2 { catchsql { INSERT INTO t1 VALUES(8, 2, 6, 3, 3) } } {0 {}} @@ -490,11 +490,11 @@ foreach {tn sql_template testdata} { } 4 "INSERT %CONF% INTO t1 VALUES(2, 7, 6, 7, 7)" { - ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} - ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + ROLLBACK 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} - FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} - REPLACE 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 0 2 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} } } { @@ -510,7 +510,9 @@ foreach {tn sql_template testdata} { } set res(0) {0 {}} - set res(1) {1 {constraint failed}} + set res(1) {1 {UNIQUE constraint failed: t1.idx}} + set res(2) {1 {rtree constraint failed: t1.(x1<=x2)}} + do_catchsql_test $testname.1 $sql $res($error) do_test $testname.2 [list sql_uses_stmt db $sql] $uses do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data diff --git a/ext/rtree/rtree3.test b/ext/rtree/rtree3.test index fea55130..1d863c6c 100644 --- a/ext/rtree/rtree3.test +++ b/ext/rtree/rtree3.test @@ -47,7 +47,8 @@ ifcapable !rtree { # # rtree3-8: Test OOM while registering the r-tree module with sqlite. # - +# rtree3-11: OOM following a constraint failure +# do_faultsim_test rtree3-1 -faults oom* -prep { faultsim_delete_and_reopen } -body { @@ -234,4 +235,32 @@ do_faultsim_test rtree3-10 -faults oom-* -prep { faultsim_test_result {0 2} } + +do_test rtree3-11.prep { + faultsim_delete_and_reopen + execsql { + CREATE VIRTUAL TABLE rt USING rtree(ii, x1, x2, y1, y2); + INSERT INTO rt VALUES(1, 2, 3, 4, 5); + } + faultsim_save_and_close +} {} +do_faultsim_test rtree3-10.1 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM rt } +} -body { + execsql { INSERT INTO rt VALUES(1, 2, 3, 4, 5) } +} -test { + faultsim_test_result {1 {UNIQUE constraint failed: rt.ii}} \ + {1 {constraint failed}} +} +do_faultsim_test rtree3-10.2 -faults oom-* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM rt } +} -body { + execsql { INSERT INTO rt VALUES(2, 2, 3, 5, 4) } +} -test { + faultsim_test_result {1 {rtree constraint failed: rt.(y1<=y2)}} \ + {1 {constraint failed}} +} + finish_test diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index 9a64df51..a26c401e 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -350,7 +350,3 @@ do_eqp_execsql_test 7.4 { } finish_test - - - -finish_test diff --git a/ext/rtree/rtreeG.test b/ext/rtree/rtreeG.test new file mode 100644 index 00000000..bffd17ec --- /dev/null +++ b/ext/rtree/rtreeG.test @@ -0,0 +1,66 @@ +# 2016-05-32 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains tests for the r-tree module. +# +# Verify that no invalid SQL is run during initialization + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } + +db close +sqlite3_shutdown +test_sqlite3_log [list lappend ::log] +set ::log [list] +sqlite3 db test.db + + +set ::log {} +do_execsql_test rtreeG-1.1 { + CREATE VIRTUAL TABLE t1 USING rtree(id,x0,x1,y0,y1); +} {} +do_test rtreeG-1.1log { + set ::log +} {} + +do_execsql_test rtreeG-1.2 { + INSERT INTO t1 VALUES(1,10,15,5,23),(2,20,21,5,23),(3,10,15,20,30); + SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25; +} {1} +do_test rtreeG-1.2log { + set ::log +} {} + +db close +sqlite3 db test.db +do_execsql_test rtreeG-1.3 { + SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25; +} {1} +do_test rtreeG-1.3log { + set ::log +} {} + +do_execsql_test rtreeG-1.4 { + DROP TABLE t1; +} {} +do_test rtreeG-1.4log { + set ::log +} {} + +db close +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +sqlite3 db test.db + +finish_test diff --git a/ext/session/changeset.c b/ext/session/changeset.c new file mode 100644 index 00000000..edb43ff8 --- /dev/null +++ b/ext/session/changeset.c @@ -0,0 +1,416 @@ +/* +** 2014-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement the "changeset" command line +** utility for displaying and transforming changesets generated by +** the Sessions extension. +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include + + +/* +** Show a usage message on stderr then quit. +*/ +static void usage(const char *argv0){ + fprintf(stderr, "Usage: %s FILENAME COMMAND ...\n", argv0); + fprintf(stderr, + "COMMANDs:\n" + " apply DB Apply the changeset to database file DB\n" + " concat FILE2 OUT Concatenate FILENAME and FILE2 into OUT\n" + " dump Show the complete content of the changeset\n" + " invert OUT Write an inverted changeset into file OUT\n" + " sql Give a pseudo-SQL rendering of the changeset\n" + ); + exit(1); +} + +/* +** Read the content of a disk file into an in-memory buffer +*/ +static void readFile(const char *zFilename, int *pSz, void **ppBuf){ + FILE *f; + int sz; + void *pBuf; + f = fopen(zFilename, "rb"); + if( f==0 ){ + fprintf(stderr, "cannot open \"%s\" for reading\n", zFilename); + exit(1); + } + fseek(f, 0, SEEK_END); + sz = (int)ftell(f); + rewind(f); + pBuf = sqlite3_malloc( sz ? sz : 1 ); + if( pBuf==0 ){ + fprintf(stderr, "cannot allocate %d to hold content of \"%s\"\n", + sz, zFilename); + exit(1); + } + if( sz>0 ){ + if( fread(pBuf, sz, 1, f)!=1 ){ + fprintf(stderr, "cannot read all %d bytes of \"%s\"\n", sz, zFilename); + exit(1); + } + fclose(f); + } + *pSz = sz; + *ppBuf = pBuf; +} + +/* Array for converting from half-bytes (nybbles) into ASCII hex +** digits. */ +static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' +}; + +/* +** Render an sqlite3_value as an SQL string. +*/ +static void renderValue(sqlite3_value *pVal){ + switch( sqlite3_value_type(pVal) ){ + case SQLITE_FLOAT: { + double r1; + char zBuf[50]; + r1 = sqlite3_value_double(pVal); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + printf("%s", zBuf); + break; + } + case SQLITE_INTEGER: { + printf("%lld", sqlite3_value_int64(pVal)); + break; + } + case SQLITE_BLOB: { + char const *zBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + int i; + printf("x'"); + for(i=0; i>4)&0x0F]); + putchar(hexdigits[(zBlob[i])&0x0F]); + } + putchar('\''); + break; + } + case SQLITE_TEXT: { + const unsigned char *zArg = sqlite3_value_text(pVal); + putchar('\''); + while( zArg[0] ){ + putchar(zArg[0]); + if( zArg[0]=='\'' ) putchar(zArg[0]); + zArg++; + } + putchar('\''); + break; + } + default: { + assert( sqlite3_value_type(pVal)==SQLITE_NULL ); + printf("NULL"); + break; + } + } +} + +/* +** Number of conflicts seen +*/ +static int nConflict = 0; + +/* +** The conflict callback +*/ +static int conflictCallback( + void *pCtx, + int eConflict, + sqlite3_changeset_iter *pIter +){ + int op, bIndirect, nCol, i; + const char *zTab; + unsigned char *abPK; + const char *zType = ""; + const char *zOp = ""; + const char *zSep = " "; + + nConflict++; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + sqlite3changeset_pk(pIter, &abPK, 0); + switch( eConflict ){ + case SQLITE_CHANGESET_DATA: zType = "DATA"; break; + case SQLITE_CHANGESET_NOTFOUND: zType = "NOTFOUND"; break; + case SQLITE_CHANGESET_CONFLICT: zType = "PRIMARY KEY"; break; + case SQLITE_CHANGESET_FOREIGN_KEY: zType = "FOREIGN KEY"; break; + case SQLITE_CHANGESET_CONSTRAINT: zType = "CONSTRAINT"; break; + } + switch( op ){ + case SQLITE_UPDATE: zOp = "UPDATE of"; break; + case SQLITE_INSERT: zOp = "INSERT into"; break; + case SQLITE_DELETE: zOp = "DELETE from"; break; + } + printf("%s conflict on %s table %s with primary key", zType, zOp, zTab); + for(i=0; i0 && fwrite(pOutBuf, szOut, 1, out)!=1 ){ + fprintf(stderr, "unable to write all %d bytes of output to \"%s\"\n", + szOut, zOut); + } + fclose(out); + sqlite3_free(pOutBuf); + sqlite3_free(pB); + }else + + /* changeset FILENAME dump + ** Show the complete content of the changeset in FILENAME + */ + if( strcmp(argv[2],"dump")==0 ){ + int cnt = 0; + int i; + sqlite3_changeset_iter *pIter; + rc = sqlite3changeset_start(&pIter, sz, pBuf); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3changeset_start() returns %d\n", rc); + exit(1); + } + while( sqlite3changeset_next(pIter)==SQLITE_ROW ){ + int op, bIndirect, nCol; + const char *zTab; + unsigned char *abPK; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + cnt++; + printf("%d: %s table=[%s] indirect=%d nColumn=%d\n", + cnt, op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : "DELETE", + zTab, bIndirect, nCol); + sqlite3changeset_pk(pIter, &abPK, 0); + for(i=0; i0 && fwrite(pOutBuf, szOut, 1, out)!=1 ){ + fprintf(stderr, "unable to write all %d bytes of output to \"%s\"\n", + szOut, zOut); + } + fclose(out); + sqlite3_free(pOutBuf); + }else + + /* changeset FILE sql + ** Show the content of the changeset as pseudo-SQL + */ + if( strcmp(argv[2],"sql")==0 ){ + int cnt = 0; + char *zPrevTab = 0; + char *zSQLTabName = 0; + sqlite3_changeset_iter *pIter = 0; + rc = sqlite3changeset_start(&pIter, sz, pBuf); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3changeset_start() returns %d\n", rc); + exit(1); + } + printf("BEGIN;\n"); + while( sqlite3changeset_next(pIter)==SQLITE_ROW ){ + int op, bIndirect, nCol; + const char *zTab; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + cnt++; + if( zPrevTab==0 || strcmp(zPrevTab,zTab)!=0 ){ + sqlite3_free(zPrevTab); + sqlite3_free(zSQLTabName); + zPrevTab = sqlite3_mprintf("%s", zTab); + if( !isalnum(zTab[0]) || sqlite3_strglob("*[^a-zA-Z0-9]*",zTab)==0 ){ + zSQLTabName = sqlite3_mprintf("\"%w\"", zTab); + }else{ + zSQLTabName = sqlite3_mprintf("%s", zTab); + } + printf("/****** Changes for table %s ***************/\n", zSQLTabName); + } + switch( op ){ + case SQLITE_DELETE: { + unsigned char *abPK; + int i; + const char *zSep = " "; + sqlite3changeset_pk(pIter, &abPK, 0); + printf("/* %d */ DELETE FROM %s WHERE", cnt, zSQLTabName); + for(i=0; i2; +} { + {INSERT t2 0 X. {} {i 3 t y}} + {INSERT t2 0 X. {} {i 4 t y}} +} + +do_iterator_test 6.1.7 * { + SELECT indirect(1); + DELETE FROM t2 WHERE x = 4; + SELECT indirect(0); + INSERT INTO t2 VALUES(4, 'new'); +} { + {UPDATE t2 0 X. {i 4 t y} {{} {} t new}} +} + +do_iterator_test 6.1.8 * { + CREATE TABLE t3(a, b PRIMARY KEY); + CREATE TABLE t4(a, b PRIMARY KEY); + CREATE TRIGGER t4t AFTER UPDATE ON t4 BEGIN + UPDATE t3 SET a = new.a WHERE b = new.b; + END; + + SELECT indirect(1); + INSERT INTO t3 VALUES('one', 1); + INSERT INTO t4 VALUES('one', 1); + SELECT indirect(0); + UPDATE t4 SET a = 'two' WHERE b = 1; +} { + {INSERT t3 1 .X {} {t two i 1}} + {INSERT t4 0 .X {} {t two i 1}} +} + +sqlite3session S db main +do_execsql_test 6.2.1 { + SELECT indirect(0); + SELECT indirect(-1); + SELECT indirect(45); + SELECT indirect(-100); +} {0 0 1 1} +S delete + +#------------------------------------------------------------------------- +# Test that if a conflict-handler that has been passed either NOTFOUND or +# CONSTRAINT returns REPLACE - the sqlite3changeset_apply() call returns +# MISUSE and rolls back any changes made so far. +# +# 7.1.*: NOTFOUND conflict-callback. +# 7.2.*: CONSTRAINT conflict-callback. +# +proc xConflict {args} {return REPLACE} +test_reset + +do_execsql_test 7.1.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); +} +do_test 7.1.2 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b NOT NULL); + INSERT INTO t1 VALUES(1, 'one'); + } db2 +} {} +do_test 7.1.3 { + set changeset [changeset_from_sql { + UPDATE t1 SET b = 'five' WHERE a = 1; + UPDATE t1 SET b = 'six' WHERE a = 2; + }] + set x [list] + sqlite3session_foreach c $changeset { lappend x $c } + set x +} [list \ + {UPDATE t1 0 X. {i 1 t one} {{} {} t five}} \ + {UPDATE t1 0 X. {i 2 t two} {{} {} t six}} \ +] +do_test 7.1.4 { + list [catch {sqlite3changeset_apply db2 $changeset xConflict} msg] $msg +} {1 SQLITE_MISUSE} +do_test 7.1.5 { execsql { SELECT * FROM t1 } db2 } {1 one} + +do_test 7.2.1 { + set changeset [changeset_from_sql { UPDATE t1 SET b = NULL WHERE a = 1 }] + + set x [list] + sqlite3session_foreach c $changeset { lappend x $c } + set x +} [list \ + {UPDATE t1 0 X. {i 1 t five} {{} {} n {}}} \ +] +do_test 7.2.2 { + list [catch {sqlite3changeset_apply db2 $changeset xConflict} msg] $msg +} {1 SQLITE_MISUSE} +do_test 7.2.3 { execsql { SELECT * FROM t1 } db2 } {1 one} + +#------------------------------------------------------------------------- +# Test that if a conflict-handler returns ABORT, application of the +# changeset is rolled back and the sqlite3changeset_apply() method returns +# SQLITE_ABORT. +# +# Also test that the same thing happens if a conflict handler returns an +# unrecognized integer value. Except, in this case SQLITE_MISUSE is returned +# instead of SQLITE_ABORT. +# +foreach {tn conflict_return apply_return} { + 1 ABORT SQLITE_ABORT + 2 567 SQLITE_MISUSE +} { + test_reset + proc xConflict {args} [list return $conflict_return] + + do_test 8.$tn.0 { + do_common_sql { + CREATE TABLE t1(x, y, PRIMARY KEY(x, y)); + INSERT INTO t1 VALUES('x', 'y'); + } + execsql { INSERT INTO t1 VALUES('w', 'w') } + + set changeset [changeset_from_sql { DELETE FROM t1 WHERE 1 }] + + set x [list] + sqlite3session_foreach c $changeset { lappend x $c } + set x + } [list \ + {DELETE t1 0 XX {t w t w} {}} \ + {DELETE t1 0 XX {t x t y} {}} \ + ] + + do_test 8.$tn.1 { + list [catch {sqlite3changeset_apply db2 $changeset xConflict} msg] $msg + } [list 1 $apply_return] + + do_test 8.$tn.2 { + execsql {SELECT * FROM t1} db2 + } {x y} +} + + +#------------------------------------------------------------------------- +# Try to cause an infinite loop as follows: +# +# 1. Have a changeset insert a row that causes a CONFLICT callback, +# 2. Have the conflict handler return REPLACE, +# 3. After the session module deletes the conflicting row, have a trigger +# re-insert it. +# 4. Goto step 1... +# +# This doesn't work, as the second invocation of the conflict handler is a +# CONSTRAINT, not a CONFLICT. There is at most one CONFLICT callback for +# each change in the changeset. +# +test_reset +proc xConflict {type args} { + if {$type == "CONFLICT"} { return REPLACE } + return OMIT +} +do_test 9.1 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + } + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('x', 2); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + INSERT INTO t1 VALUES(old.a, old.b); + END; + } db2 +} {} +do_test 9.2 { + set changeset [changeset_from_sql { INSERT INTO t1 VALUES('x', 1) }] + sqlite3changeset_apply db2 $changeset xConflict +} {} +do_test 9.3 { + execsql { SELECT * FROM t1 } db2 +} {x 2} + +#------------------------------------------------------------------------- +# +test_reset +db function enable [list S enable] + +do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('x', 'X'); +} + +do_iterator_test 10.1 t1 { + INSERT INTO t1 VALUES('y', 'Y'); + SELECT enable(0); + INSERT INTO t1 VALUES('z', 'Z'); + SELECT enable(1); +} { + {INSERT t1 0 X. {} {t y t Y}} +} + +sqlite3session S db main +do_execsql_test 10.2 { + SELECT enable(0); + SELECT enable(-1); + SELECT enable(1); + SELECT enable(-1); +} {0 0 1 1} +S delete + +finish_test diff --git a/ext/session/session3.test b/ext/session/session3.test new file mode 100644 index 00000000..e15407c2 --- /dev/null +++ b/ext/session/session3.test @@ -0,0 +1,211 @@ +# 2011 March 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the session module. More +# specifically, it focuses on testing the session modules response to +# database schema modifications and mismatches. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session3 + +#------------------------------------------------------------------------- +# These tests - session3-1.* - verify that the session module behaves +# correctly when confronted with a schema mismatch when applying a +# changeset (in function sqlite3changeset_apply()). +# +# session3-1.1.*: Table does not exist in target db. +# session3-1.2.*: Table has wrong number of columns in target db. +# session3-1.3.*: Table has wrong PK columns in target db. +# +db close +sqlite3_shutdown +test_sqlite3_log log +sqlite3 db test.db + +proc log {code msg} { lappend ::log $code $msg } + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); +} +do_test 1.1 { + set ::log {} + do_then_apply_sql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + set ::log +} {SQLITE_SCHEMA {sqlite3changeset_apply(): no such table: t1}} + +do_test 1.2.0 { + execsql { CREATE TABLE t1(a PRIMARY KEY, b, c) } db2 +} {} +do_test 1.2.1 { + set ::log {} + do_then_apply_sql { + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + } + set ::log +} {SQLITE_SCHEMA {sqlite3changeset_apply(): table t1 has 3 columns, expected 2}} + +do_test 1.3.0 { + execsql { + DROP TABLE t1; + CREATE TABLE t1(a, b PRIMARY KEY); + } db2 +} {} +do_test 1.3.1 { + set ::log {} + do_then_apply_sql { + INSERT INTO t1 VALUES(9, 10); + INSERT INTO t1 VALUES(11, 12); + } + set ::log +} {SQLITE_SCHEMA {sqlite3changeset_apply(): primary key mismatch for table t1}} + +#------------------------------------------------------------------------- +# These tests - session3-2.* - verify that the session module behaves +# correctly when the schema of an attached table is modified during the +# session. +# +# session3-2.1.*: Table is dropped midway through the session. +# session3-2.2.*: Table is dropped and recreated with a different # cols. +# session3-2.3.*: Table is dropped and recreated with a different PK. +# +# In all of these scenarios, the call to sqlite3session_changeset() will +# return SQLITE_SCHEMA. Also: +# +# session3-2.4.*: Table is dropped and recreated with an identical schema. +# In this case sqlite3session_changeset() returns SQLITE_OK. +# + +do_test 2.1 { + execsql { CREATE TABLE t2(a, b PRIMARY KEY) } + sqlite3session S db main + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2); + DROP TABLE t2; + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 2.2.1 { + S delete + sqlite3session S db main + execsql { CREATE TABLE t2(a, b PRIMARY KEY, c) } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} +do_test 2.2.2 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c, d); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} +do_test 2.2.3 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + INSERT INTO t2 VALUES(4, 5); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} +do_test 2.2.4 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c, d); + INSERT INTO t2 VALUES(4, 5, 6, 7); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 2.3 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2); + DROP TABLE t2; + CREATE TABLE t2(a PRIMARY KEY, b); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 2.4 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + list [catch { S changeset } msg] $msg +} {0 {}} + +S delete + + +catch { db close } +catch { db2 close } +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize + +finish_test diff --git a/ext/session/session4.test b/ext/session/session4.test new file mode 100644 index 00000000..120a230b --- /dev/null +++ b/ext/session/session4.test @@ -0,0 +1,67 @@ +# 2011 March 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session4 + +do_test 1.0 { + execsql { + CREATE TABLE x(a, b, c, d, e, PRIMARY KEY(c, e)); + INSERT INTO x VALUES(65.21, X'28B0', 16.35, NULL, 'doers'); + INSERT INTO x VALUES(NULL, 78.49, 2, X'60', -66); + INSERT INTO x VALUES('cathedral', NULL, 35, NULL, X'B220937E80A2D8'); + INSERT INTO x VALUES(NULL, 'masking', -91.37, NULL, X'596D'); + INSERT INTO x VALUES(19, 'domains', 'espouse', -94, 'throw'); + } + + set changeset [changeset_from_sql { + DELETE FROM x WHERE e = -66; + UPDATE x SET a = 'parameterizable', b = 31.8 WHERE c = 35; + INSERT INTO x VALUES(-75.61, -17, 16.85, NULL, X'D73DB02678'); + }] + set {} {} +} {} + + +# This currently causes crashes. sqlite3changeset_invert() does not handle +# corrupt changesets well. +if 0 { + do_test 1.1 { + for {set i 0} {$i < [string length $changeset]} {incr i} { + set before [string range $changeset 0 [expr $i-1]] + set after [string range $changeset [expr $i+1] end] + for {set j 10} {$j < 260} {incr j} { + set x [binary format "a*ca*" $before $j $after] + catch { sqlite3changeset_invert $x } + } + } + } {} +} + +do_test 1.2 { + set x [binary format "ca*" 0 [string range $changeset 1 end]] + list [catch { sqlite3changeset_invert $x } msg] $msg +} {1 SQLITE_CORRUPT} + +do_test 1.3 { + set x [binary format "ca*" 0 [string range $changeset 1 end]] + list [catch { sqlite3changeset_apply db $x xConflict } msg] $msg +} {1 SQLITE_CORRUPT} + +finish_test diff --git a/ext/session/session5.test b/ext/session/session5.test new file mode 100644 index 00000000..9b8f9ffe --- /dev/null +++ b/ext/session/session5.test @@ -0,0 +1,408 @@ +# 2011 April 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the session module. +# Specifically, for the sqlite3changeset_concat() command. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session5 + +# Organization of tests: +# +# session5-1.*: Simple tests to check the concat() function produces +# correct results. +# +# session5-2.*: More complicated tests. +# +# session5-3.*: Schema mismatch errors. +# +# session5-4.*: Test the concat cases that indicate that the database +# was modified in between recording of the two changesets +# being concatenated (i.e. two changesets that INSERT rows +# with the same PK values). +# + +proc do_concat_test {tn args} { + + set subtest 0 + foreach sql $args { + incr subtest + sqlite3session S db main ; S attach * + execsql $sql + + set c [S changeset] + if {[info commands s_prev] != ""} { + set c_concat [sqlite3changeset_concat $c_prev $c] + set c_two [s_prev changeset] + s_prev delete + + set h_concat [changeset_to_list $c_concat] + set h_two [changeset_to_list $c_two] + + do_test $tn.$subtest [list set {} $h_concat] $h_two + } + set c_prev $c + rename S s_prev + } + + catch { s_prev delete } +} + +#------------------------------------------------------------------------- +# Test cases session5-1.* - simple tests. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); +} + +do_concat_test 1.1.1 { + INSERT INTO t1 VALUES(1, 'one'); +} { + INSERT INTO t1 VALUES(2, 'two'); +} + +do_concat_test 1.1.2 { + UPDATE t1 SET b = 'five' WHERE a = 1; +} { + UPDATE t1 SET b = 'six' WHERE a = 2; +} + +do_concat_test 1.1.3 { + DELETE FROM t1 WHERE a = 1; +} { + DELETE FROM t1 WHERE a = 2; +} + + +# 1.2.1: INSERT + DELETE -> (none) +# 1.2.2: INSERT + UPDATE -> INSERT +# +# 1.2.3: DELETE + INSERT (matching data) -> (none) +# 1.2.4: DELETE + INSERT (non-matching data) -> UPDATE +# +# 1.2.5: UPDATE + UPDATE (matching data) -> (none) +# 1.2.6: UPDATE + UPDATE (non-matching data) -> UPDATE +# 1.2.7: UPDATE + DELETE -> DELETE +# +do_concat_test 1.2.1 { + INSERT INTO t1 VALUES('x', 'y'); +} { + DELETE FROM t1 WHERE a = 'x'; +} +do_concat_test 1.2.2 { + INSERT INTO t1 VALUES(5.0, 'five'); +} { + UPDATE t1 SET b = 'six' WHERE a = 5.0; +} + +do_execsql_test 1.2.3.1 "INSERT INTO t1 VALUES('I', 'one')" +do_concat_test 1.2.3.2 { + DELETE FROM t1 WHERE a = 'I'; +} { + INSERT INTO t1 VALUES('I', 'one'); +} +do_concat_test 1.2.4 { + DELETE FROM t1 WHERE a = 'I'; +} { + INSERT INTO t1 VALUES('I', 'two'); +} +do_concat_test 1.2.5 { + UPDATE t1 SET b = 'five' WHERE a = 'I'; +} { + UPDATE t1 SET b = 'two' WHERE a = 'I'; +} +do_concat_test 1.2.6 { + UPDATE t1 SET b = 'six' WHERE a = 'I'; +} { + UPDATE t1 SET b = 'seven' WHERE a = 'I'; +} +do_concat_test 1.2.7 { + UPDATE t1 SET b = 'eight' WHERE a = 'I'; +} { + DELETE FROM t1 WHERE a = 'I'; +} + + +#------------------------------------------------------------------------- +# Test cases session5-2.* - more complex tests. +# +db function indirect indirect +proc indirect {{x -1}} { + S indirect $x + s_prev indirect $x +} +do_concat_test 2.1 { + CREATE TABLE abc(a, b, c PRIMARY KEY); + INSERT INTO abc VALUES(NULL, NULL, 1); + INSERT INTO abc VALUES('abcdefghijkl', NULL, 2); +} { + DELETE FROM abc WHERE c = 1; + UPDATE abc SET c = 1 WHERE c = 2; +} { + INSERT INTO abc VALUES('abcdefghijkl', NULL, 2); + INSERT INTO abc VALUES(1.0, 2.0, 3); +} { + UPDATE abc SET a = a-1; +} { + CREATE TABLE def(d, e, f, PRIMARY KEY(e, f)); + INSERT INTO def VALUES('x', randomblob(11000), 67); + INSERT INTO def SELECT d, e, f+1 FROM def; + INSERT INTO def SELECT d, e, f+2 FROM def; + INSERT INTO def SELECT d, e, f+4 FROM def; +} { + DELETE FROM def WHERE rowid>4; +} { + INSERT INTO def SELECT d, e, f+4 FROM def; +} { + INSERT INTO abc VALUES(22, 44, -1); +} { + UPDATE abc SET c=-2 WHERE c=-1; + UPDATE abc SET c=-3 WHERE c=-2; +} { + UPDATE abc SET c=-4 WHERE c=-3; +} { + UPDATE abc SET a=a+1 WHERE c=-3; + UPDATE abc SET a=a+1 WHERE c=-3; +} { + UPDATE abc SET a=a+1 WHERE c=-3; + UPDATE abc SET a=a+1 WHERE c=-3; +} { + INSERT INTO abc VALUES('one', 'two', 'three'); +} { + SELECT indirect(1); + UPDATE abc SET a='one point five' WHERE c = 'three'; +} { + SELECT indirect(0); + UPDATE abc SET a='one point six' WHERE c = 'three'; +} { + CREATE TABLE x1(a, b, PRIMARY KEY(a)); + SELECT indirect(1); + INSERT INTO x1 VALUES(1, 2); +} { + SELECT indirect(1); + UPDATE x1 SET b = 3 WHERE a = 1; +} + +catch {db close} +forcedelete test.db +sqlite3 db test.db +do_concat_test 2.2 { + CREATE TABLE t1(a, b, PRIMARY KEY(b)); + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('string', 1); + INSERT INTO t1 VALUES(4, 2); + INSERT INTO t1 VALUES(X'FFAAFFAAFFAA', 3); +} { + INSERT INTO t2 VALUES('one', 'two'); + INSERT INTO t2 VALUES(1, NULL); + UPDATE t1 SET a = 5 WHERE a = 2; +} { + DELETE FROM t2 WHERE a = 1; + UPDATE t1 SET a = 4 WHERE a = 2; + INSERT INTO t2 VALUES('x', 'y'); +} + +do_test 2.3.0 { + catch {db close} + forcedelete test.db + sqlite3 db test.db + + set sql1 "" + set sql2 "" + for {set i 1} {$i < 120} {incr i} { + append sql1 "INSERT INTO x1 VALUES($i*4, $i);" + } + for {set i 1} {$i < 120} {incr i} { + append sql2 "DELETE FROM x1 WHERE a = $i*4;" + } + set {} {} +} {} +do_concat_test 2.3 { + CREATE TABLE x1(a PRIMARY KEY, b) +} $sql1 $sql2 $sql1 $sql2 + +do_concat_test 2.4 { + CREATE TABLE x2(a PRIMARY KEY, b); + CREATE TABLE x3(a PRIMARY KEY, b); + + INSERT INTO x2 VALUES('a', 'b'); + INSERT INTO x2 VALUES('x', 'y'); + INSERT INTO x3 VALUES('a', 'b'); +} { + INSERT INTO x2 VALUES('c', 'd'); + INSERT INTO x3 VALUES('e', 'f'); + INSERT INTO x3 VALUES('x', 'y'); +} + +do_concat_test 2.5 { + UPDATE x3 SET b = 'Y' WHERE a = 'x' +} { + DELETE FROM x3 WHERE a = 'x' +} { + DELETE FROM x2 WHERE a = 'a' +} { + INSERT INTO x2 VALUES('a', 'B'); +} + +for {set k 1} {$k <=10} {incr k} { + do_test 2.6.$k.1 { + drop_all_tables + set sql1 "" + set sql2 "" + for {set i 1} {$i < 120} {incr i} { + append sql1 "INSERT INTO x1 VALUES(randomblob(20+(random()%10)), $i);" + } + for {set i 1} {$i < 120} {incr i} { + append sql2 "DELETE FROM x1 WHERE rowid = $i;" + } + set {} {} + } {} + do_concat_test 2.6.$k { + CREATE TABLE x1(a PRIMARY KEY, b) + } $sql1 $sql2 $sql1 $sql2 +} + +for {set k 1} {$k <=10} {incr k} { + do_test 2.7.$k.1 { + drop_all_tables + set sql1 "" + set sql2 "" + for {set i 1} {$i < 120} {incr i} { + append sql1 { + INSERT INTO x1 VALUES( + CASE WHEN random()%2 THEN random() ELSE randomblob(20+random()%10) END, + CASE WHEN random()%2 THEN random() ELSE randomblob(20+random()%10) END + ); + } + } + for {set i 1} {$i < 120} {incr i} { + append sql2 "DELETE FROM x1 WHERE rowid = $i;" + } + set {} {} + } {} + do_concat_test 2.7.$k { + CREATE TABLE x1(a PRIMARY KEY, b) + } $sql1 $sql2 $sql1 $sql2 +} + + +#------------------------------------------------------------------------- +# Test that schema incompatibilities are detected correctly. +# +# session5-3.1: Incompatible number of columns. +# session5-3.2: Incompatible PK definition. +# + +do_test 3.1 { + db close + forcedelete test.db + sqlite3 db test.db + + execsql { CREATE TABLE t1(a PRIMARY KEY, b) } + set c1 [changeset_from_sql { INSERT INTO t1 VALUES(1, 2) }] + execsql { + DROP TABLE t1; + CREATE TABLE t1(a PRIMARY KEY, b, c); + } + set c2 [changeset_from_sql { INSERT INTO t1 VALUES(2, 3, 4) }] + + list [catch { sqlite3changeset_concat $c1 $c2 } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 3.2 { + db close + forcedelete test.db + sqlite3 db test.db + + execsql { CREATE TABLE t1(a PRIMARY KEY, b) } + set c1 [changeset_from_sql { INSERT INTO t1 VALUES(1, 2) }] + execsql { + DROP TABLE t1; + CREATE TABLE t1(a, b PRIMARY KEY); + } + set c2 [changeset_from_sql { INSERT INTO t1 VALUES(2, 3) }] + + list [catch { sqlite3changeset_concat $c1 $c2 } msg] $msg +} {1 SQLITE_SCHEMA} + +#------------------------------------------------------------------------- +# Test that concat() handles these properly: +# +# session5-4.1: INSERT + INSERT +# session5-4.2: UPDATE + INSERT +# session5-4.3: DELETE + UPDATE +# session5-4.4: DELETE + DELETE +# + +proc do_concat_test2 {tn sql1 sqlX sql2 expected} { + sqlite3session S db main ; S attach * + execsql $sql1 + set ::c1 [S changeset] + S delete + + execsql $sqlX + + sqlite3session S db main ; S attach * + execsql $sql2 + set ::c2 [S changeset] + S delete + + uplevel do_test $tn [list { + changeset_to_list [sqlite3changeset_concat $::c1 $::c2] + }] [list [normalize_list $expected]] +} + +drop_all_tables db +do_concat_test2 4.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('key', 'value'); +} { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'xxx'); +} { + {INSERT t1 0 X. {} {t key t value}} +} +do_concat_test2 4.2 { + UPDATE t1 SET b = 'yyy'; +} { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'value'); +} { + {UPDATE t1 0 X. {t key t xxx} {{} {} t yyy}} +} +do_concat_test2 4.3 { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'www'); +} { + UPDATE t1 SET b = 'valueX' WHERE a = 'key'; +} { + {DELETE t1 0 X. {t key t value} {}} +} +do_concat_test2 4.4 { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'ttt'); +} { + DELETE FROM t1 WHERE a = 'key'; +} { + {DELETE t1 0 X. {t key t valueX} {}} +} + +finish_test diff --git a/ext/session/session6.test b/ext/session/session6.test new file mode 100644 index 00000000..8a1f172c --- /dev/null +++ b/ext/session/session6.test @@ -0,0 +1,90 @@ +# 2011 July 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite sessions extension. +# Specifically, it tests that sessions work when the database is modified +# using incremental blob handles. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session6 + +proc do_then_apply_tcl {tcl {dbname main}} { + proc xConflict args { return "OMIT" } + set rc [catch { + sqlite3session S db $dbname + db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { + S attach $name + } + eval $tcl + sqlite3changeset_apply db2 [S changeset] xConflict + } msg] + + catch { S delete } + if {$rc} {error $msg} +} + +test_sqlite3_log x +proc x {args} {puts $args} + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(c PRIMARY KEY, d); +} + +# Test a blob update. +# +do_test 1.1 { + do_then_apply_tcl { + db eval { INSERT INTO t1 VALUES(1, 'helloworld') } + db eval { INSERT INTO t2 VALUES(2, 'onetwothree') } + } + compare_db db db2 +} {} +do_test 1.2 { + do_then_apply_tcl { + set fd [db incrblob t1 b 1] + puts -nonewline $fd 1234567890 + close $fd + } + compare_db db db2 +} {} + +# Test an attached database. +# +do_test 2.1 { + forcedelete test.db3 + file copy test.db2 test.db3 + execsql { ATTACH 'test.db3' AS aux; } + + do_then_apply_tcl { + set fd [db incrblob aux t2 d 1] + puts -nonewline $fd fourfivesix + close $fd + } aux + + sqlite3 db3 test.db3 + compare_db db2 db3 +} {} + + +db3 close +db2 close + +finish_test diff --git a/ext/session/session8.test b/ext/session/session8.test new file mode 100644 index 00000000..9f70fe28 --- /dev/null +++ b/ext/session/session8.test @@ -0,0 +1,91 @@ +# 2011 July 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session8 + +proc noop {args} {} + +# Like [dbcksum] in tester.tcl. Except this version is not sensitive +# to changes in the value of implicit IPK columns. +# +proc udbcksum {db dbname} { + if {$dbname=="temp"} { + set master sqlite_temp_master + } else { + set master $dbname.sqlite_master + } + set alltab [$db eval "SELECT name FROM $master WHERE type='table'"] + set txt [$db eval "SELECT * FROM $master"]\n + foreach tab $alltab { + append txt [lsort [$db eval "SELECT * FROM $dbname.$tab"]]\n + } + return [md5 $txt] +} + +proc do_then_undo {tn sql} { + set ck1 [udbcksum db main] + + sqlite3session S db main + S attach * + db eval $sql + + set ck2 [udbcksum db main] + + set invert [sqlite3changeset_invert [S changeset]] + S delete + sqlite3changeset_apply db $invert noop + + set ck3 [udbcksum db main] + + set a [expr {$ck1==$ck2}] + set b [expr {$ck1==$ck3}] + uplevel [list do_test $tn.1 "set {} $a" 0] + uplevel [list do_test $tn.2 "set {} $b" 1] +} + +do_execsql_test 1.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES("abc", "xyz"); +} +do_then_undo 1.2 { INSERT INTO t1 VALUES(3, 4); } +do_then_undo 1.3 { DELETE FROM t1 WHERE b=2; } +do_then_undo 1.4 { UPDATE t1 SET b = 3 WHERE a = 1; } + +do_execsql_test 2.1 { + CREATE TABLE t2(a, b PRIMARY KEY); + INSERT INTO t2 VALUES(1, 2); + INSERT INTO t2 VALUES('abc', 'xyz'); +} +do_then_undo 1.2 { INSERT INTO t2 VALUES(3, 4); } +do_then_undo 1.3 { DELETE FROM t2 WHERE b=2; } +do_then_undo 1.4 { UPDATE t1 SET a = '123' WHERE b = 'xyz'; } + +do_execsql_test 3.1 { + CREATE TABLE t3(a, b, c, d, e, PRIMARY KEY(c, e)); + INSERT INTO t3 VALUES('x', 45, 0.0, 'abcdef', 12); + INSERT INTO t3 VALUES(45, 0.0, 'abcdef', 12, 'x'); + INSERT INTO t3 VALUES(0.0, 'abcdef', 12, 'x', 45); +} + +do_then_undo 3.2 { UPDATE t3 SET b=b||b WHERE e!='x' } +do_then_undo 3.3 { UPDATE t3 SET a = 46 } + +finish_test diff --git a/ext/session/session9.test b/ext/session/session9.test new file mode 100644 index 00000000..ebb88ffa --- /dev/null +++ b/ext/session/session9.test @@ -0,0 +1,287 @@ +# 2013 July 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that the sessions module handles foreign key constraint +# violations when applying changesets as required. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix session9 + + +#-------------------------------------------------------------------- +# Basic tests. +# +proc populate_db {} { + drop_all_tables + execsql { + PRAGMA foreign_keys = 1; + CREATE TABLE p1(a PRIMARY KEY, b); + CREATE TABLE c1(a PRIMARY KEY, b REFERENCES p1); + CREATE TABLE c2(a PRIMARY KEY, + b REFERENCES p1 DEFERRABLE INITIALLY DEFERRED + ); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + } +} + +proc capture_changeset {sql} { + sqlite3session S db main + + foreach t [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $t + } + execsql $sql + set ret [S changeset] + S delete + + return $ret +} + +do_test 1.1 { + populate_db + set cc [capture_changeset { + INSERT INTO c1 VALUES('ii', 2); + INSERT INTO c2 VALUES('iii', 3); + }] + set {} {} +} {} + +proc xConflict {args} { + lappend ::xConflict {*}$args + return $::conflictret +} + +foreach {tn delrow trans conflictargs conflictret} { + 1 2 0 {FOREIGN_KEY 1} OMIT + 2 3 0 {FOREIGN_KEY 1} OMIT + 3 2 1 {FOREIGN_KEY 1} OMIT + 4 3 1 {FOREIGN_KEY 1} OMIT + 5 2 0 {FOREIGN_KEY 1} ABORT + 6 3 0 {FOREIGN_KEY 1} ABORT + 7 2 1 {FOREIGN_KEY 1} ABORT + 8 3 1 {FOREIGN_KEY 1} ABORT +} { + + set A(OMIT) {0 {}} + set A(ABORT) {1 SQLITE_CONSTRAINT} + do_test 1.2.$tn.1 { + populate_db + execsql { DELETE FROM p1 WHERE a=($delrow+0) } + if {$trans} { execsql BEGIN } + + set ::xConflict [list] + list [catch {sqlite3changeset_apply db $::cc xConflict} msg] $msg + } $A($conflictret) + + do_test 1.2.$tn.2 { set ::xConflict } $conflictargs + + set A(OMIT) {1 1} + set A(ABORT) {0 0} + do_test 1.2.$tn.3 { + execsql { SELECT count(*) FROM c1 UNION ALL SELECT count(*) FROM c2 } + } $A($conflictret) + + do_test 1.2.$tn.4 { expr ![sqlite3_get_autocommit db] } $trans + do_test 1.2.$tn.5 { + if { $trans } { execsql COMMIT } + } {} +} + +#-------------------------------------------------------------------- +# Test that closing a transaction clears the defer_foreign_keys flag. +# +foreach {tn open noclose close} { + 1 BEGIN {} COMMIT + 2 BEGIN {} ROLLBACK + + 3 {SAVEPOINT one} {} {RELEASE one} + 4 {SAVEPOINT one} {ROLLBACK TO one} {RELEASE one} +} { + execsql $open + do_execsql_test 2.$tn.1 { PRAGMA defer_foreign_keys } {0} + + do_execsql_test 2.$tn.2 { + PRAGMA defer_foreign_keys = 1; + PRAGMA defer_foreign_keys; + } {1} + + execsql $noclose + do_execsql_test 2.$tn.3 { PRAGMA defer_foreign_keys } {1} + + execsql $close + do_execsql_test 2.$tn.4 { PRAGMA defer_foreign_keys } {0} +} + +#-------------------------------------------------------------------- +# Test that a cyclic relationship can be inserted and deleted. +# +# This situation does not come up in practice, but testing it serves to +# show that it does not matter which order parent and child keys +# are processed in internally when applying a changeset. +# +drop_all_tables + +do_execsql_test 3.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(x PRIMARY KEY, y); +} + +# Create changesets as follows: +# +# $cc1 - Insert a row into t1. +# $cc2 - Insert a row into t2. +# $cc - Combination of $cc1 and $cc2. +# +# $ccdel1 - Delete the row from t1. +# $ccdel2 - Delete the row from t2. +# $ccdel - Combination of $cc1 and $cc2. +# +do_test 3.2 { + set cc1 [capture_changeset { + INSERT INTO t1 VALUES('one', 'value one'); + }] + set ccdel1 [capture_changeset { DELETE FROM t1; }] + set cc2 [capture_changeset { + INSERT INTO t2 VALUES('value one', 'one'); + }] + set ccdel2 [capture_changeset { DELETE FROM t2; }] + set cc [capture_changeset { + INSERT INTO t1 VALUES('one', 'value one'); + INSERT INTO t2 VALUES('value one', 'one'); + }] + set ccdel [capture_changeset { + DELETE FROM t1; + DELETE FROM t2; + }] + set {} {} +} {} + +# Now modify the database schema to create a cyclic foreign key dependency +# between tables t1 and t2. This means that although changesets $cc and +# $ccdel can be applied, none of the others may without violating the +# foreign key constraints. +# +do_test 3.3 { + + drop_all_tables + execsql { + CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t2); + CREATE TABLE t2(x PRIMARY KEY, y REFERENCES t1); + } + + + proc conflict_handler {args} { return "ABORT" } + sqlite3changeset_apply db $cc conflict_handler + + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {one {value one} {value one} one} + +do_test 3.3.1 { + list [catch {sqlite3changeset_apply db $::ccdel1 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} + +do_test 3.3.2 { + list [catch {sqlite3changeset_apply db $::ccdel2 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} + +do_test 3.3.4.1 { + list [catch {sqlite3changeset_apply db $::ccdel conflict_handler} msg] $msg +} {0 {}} +do_execsql_test 3.3.4.2 { + SELECT * FROM t1; + SELECT * FROM t2; +} {} + +do_test 3.5.1 { + list [catch {sqlite3changeset_apply db $::cc1 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} +do_test 3.5.2 { + list [catch {sqlite3changeset_apply db $::cc2 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} + +#-------------------------------------------------------------------- +# Test that if a change that affects FK processing is not applied +# due to a separate constraint, SQLite does not get confused and +# increment FK counters anyway. +# +drop_all_tables +do_execsql_test 4.1 { + CREATE TABLE p1(x PRIMARY KEY, y); + CREATE TABLE c1(a PRIMARY KEY, b REFERENCES p1); + INSERT INTO p1 VALUES(1,1); +} + +do_execsql_test 4.2.1 { + BEGIN; + PRAGMA defer_foreign_keys = 1; + INSERT INTO c1 VALUES('x', 'x'); +} +do_catchsql_test 4.2.2 { COMMIT } {1 {FOREIGN KEY constraint failed}} +do_catchsql_test 4.2.3 { ROLLBACK } {0 {}} + +do_execsql_test 4.3.1 { + BEGIN; + PRAGMA defer_foreign_keys = 1; + INSERT INTO c1 VALUES(1, 1); +} +do_catchsql_test 4.3.2 { + INSERT INTO c1 VALUES(1, 'x') +} {1 {UNIQUE constraint failed: c1.a}} + +do_catchsql_test 4.3.3 { COMMIT } {0 {}} +do_catchsql_test 4.3.4 { BEGIN ; COMMIT } {0 {}} + +#-------------------------------------------------------------------- +# Test that if a DELETE change cannot be applied due to an +# SQLITE_CONSTRAINT error thrown by a trigger program, things do not +# go awry. + +drop_all_tables +reset_db +do_execsql_test 5.1 { + CREATE TABLE x1(x PRIMARY KEY, y); + CREATE TABLE x2(x PRIMARY KEY, y); + INSERT INTO x2 VALUES(1, 1); + INSERT INTO x1 VALUES(1, 1); +} + +set ::cc [changeset_from_sql { DELETE FROM x1; }] + +do_execsql_test 5.2 { + INSERT INTO x1 VALUES(1, 1); + CREATE TRIGGER tr1 AFTER DELETE ON x1 BEGIN + INSERT INTO x2 VALUES(old.x, old.y); + END; +} {} + +proc conflict_handler {args} { return "ABORT" } +do_test 5.3 { + list [catch {sqlite3changeset_apply db $::cc conflict_handler} msg] $msg +} {1 SQLITE_ABORT} + +do_execsql_test 5.4 { + SELECT * FROM X1; +} {1 1} + +finish_test diff --git a/ext/session/sessionA.test b/ext/session/sessionA.test new file mode 100644 index 00000000..0e0a14e3 --- /dev/null +++ b/ext/session/sessionA.test @@ -0,0 +1,106 @@ +# 2013 July 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that filter callbacks work as required. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionA + + +forcedelete test.db2 +sqlite3 db2 test.db2 +foreach {tn db} {1 db 2 db2} { + do_test 1.$tn.1 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(a PRIMARY KEY, b); + CREATE TABLE t3(a PRIMARY KEY, b); + } $db + } {} +} + +proc tbl_filter {zTbl} { + return $::table_filter($zTbl) +} + +do_test 2.1 { + set ::table_filter(t1) 1 + set ::table_filter(t2) 0 + set ::table_filter(t3) 1 + + sqlite3session S db main + S table_filter tbl_filter + + execsql { + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t3 VALUES('e', 'f'); + } + + set changeset [S changeset] + S delete + sqlite3changeset_apply db2 $changeset xConflict + + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } db2 +} {a b e f} + +#------------------------------------------------------------------------- +# Test that filter callbacks passed to sqlite3changeset_apply() are +# invoked correctly. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(x PRIMARY KEY, y); +} + +do_test 3.2 { + execsql BEGIN + set ::cs [changeset_from_sql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES('x', 'y'); + }] + execsql ROLLBACK + set {} {} +} {} + +proc filter {x y} { + return [string equal $x $y] +} + +do_test 3.3 { + sqlite3changeset_apply db $::cs {} [list filter t1] + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2} + +do_test 3.4 { + execsql { DELETE FROM t1 } + sqlite3changeset_apply db $::cs {} [list filter t2] + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {x y} + +finish_test diff --git a/ext/session/sessionB.test b/ext/session/sessionB.test new file mode 100644 index 00000000..eae41f76 --- /dev/null +++ b/ext/session/sessionB.test @@ -0,0 +1,508 @@ +# 2014 August 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for sessions SQLite extension. +# Specifically, this file contains tests for "patchset" changes. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionB + +# +# 1.*: Test that the blobs returned by the session_patchset() API are +# as expected. Also the sqlite3_changeset_iter functions. +# +# 2.*: Test that patchset blobs are handled by sqlite3changeset_apply(). +# +# 3.*: Test that sqlite3changeset_invert() works with patchset blobs. +# Correct behaviour is to return SQLITE_CORRUPT. + +proc do_sql2patchset_test {tn sql res} { + sqlite3session S db main + S attach * + execsql $sql + uplevel [list do_patchset_test $tn S $res] + S delete +} + +#------------------------------------------------------------------------- +# Run simple tests of the _patchset() API. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, d, PRIMARY KEY(d, a)); + INSERT INTO t1 VALUES(1, 2, 3, 4); + INSERT INTO t1 VALUES(5, 6, 7, 8); + INSERT INTO t1 VALUES(9, 10, 11, 12); +} + +do_test 1.1 { + sqlite3session S db main + S attach t1 + execsql { + INSERT INTO t1 VALUES('w', 'x', 'y', 'z'); + DELETE FROM t1 WHERE d=4; + UPDATE t1 SET c = 14 WHERE a=5; + } +} {} + +do_patchset_test 1.2 S { + {UPDATE t1 0 X..X {i 5 {} {} {} {} i 8} {{} {} {} {} i 14 {} {}}} + {INSERT t1 0 X..X {} {t w t x t y t z}} + {DELETE t1 0 X..X {i 1 {} {} {} {} i 4} {}} +} + +do_test 1.3 { + S delete +} {} + +do_sql2patchset_test 1.4 { + DELETE FROM t1; +} { + {DELETE t1 0 X..X {i 5 {} {} {} {} i 8} {}} + {DELETE t1 0 X..X {t w {} {} {} {} t z} {}} + {DELETE t1 0 X..X {i 9 {} {} {} {} i 12} {}} +} + +do_sql2patchset_test 1.5 { + INSERT INTO t1 VALUES(X'61626364', NULL, NULL, 4.2); + INSERT INTO t1 VALUES(4.2, NULL, NULL, X'61626364'); +} { + {INSERT t1 0 X..X {} {f 4.2 n {} n {} b abcd}} + {INSERT t1 0 X..X {} {b abcd n {} n {} f 4.2}} +} + +do_sql2patchset_test 1.6 { + UPDATE t1 SET b=45 WHERE typeof(a)=='blob'; + UPDATE t1 SET c='zzzz' WHERE typeof(a)!='blob'; +} { + {UPDATE t1 0 X..X {f 4.2 {} {} {} {} b abcd} {{} {} {} {} t zzzz {} {}}} + {UPDATE t1 0 X..X {b abcd {} {} {} {} f 4.2} {{} {} i 45 {} {} {} {}}} +} + +do_sql2patchset_test 1.7 { + UPDATE t1 SET b='xyz' WHERE typeof(a)=='blob'; + UPDATE t1 SET c='xyz' WHERE typeof(a)!='blob'; + UPDATE t1 SET b=45 WHERE typeof(a)=='blob'; + UPDATE t1 SET c='zzzz' WHERE typeof(a)!='blob'; +} { +} + +do_sql2patchset_test 1.8 { + DELETE FROM t1; +} { + {DELETE t1 0 X..X {f 4.2 {} {} {} {} b abcd} {}} + {DELETE t1 0 X..X {b abcd {} {} {} {} f 4.2} {}} +} + +#------------------------------------------------------------------------- +# Run simple tests of _apply() with patchset objects. +# +reset_db + +proc noop {args} { error $args } +proc exec_rollback_replay {sql} { + sqlite3session S db main + S attach * + execsql BEGIN + execsql $sql + set patchset [S patchset] + S delete + execsql ROLLBACK + sqlite3changeset_apply db $patchset noop +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a, b, c, d, PRIMARY KEY(b,c)); + CREATE TABLE t3(w, x, y, z, PRIMARY KEY(w)); +} + +do_test 2.1 { + exec_rollback_replay { + INSERT INTO t2 VALUES(1, 2, 3, 4); + INSERT INTO t2 VALUES('w', 'x', 'y', 'z'); + } + execsql { SELECT * FROM t2 } +} {1 2 3 4 w x y z} + +do_test 2.2 { + exec_rollback_replay { + DELETE FROM t2 WHERE a=1; + UPDATE t2 SET d = 'a'; + } + execsql { SELECT * FROM t2 } +} {w x y a} + +#------------------------------------------------------------------------- +# sqlite3changeset_invert() +# +reset_db + +do_execsql_test 3.1 { CREATE TABLE t1(x PRIMARY KEY, y) } +do_test 3.2 { + sqlite3session S db main + S attach * + execsql { INSERT INTO t1 VALUES(1, 2) } + set patchset [S patchset] + S delete + list [catch { sqlite3changeset_invert $patchset } msg] [set msg] +} {1 SQLITE_CORRUPT} + + +#------------------------------------------------------------------------- +# sqlite3changeset_concat() +# +reset_db + +proc do_patchconcat_test {tn args} { + set bRevert 0 + if {[lindex $args 0] == "-revert"} { + set bRevert 1 + set args [lrange $args 1 end] + } + set nSql [expr [llength $args]-1] + set res [lindex $args $nSql] + set patchlist [list] + + execsql BEGIN + if {$bRevert} { execsql { SAVEPOINT x } } + foreach sql [lrange $args 0 end-1] { + sqlite3session S db main + S attach * + execsql $sql + lappend patchlist [S patchset] + S delete + if {$bRevert} { execsql { ROLLBACK TO x } } + } + execsql ROLLBACK + + set patch [lindex $patchlist 0] + foreach p [lrange $patchlist 1 end] { + set patch [sqlite3changeset_concat $patch $p] + } + + set x [list] + sqlite3session_foreach c $patch { lappend x $c } + + uplevel [list do_test $tn [list set {} $x] [list {*}$res]] +} + +do_execsql_test 4.1.1 { + CREATE TABLE t1(x PRIMARY KEY, y, z); +} +do_patchconcat_test 4.1.2 { + INSERT INTO t1 VALUES(1, 2, 3); +} { + INSERT INTO t1 VALUES(4, 5, 6); +} { + {INSERT t1 0 X.. {} {i 1 i 2 i 3}} + {INSERT t1 0 X.. {} {i 4 i 5 i 6}} +} + +do_execsql_test 4.2.1 { + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_patchconcat_test 4.2.2 { + UPDATE t1 SET z = 'abc' WHERE x=1 +} { + UPDATE t1 SET z = 'def' WHERE x=4 +} { + {UPDATE t1 0 X.. {i 1 {} {} {} {}} {{} {} {} {} t abc}} + {UPDATE t1 0 X.. {i 4 {} {} {} {}} {{} {} {} {} t def}} +} + +do_patchconcat_test 4.2.3 { + DELETE FROM t1 WHERE x=1; +} { + DELETE FROM t1 WHERE x=4; +} { + {DELETE t1 0 X.. {i 1 {} {} {} {}} {}} + {DELETE t1 0 X.. {i 4 {} {} {} {}} {}} +} + + +do_execsql_test 4.3.1 { + CREATE TABLE t2(a, b, c, d, PRIMARY KEY(c, b)); + INSERT INTO t2 VALUES('.', 1, 1, '.'); + INSERT INTO t2 VALUES('.', 1, 2, '.'); + INSERT INTO t2 VALUES('.', 2, 1, '.'); + INSERT INTO t2 VALUES('.', 2, 2, '.'); +} + +# INSERT + INSERT +do_patchconcat_test 4.3.2 -revert { + INSERT INTO t2 VALUES('a', 'a', 'a', 'a'); +} { + INSERT INTO t2 VALUES('b', 'a', 'a', 'b'); +} { + {INSERT t2 0 .XX. {} {t a t a t a t a}} +} + +# INSERT + DELETE +do_patchconcat_test 4.3.3 { + INSERT INTO t2 VALUES('a', 'a', 'a', 'a'); +} { + DELETE FROM t2 WHERE c = 'a'; +} { +} + +# INSERT + UPDATE +do_patchconcat_test 4.3.4 { + INSERT INTO t2 VALUES('a', 'a', 'a', 'a'); +} { + UPDATE t2 SET d = 'b' WHERE c='a'; +} { + {INSERT t2 0 .XX. {} {t a t a t a t b}} +} + +# UPDATE + UPDATE +do_patchconcat_test 4.3.5 { + UPDATE t2 SET a = 'a' WHERE c=1 AND b=2; +} { + UPDATE t2 SET d = 'd' WHERE c=1 AND b=2; +} { + {UPDATE t2 0 .XX. {{} {} i 2 i 1 {} {}} {t a {} {} {} {} t d}} +} + +# UPDATE + DELETE +do_patchconcat_test 4.3.6 { + UPDATE t2 SET a = 'a' WHERE c=1 AND b=2; +} { + DELETE FROM t2 WHERE c=1 AND b=2; +} { + {DELETE t2 0 .XX. {{} {} i 2 i 1 {} {}} {}} +} + +# DELETE + INSERT +do_patchconcat_test 4.3.7 { + DELETE FROM t2 WHERE b=1; +} { + INSERT INTO t2 VALUES('x', 1, 2, '.'); +} { + {DELETE t2 0 .XX. {{} {} i 1 i 1 {} {}} {}} + {UPDATE t2 0 .XX. {{} {} i 1 i 2 {} {}} {t x {} {} {} {} t .}} +} + +# DELETE + UPDATE +do_patchconcat_test 4.3.8 -revert { + DELETE FROM t2 WHERE b=1 AND c=2; +} { + UPDATE t2 SET a=5 WHERE b=1 AND c=2; +} { + {DELETE t2 0 .XX. {{} {} i 1 i 2 {} {}} {}} +} + +# DELETE + UPDATE +do_patchconcat_test 4.3.9 -revert { + DELETE FROM t2 WHERE b=1 AND c=2; +} { + DELETE FROM t2 WHERE b=1; +} { + {DELETE t2 0 .XX. {{} {} i 1 i 1 {} {}} {}} + {DELETE t2 0 .XX. {{} {} i 1 i 2 {} {}} {}} +} + +#------------------------------------------------------------------------- +# More rigorous testing of the _patchset(), _apply and _concat() APIs. +# +# The inputs to each test are a populate database and a list of DML +# statements. This test determines that the final database is the same +# if: +# +# 1) the statements are executed directly on the database. +# +# 2) a single patchset is collected while executing the statements and +# then applied to a copy of the original database file. +# +# 3) individual patchsets are collected for statement while executing +# them and concatenated together before being applied to a copy of +# the original database. The concatenation is done in a couple of +# different ways - linear, pairwise etc. +# +# All tests, as it happens, are run with both changesets and patchsets. +# But the focus is on patchset capabilities. +# + +# Return a checksum of the contents of the database file. Implicit IPK +# columns are not included in the checksum - just modifying rowids does +# not change the database checksum. +# +proc databasecksum {db} { + set alltab [$db eval {SELECT name FROM sqlite_master WHERE type='table'}] + foreach tab $alltab { + $db eval "SELECT * FROM $tab LIMIT 1" res { } + set slist [list] + foreach col [lsort $res(*)] { + lappend slist "quote($col)" + } + set sql "SELECT [join $slist ,] FROM $tab" + append txt "[lsort [$db eval $sql]]\n" + } + return [md5 $txt] +} + +proc do_patchset_test {tn tstcmd lSql} { + if {$tstcmd != "patchset" && $tstcmd != "changeset"} { + error "have $tstcmd: must be patchset or changeset" + } + + foreach fname {test.db2 test.db3 test.db4 test.db5} { + forcedelete $fname + forcecopy test.db $fname + } + + # Execute the SQL statements on [db]. Collect a patchset for each + # individual statement, as well as a single patchset for the entire + # operation. + sqlite3session S db main + S attach * + foreach sql $lSql { + sqlite3session T db main + T attach * + db eval $sql + lappend lPatch [T $tstcmd] + T delete + } + set patchset [S $tstcmd] + S delete + + # Calculate a checksum for the final database. + set cksum [databasecksum db] + + # 1. Apply the single large patchset to test.db2 + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $patchset noop + uplevel [list do_test $tn.1 { databasecksum db2 } $cksum ] + db2 close + + # 2. Apply each of the single-statement patchsets to test.db3 + sqlite3 db2 test.db3 + foreach p $lPatch { + sqlite3changeset_apply db2 $p noop + } + uplevel [list do_test $tn.2 { databasecksum db2 } $cksum ] + db2 close + + # 3. Concatenate all single-statement patchsets into a single large + # patchset, then apply it to test.db4. + # + sqlite3 db2 test.db4 + set big "" + foreach p $lPatch { + set big [sqlite3changeset_concat $big $p] + } + sqlite3changeset_apply db2 $big noop + uplevel [list do_test $tn.3 { databasecksum db2 } $cksum ] + db2 close + + # 4. Concatenate all single-statement patchsets pairwise into a single + # large patchset, then apply it to test.db5. Pairwise concatenation: + # + # a b c d e f g h i j k + # -> {a b} {c d} {e f} {g h} {i j} k + # -> {a b c d} {e f g h} {i j k} + # -> {a b c d e f g h} {i j k} + # -> {a b c d e f g h i j k} + # -> APPLY! + # + sqlite3 db2 test.db5 + set L $lPatch + while {[llength $L] > 1} { + set O [list] + for {set i 0} {$i < [llength $L]} {incr i 2} { + if {$i==[llength $L]-1} { + lappend O [lindex $L $i] + } else { + set i1 [expr $i+1] + lappend O [sqlite3changeset_concat [lindex $L $i] [lindex $L $i1]] + } + } + set L $O + } + sqlite3changeset_apply db2 [lindex $L 0] noop + uplevel [list do_test $tn.4 { databasecksum db2 } $cksum ] + db2 close +} + +proc do_patchset_changeset_test {tn initsql args} { + foreach tstcmd {patchset changeset} { + reset_db + execsql $initsql + set x 0 + foreach sql $args { + incr x + set lSql [split $sql ";"] + uplevel [list do_patchset_test $tn.$tstcmd.$x $tstcmd $lSql] + } + } +} + +do_patchset_changeset_test 5.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); +} { + INSERT INTO t1 VALUES(4, 5, 6); + DELETE FROM t1 WHERE a=1; +} { + INSERT INTO t1 VALUES(7, 8, 9); + UPDATE t1 SET c = 5; + INSERT INTO t1 VALUES(10, 11, 12); + UPDATE t1 SET c = 6; + INSERT INTO t1 VALUES(13, 14, 15); +} { + UPDATE t1 SET c=c+1; + DELETE FROM t1 WHERE (a%2); +} + +do_patchset_changeset_test 5.2 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE TABLE t2(a, b, c, d, PRIMARY KEY(c, b)); +} { + INSERT INTO t1 VALUES(x'00', 0, 'zero'); + INSERT INTO t1 VALUES(x'01', 1, 'one'); + INSERT INTO t1 VALUES(x'02', 4, 'four'); + INSERT INTO t1 VALUES(x'03', 9, 'nine'); + INSERT INTO t1 VALUES(x'04', 16, 'sixteen'); + INSERT INTO t1 VALUES(x'05', 25, 'twenty-five'); +} { + UPDATE t1 SET a = b WHERE b<=4; + INSERT INTO t2 SELECT NULL, * FROM t1; + DELETE FROM t1 WHERE b=25; +} { + DELETE FROM t2; + INSERT INTO t2 SELECT NULL, * FROM t1; + DELETE FROM t1; + INSERT INTO t1 SELECT b, c, d FROM t2; + UPDATE t1 SET b = b+1; + UPDATE t1 SET b = b+1; + UPDATE t1 SET b = b+1; +} + +set initsql { CREATE TABLE t1(a, b, c, PRIMARY KEY(c, b)); } +for {set i 0} {$i < 1000} {incr i} { + append insert "INSERT INTO t1 VALUES($i, $i, $i);" + append delete "DELETE FROM t1 WHERE b=$i;" +} +do_patchset_changeset_test 5.3 \ + $initsql $insert $delete \ + $insert $delete \ + "$insert $delete" \ + $delete + + +finish_test diff --git a/ext/session/sessionC.test b/ext/session/sessionC.test new file mode 100644 index 00000000..390a31de --- /dev/null +++ b/ext/session/sessionC.test @@ -0,0 +1,198 @@ +# 2014 August 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionC + +#------------------------------------------------------------------------- +# Test the outcome of a DELETE operation made as part of applying a +# changeset failing with SQLITE_CONSTRAINT. This may happen if an +# ON DELETE RESTRICT foreign key action is triggered, or if a trigger +# program raises a constraint somehow. +# +# UPDATE: The above is no longer true, as "PRAGMA defer_foreign_keys" +# now disables "RESTRICT" processing. The test below has been rewritten +# to use a trigger instead of a foreign key to test this case. +# +do_execsql_test 1.0 { + PRAGMA foreign_keys = 1; + + CREATE TABLE p(a PRIMARY KEY, b, c); + CREATE TABLE c(d PRIMARY KEY, e /* REFERENCES p ON DELETE RESTRICT */); + + CREATE TRIGGER restrict_trig BEFORE DELETE ON p BEGIN + SELECT raise(ABORT, 'error!') FROM c WHERE e=old.a; + END; + + INSERT INTO p VALUES('one', 1, 1); + INSERT INTO p VALUES('two', 2, 2); + INSERT INTO p VALUES('three', 3, 3); + + INSERT INTO c VALUES(1, 'one'); + INSERT INTO c VALUES(3, 'three'); +} + +do_test 1.1 { + execsql BEGIN + set C [changeset_from_sql { + INSERT INTO c VALUES(4, 'one'); + DELETE FROM p WHERE a='two'; + }] + execsql ROLLBACK + execsql { + INSERT INTO c VALUES(2, 'two'); + } +} {} + +do_test 1.2.1 { + proc xConflict {args} { return "ABORT" } + catch { sqlite3changeset_apply db $C xConflict } msg + set msg +} {SQLITE_ABORT} +do_execsql_test 1.2.2 { SELECT * FROM c } {1 one 3 three 2 two} + +do_test 1.3.1 { + proc xConflict {args} { return "OMIT" } + catch { sqlite3changeset_apply db $C xConflict } msg + set msg +} {} +do_execsql_test 1.3.2 { SELECT * FROM c } {1 one 3 three 2 two 4 one} +do_execsql_test 1.3.3 { + SELECT * FROM p; +} {one 1 1 two 2 2 three 3 3} + + +#------------------------------------------------------------------------- +# Test that concatenating a changeset with a patchset does not work. +# Any attempt to do so returns SQLITE_ERROR. +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(t, v PRIMARY KEY); + INSERT INTO x1 VALUES(12, 55); + INSERT INTO x1 VALUES(55, 14); +} + +do_test 2.1 { + execsql BEGIN + + sqlite3session S1 db main + S1 attach * + execsql { + UPDATE x1 SET t=13 WHERE v=55; + INSERT INTO x1 VALUES(99, 123); + } + set patchset [S1 patchset] + S1 delete + + sqlite3session S1 db main + S1 attach * + execsql { + UPDATE x1 SET t=56 WHERE v=14; + INSERT INTO x1 VALUES(22, 998); + } + set changeset [S1 changeset] + S1 delete + + execsql ROLLBACK +} {} + +do_test 2.2 { + set rc [catch { sqlite3changeset_concat $patchset $changeset } msg] + list $rc $msg +} {1 SQLITE_ERROR} + +do_test 2.3 { + set rc [catch { sqlite3changeset_concat $changeset $patchset } msg] + list $rc $msg +} {1 SQLITE_ERROR} + +do_test 2.4 { + set rc [catch { sqlite3changeset_concat {} $patchset } msg] + list $rc $msg +} [list 0 $patchset] + +do_test 2.5 { + set rc [catch { sqlite3changeset_concat $patchset {} } msg] + list $rc $msg +} [list 0 $patchset] + +do_test 2.6 { + set rc [catch { sqlite3changeset_concat {} $changeset } msg] + list $rc $msg +} [list 0 $changeset] + +do_test 2.7 { + set rc [catch { sqlite3changeset_concat $changeset {} } msg] + list $rc $msg +} [list 0 $changeset] + +do_test 2.8 { + set rc [catch { sqlite3changeset_concat {} {} } msg] + list $rc $msg +} [list 0 {}] + + +#------------------------------------------------------------------------- +# Test that the xFilter argument to sqlite3changeset_apply() works. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(a PRIMARY KEY, b); + CREATE TABLE t3(a PRIMARY KEY, b); +} +do_test 3.1 { + execsql BEGIN + set changeset [changeset_from_sql { + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t3 VALUES(3, 3); + }] + execsql ROLLBACK +} {} +do_test 3.2 { + proc xFilter {zName} { + if {$zName == "t1"} { return 1 } + return 0 + } + sqlite3changeset_apply db $changeset noop xFilter + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 1} +do_test 3.3 { + proc xFilter {zName} { + if {$zName == "t3"} { return 1 } + return 0 + } + sqlite3changeset_apply db $changeset noop xFilter + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 1 3 3} + + + +finish_test + diff --git a/ext/session/sessionD.test b/ext/session/sessionD.test new file mode 100644 index 00000000..b8572782 --- /dev/null +++ b/ext/session/sessionD.test @@ -0,0 +1,225 @@ +# 2014 August 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file focuses on the sqlite3session_diff() function. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionD + +proc scksum {db dbname} { + + if {$dbname=="temp"} { + set master sqlite_temp_master + } else { + set master $dbname.sqlite_master + } + + set alltab [$db eval "SELECT name FROM $master WHERE type='table'"] + set txt [$db eval "SELECT * FROM $master ORDER BY type,name,sql"] + foreach tab $alltab { + set cols [list] + db eval "PRAGMA $dbname.table_info = $tab" x { + lappend cols "quote($x(name))" + } + set cols [join $cols ,] + append txt [db eval "SELECT $cols FROM $tab ORDER BY $cols"] + } + return [md5 $txt] +} + +proc do_diff_test {tn setup} { + reset_db + forcedelete test.db2 + execsql { ATTACH 'test.db2' AS aux } + execsql $setup + + sqlite3session S db main + foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $tbl + S diff aux $tbl + } + + set C [S changeset] + S delete + + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $C "" + uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok + db2 close + + set cksum [scksum db main] + uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum] +} + +# Ensure that the diff produced by comparing the current contents of [db] +# with itself is empty. +proc do_empty_diff_test {tn} { + forcedelete test.db2 + forcecopy test.db test.db2 + + execsql { ATTACH 'test.db2' AS aux } + sqlite3session S db main + foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $tbl + S diff aux $tbl + } + + set ::C [S changeset] + S delete + + uplevel [list do_test $tn {string length $::C} 0] +} + + +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t2 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t2(a PRIMARY KEY, b); +} + +do_test 1.1 { + sqlite3session S db main + S attach t2 + S diff aux t2 + set C [S changeset] + S delete +} {} + +do_test 1.2 { + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $C "" + db2 close + db eval { SELECT * FROM aux.t2 } +} {1 one 2 two} + +do_diff_test 2.1 { + CREATE TABLE aux.t1(x, y, PRIMARY KEY(y)); + CREATE TABLE t1(x, y, PRIMARY KEY(y)); + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(NULL, 'xyz'); + INSERT INTO t1 VALUES(4.5, 5.5); +} + +do_diff_test 2.2 { + CREATE TABLE aux.t1(x, y, PRIMARY KEY(y)); + CREATE TABLE t1(x, y, PRIMARY KEY(y)); + + INSERT INTO aux.t1 VALUES(1, 2); + INSERT INTO aux.t1 VALUES(NULL, 'xyz'); + INSERT INTO aux.t1 VALUES(4.5, 5.5); +} + +do_diff_test 2.3 { + CREATE TABLE aux.t1(a PRIMARY KEY, b TEXT); + CREATE TABLE t1(a PRIMARY KEY, b TEXT); + + INSERT INTO aux.t1 VALUES(1, 'one'); + INSERT INTO aux.t1 VALUES(2, 'two'); + INSERT INTO aux.t1 VALUES(3, 'three'); + + INSERT INTO t1 VALUES(1, 'I'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'III'); +} + +do_diff_test 2.4 { + CREATE TABLE aux.t1(a, b, c, d, PRIMARY KEY(c, b, a)); + CREATE TABLE t1(a, b, c, d, PRIMARY KEY(c, b, a)); + + INSERT INTO t1 VALUES('hvkzyipambwdqlvwv','',-458331.50,X'DA51ED5E84'); + INSERT INTO t1 VALUES(X'C5C6B5DD','jjxrath',40917,830244); + INSERT INTO t1 VALUES(-204877.54,X'1704C253D5F3AFA8',155120.88,NULL); + INSERT INTO t1 + VALUES('ckmqmzoeuvxisxqy',X'EB5A5D3A1DD22FD1','tidhjcbvbppdt',-642987.37); + INSERT INTO t1 VALUES(-851726,-161992,-469943,-159541); + INSERT INTO t1 VALUES(X'4A6A667F858938',185083,X'7A',NULL); + + INSERT INTO aux.t1 VALUES(415075.74,'auawczkb',X'',X'57B4FAAF2595'); + INSERT INTO aux.t1 VALUES(727637,711560,-181340,'hphuo'); + INSERT INTO aux.t1 + VALUES(-921322.81,662959,'lvlgwdgxaurr','ajjrzrbhqflsutnymgc'); + INSERT INTO aux.t1 VALUES(-146061,-377892,X'4E','gepvpvvuhszpxabbb'); + INSERT INTO aux.t1 VALUES(-851726,-161992,-469943,-159541); + INSERT INTO aux.t1 VALUES(X'4A6A667F858938',185083,X'7A',NULL); + INSERT INTO aux.t1 VALUES(-204877.54,X'1704C253D5F3AFA8',155120.88, 4); + INSERT INTO aux.t1 + VALUES('ckmqmzoeuvxisxqy',X'EB5A5D3A1DD22FD1','tidgtsplhjcbvbppdt',-642987.3); +} + +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(4, 5, 6); + INSERT INTO t2 VALUES(7, 8, 9); + + CREATE TABLE t3(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO t3 VALUES(1, 2, 3); + INSERT INTO t3 VALUES(4, 5, 6); + INSERT INTO t3 VALUES(7, 8, 9); +} +do_empty_diff_test 3.1 + + +#------------------------------------------------------------------------- +# Test some error cases: +# +# 1) schema mismatches between the two dbs, and +# 2) tables with no primary keys. This is not actually an error, but +# should not add any changes to the session object. +# +reset_db +forcedelete test.db2 +do_execsql_test 4.0 { + ATTACH 'test.db2' AS ixua; + CREATE TABLE ixua.t1(a, b, c); + CREATE TABLE main.t1(a, b, c); + INSERT INTO main.t1 VALUES(1, 2, 3); + + CREATE TABLE ixua.t2(a PRIMARY KEY, b, c); + CREATE TABLE main.t2(a PRIMARY KEY, b, x); +} + +do_test 4.1.1 { + sqlite3session S db main + S attach t1 + list [catch { S diff ixua t1 } msg] $msg +} {0 {}} +do_test 4.1.2 { + string length [S changeset] +} {0} +S delete + +do_test 4.2.2 { + sqlite3session S db main + S attach t2 + list [catch { S diff ixua t2 } msg] $msg +} {1 {SQLITE_SCHEMA - table schemas do not match}} +S delete + +finish_test + diff --git a/ext/session/sessionE.test b/ext/session/sessionE.test new file mode 100644 index 00000000..27e530ec --- /dev/null +++ b/ext/session/sessionE.test @@ -0,0 +1,117 @@ +# 2015 June 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that operations on tables without primary keys +# are ignored. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionE + +# +# Test plan: +# +# 1.*: Test that non-PK tables are not auto-attached. +# 2.*: Test that explicitly attaching a non-PK table is a no-op. +# 3.*: Test that sqlite3session_diff() on a non-PK table is a no-op. +# + + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a PRIMARY KEY, b); +} +do_test 1.1 { + sqlite3session S db main + S attach * + breakpoint + execsql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES(1, 2); + } +} {} +do_changeset_test 1.2 S { + {INSERT t2 0 X. {} {i 1 i 2}} +} +S delete + +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a PRIMARY KEY, b); +} +do_test 2.1 { + sqlite3session S db main + S attach t1 + S attach t2 + breakpoint + execsql { + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t2 VALUES(5, 6); + } +} {} +do_changeset_test 2.2 S { + {INSERT t2 0 X. {} {i 3 i 4}} + {INSERT t2 0 X. {} {i 5 i 6}} +} +S delete + +reset_db +forcedelete test.db2 +do_execsql_test 3.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t1(a, b); + CREATE TABLE aux.t2(a PRIMARY KEY, b); + + CREATE TABLE t1(a, b); + CREATE TABLE t2(a PRIMARY KEY, b); + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES(3, 4); +} +do_test 3.1 { + sqlite3session S db main + S attach t1 + S diff aux t1 + + S attach t2 + S diff aux t2 +} {} +do_changeset_test 3.2 S { + {INSERT t2 0 X. {} {i 3 i 4}} +} +do_execsql_test 3.3 { + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t2 VALUES(7, 8); +} +do_changeset_test 3.4 S { + {INSERT t2 0 X. {} {i 3 i 4}} + {INSERT t2 0 X. {} {i 7 i 8}} +} + + +S delete + +finish_test + + diff --git a/ext/session/sessionF.test b/ext/session/sessionF.test new file mode 100644 index 00000000..ba57bf85 --- /dev/null +++ b/ext/session/sessionF.test @@ -0,0 +1,295 @@ +# 2015 June 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that tables appear in the correct order +# within changesets and patchsets. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionF + +# +# Test plan: +# +# 1.*: Test that sqlite3session_changeset() and sqlite3session_patchset() +# output tables in the right order. +# +# 2.*: Test that sqlite3session_invert() does not modify the order of +# tables within a changeset. +# +# 3.*: Test that sqlite3session_concat outputs tables in the right order. +# + +# Create a db schema to use. +# +do_execsql_test 1.0 { + CREATE TABLE t3(e PRIMARY KEY, f); + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(c PRIMARY KEY, d); +} + +#----------------------------------------------------------------------- +# 1.* - changeset() and patchset(). +# + +foreach {tn setup result} { + 1 { + S attach * + } { + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t3 0 X. {} {i 3 t three}} + } + + 2 { + S attach t1 + S attach * + } { + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t3 0 X. {} {i 3 t three}} + } + + 3 { + S attach t3 + S attach t2 + S attach t1 + } { + {INSERT t3 0 X. {} {i 3 t three}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t1 0 X. {} {i 1 t one}} + } +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup + + do_execsql_test 1.$tn.1 { + INSERT INTO t2 VALUES(2, 'two'); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t3 VALUES(3, 'three'); + } + + do_changeset_test 1.1.$tn.2 S $result + do_patchset_test 1.1.$tn.3 S $result + + S delete +} + +foreach {tn setup result} { + 1 { + S attach * + } { + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t2 0 X. {} {i 5 t five}} + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t3 0 X. {} {i 6 t six}} + } + + 2 { + S attach t1 + S attach * + } { + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t2 0 X. {} {i 5 t five}} + {INSERT t3 0 X. {} {i 6 t six}} + } + + 3 { + S attach t3 + S attach t2 + S attach t1 + } { + {INSERT t3 0 X. {} {i 6 t six}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t2 0 X. {} {i 5 t five}} + {INSERT t1 0 X. {} {i 1 t one}} + } +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup + + do_execsql_test 1.$tn.1 { + INSERT INTO t2 VALUES(2, 'two'); + INSERT INTO t1 VALUES(1, 'one'); + DELETE FROM t2; + INSERT INTO t2 VALUES(4, 'four'); + INSERT INTO t2 VALUES(5, 'five'); + INSERT INTO t3 VALUES(6, 'six'); + } + + do_changeset_test 1.2.$tn.2 S $result + do_patchset_test 1.2.$tn.2 S $result + + S delete +} + +#------------------------------------------------------------------------- +# 2.* - invert() +# + +foreach {tn setup result} { + 1 { + S attach * + } { + {DELETE t2 0 X. {i 4 t four} {}} + {DELETE t2 0 X. {i 5 t five} {}} + {DELETE t1 0 X. {i 1 t one} {}} + {DELETE t3 0 X. {i 6 t six} {}} + } + + 2 { + S attach t1 + S attach * + } { + {DELETE t1 0 X. {i 1 t one} {}} + {DELETE t2 0 X. {i 4 t four} {}} + {DELETE t2 0 X. {i 5 t five} {}} + {DELETE t3 0 X. {i 6 t six} {}} + } + + 3 { + S attach t3 + S attach t2 + S attach t1 + } { + {DELETE t3 0 X. {i 6 t six} {}} + {DELETE t2 0 X. {i 4 t four} {}} + {DELETE t2 0 X. {i 5 t five} {}} + {DELETE t1 0 X. {i 1 t one} {}} + } +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup + + do_execsql_test 1.$tn.1 { + INSERT INTO t2 VALUES(2, 'two'); + INSERT INTO t1 VALUES(1, 'one'); + DELETE FROM t2; + INSERT INTO t2 VALUES(4, 'four'); + INSERT INTO t2 VALUES(5, 'five'); + INSERT INTO t3 VALUES(6, 'six'); + } + + do_changeset_invert_test 2.$tn.2 S $result + + S delete +} + +#------------------------------------------------------------------------- +# 3.* - concat() +# +foreach {tn setup1 sql1 setup2 sql2 result} { + 1 { + S attach * + } { + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + } { + S attach t2 + S attach t1 + } { + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t2 VALUES(4, 'four'); + } { + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t1 0 X. {} {i 3 t three}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 4 t four}} + } + + 1 { + S attach t2 + S attach t1 + } { + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + } { + S attach * + } { + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t2 VALUES(4, 'four'); + } { + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t1 0 X. {} {i 3 t three}} + } + + 1 { + S attach * + } { + INSERT INTO t2 VALUES(2, 'two'); + } { + S attach * + } { + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t2 VALUES(4, 'four'); + INSERT INTO t3 VALUES(5, 'five'); + } { + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t1 0 X. {} {i 3 t three}} + {INSERT t3 0 X. {} {i 5 t five}} + } + +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup1 + execsql $sql1 + set c1 [S changeset] + S delete + + sqlite3session S db main + eval $setup2 + execsql $sql2 + set c2 [S changeset] + S delete + + set res [list] + sqlite3session_foreach x [sqlite3changeset_concat $c1 $c2] { + lappend res $x + } + + do_test 3.$tn { set res } [list {*}$result] +} + + +finish_test + diff --git a/ext/session/sessionG.test b/ext/session/sessionG.test new file mode 100644 index 00000000..5c057350 --- /dev/null +++ b/ext/session/sessionG.test @@ -0,0 +1,177 @@ +# 2016 March 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that UNIQUE constraints are dealt with correctly. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionG + + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 1.0 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + } + do_then_apply_sql { + DELETE FROM t1 WHERE a=1; + INSERT INTO t1 VALUES(4, 'one'); + } + compare_db db db2 +} {} + +do_test 1.1 { + do_then_apply_sql { + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(1, 'one'); + } + compare_db db db2 +} {} + +do_test 1.2 { + execsql { INSERT INTO t1 VALUES(5, 'five') } db2 + do_then_apply_sql { + INSERT INTO t1 VALUES(11, 'eleven'); + INSERT INTO t1 VALUES(12, 'five'); + } + execsql { SELECT * FROM t1 } db2 +} {2 two 3 three 1 one 5 five 11 eleven} + +do_test 1.3 { + execsql { SELECT * FROM t1 } +} {2 two 3 three 1 one 11 eleven 12 five} + +#------------------------------------------------------------------------- +# +reset_db +db2 close +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 2.1 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE, c UNIQUE); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } +} {} + +do_test 2.2.1 { + # It is not possible to apply the changeset generated by the following + # SQL, as none of the three updated rows may be updated as part of the + # first pass. + do_then_apply_sql { + UPDATE t1 SET b=0 WHERE a=1; + UPDATE t1 SET b=1 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=1; + } + db2 eval { SELECT a, b FROM t1 } +} {1 1 2 2 3 3} +do_test 2.2.2 { db eval { SELECT a, b FROM t1 } } {1 3 2 1 3 2} + +#------------------------------------------------------------------------- +# +reset_db +db2 close +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 3.1 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE, c UNIQUE); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } +} {} + +do_test 3.3 { + do_then_apply_sql { + UPDATE t1 SET b=4 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=1; + } + compare_db db db2 +} {} + +do_test 3.4 { + do_then_apply_sql { + UPDATE t1 SET b=1 WHERE a=1; + UPDATE t1 SET b=2 WHERE a=2; + UPDATE t1 SET b=3 WHERE a=3; + } + compare_db db db2 +} {} + +#------------------------------------------------------------------------- +# +reset_db +db2 close +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 4.1 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + + CREATE TABLE t2(a PRIMARY KEY, b UNIQUE); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t2 VALUES(3, 3); + } +} {} + +do_test 4.2 { + do_then_apply_sql { + UPDATE t1 SET b=4 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=1; + + UPDATE t2 SET b=0 WHERE a=1; + UPDATE t2 SET b=1 WHERE a=2; + UPDATE t2 SET b=2 WHERE a=3; + } + compare_db db db2 +} {} + +do_test 4.3 { + do_then_apply_sql { + UPDATE t1 SET b=1 WHERE a=1; + UPDATE t1 SET b=2 WHERE a=2; + UPDATE t1 SET b=3 WHERE a=3; + + UPDATE t2 SET b=3 WHERE a=3; + UPDATE t2 SET b=2 WHERE a=2; + UPDATE t2 SET b=1 WHERE a=1; + } + compare_db db db2 +} {} + +finish_test + diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl new file mode 100644 index 00000000..06b05509 --- /dev/null +++ b/ext/session/session_common.tcl @@ -0,0 +1,170 @@ + +proc do_changeset_test {tn session res} { + set r [list] + foreach x $res {lappend r $x} + uplevel do_test $tn [list [subst -nocommands { + set x [list] + sqlite3session_foreach c [$session changeset] { lappend x [set c] } + set x + }]] [list $r] +} + +proc do_patchset_test {tn session res} { + set r [list] + foreach x $res {lappend r $x} + uplevel do_test $tn [list [subst -nocommands { + set x [list] + sqlite3session_foreach c [$session patchset] { lappend x [set c] } + set x + }]] [list $r] +} + + +proc do_changeset_invert_test {tn session res} { + set r [list] + foreach x $res {lappend r $x} + uplevel do_test $tn [list [subst -nocommands { + set x [list] + set changeset [sqlite3changeset_invert [$session changeset]] + sqlite3session_foreach c [set changeset] { lappend x [set c] } + set x + }]] [list $r] +} + + +proc do_conflict_test {tn args} { + proc xConflict {args} { + lappend ::xConflict $args + return "" + } + proc bgerror {args} { set ::background_error $args } + + + set O(-tables) [list] + set O(-sql) [list] + set O(-conflicts) [list] + + array set V $args + foreach key [array names V] { + if {![info exists O($key)]} {error "no such option: $key"} + } + array set O $args + + sqlite3session S db main + foreach t $O(-tables) { S attach $t } + execsql $O(-sql) + + set ::xConflict [list] + sqlite3changeset_apply db2 [S changeset] xConflict + + set conflicts [list] + foreach c $O(-conflicts) { + lappend conflicts $c + } + + after 1 {set go 1} + vwait go + + uplevel do_test $tn [list { set ::xConflict }] [list $conflicts] + S delete +} + +proc do_common_sql {sql} { + execsql $sql db + execsql $sql db2 +} + +proc changeset_from_sql {sql {dbname main}} { + if {$dbname == "main"} { + return [sql_exec_changeset db $sql] + } + set rc [catch { + sqlite3session S db $dbname + db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { + S attach $name + } + db eval $sql + S changeset + } changeset] + catch { S delete } + + if {$rc} { + error $changeset + } + return $changeset +} + +proc do_then_apply_sql {sql {dbname main}} { + proc xConflict args { return "OMIT" } + set rc [catch { + sqlite3session S db $dbname + db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { + S attach $name + } + db eval $sql + sqlite3changeset_apply db2 [S changeset] xConflict + } msg] + + catch { S delete } + + if {$rc} {error $msg} +} + +proc do_iterator_test {tn tbl_list sql res} { + sqlite3session S db main + if {[llength $tbl_list]==0} { S attach * } + foreach t $tbl_list {S attach $t} + + execsql $sql + + set r [list] + foreach v $res { lappend r $v } + + set x [list] + sqlite3session_foreach c [S changeset] { lappend x $c } + uplevel do_test $tn [list [list set {} $x]] [list $r] + + S delete +} + +# Compare the contents of all tables in [db1] and [db2]. Throw an error if +# they are not identical, or return an empty string if they are. +# +proc compare_db {db1 db2} { + + set sql {SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name} + set lot1 [$db1 eval $sql] + set lot2 [$db2 eval $sql] + + if {$lot1 != $lot2} { + puts $lot1 + puts $lot2 + error "databases contain different tables" + } + + foreach tbl $lot1 { + set col1 [list] + set col2 [list] + + $db1 eval "PRAGMA table_info = $tbl" { lappend col1 $name } + $db2 eval "PRAGMA table_info = $tbl" { lappend col2 $name } + if {$col1 != $col2} { error "table $tbl schema mismatch" } + + set sql "SELECT * FROM $tbl ORDER BY [join $col1 ,]" + set data1 [$db1 eval $sql] + set data2 [$db2 eval $sql] + if {$data1 != $data2} { + puts "$data1" + puts "$data2" + error "table $tbl data mismatch" + } + } + + return "" +} + +proc changeset_to_list {c} { + set list [list] + sqlite3session_foreach elem $c { lappend list $elem } + lsort $list +} diff --git a/ext/session/sessionfault.test b/ext/session/sessionfault.test new file mode 100644 index 00000000..be6c4568 --- /dev/null +++ b/ext/session/sessionfault.test @@ -0,0 +1,589 @@ +# 2011 Mar 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionfault + +forcedelete test.db2 +sqlite3 db2 test.db2 +do_common_sql { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} +faultsim_save_and_close +db2 close + +#------------------------------------------------------------------------- +# Test OOM error handling when collecting and applying a simple changeset. +# +# Test 1.1 attaches tables individually by name to the session object. +# Whereas test 1.2 passes NULL to sqlite3session_attach() to attach all +# tables. +# +do_faultsim_test 1.1 -faults oom-* -prep { + catch {db2 close} + catch {db close} + faultsim_restore_and_reopen + sqlite3 db2 test.db2 +} -body { + do_then_apply_sql { + INSERT INTO t1 VALUES('a string value', 8, 9); + UPDATE t1 SET c = 10 WHERE a = 1; + DELETE FROM t1 WHERE a = 4; + } +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + faultsim_integrity_check + if {$testrc==0} { compare_db db db2 } +} + +do_faultsim_test 1.2 -faults oom-* -prep { + catch {db2 close} + catch {db close} + faultsim_restore_and_reopen +} -body { + sqlite3session S db main + S attach * + execsql { + INSERT INTO t1 VALUES('a string value', 8, 9); + UPDATE t1 SET c = 10 WHERE a = 1; + DELETE FROM t1 WHERE a = 4; + } + set ::changeset [S changeset] + set {} {} +} -test { + catch { S delete } + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + faultsim_integrity_check + if {$testrc==0} { + proc xConflict {args} { return "OMIT" } + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $::changeset xConflict + compare_db db db2 + } +} + +#------------------------------------------------------------------------- +# The following block of tests - 2.* - are designed to check +# the handling of faults in the sqlite3changeset_apply() function. +# +catch {db close} +catch {db2 close} +forcedelete test.db2 test.db +sqlite3 db2 test.db2 +sqlite3 db test.db +do_common_sql { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)); + INSERT INTO t1 VALUES('apple', 'orange', 'pear'); + + CREATE TABLE t2(x PRIMARY KEY, y); +} +db2 close +faultsim_save_and_close + + +foreach {tn conflict_policy sql sql2} { + 1 OMIT { INSERT INTO t1 VALUES('one text', 'two text', X'00ff00') } {} + 2 OMIT { DELETE FROM t1 WHERE a = 'apple' } {} + 3 OMIT { UPDATE t1 SET c = 'banana' WHERE b = 'orange' } {} + 4 REPLACE { INSERT INTO t2 VALUES('keyvalue', 'value 1') } { + INSERT INTO t2 VALUES('keyvalue', 'value 2'); + } +} { + proc xConflict args [list return $conflict_policy] + + do_faultsim_test 2.$tn -faults oom-transient -prep { + catch {db2 close} + catch {db close} + faultsim_restore_and_reopen + set ::changeset [changeset_from_sql $::sql] + sqlite3 db2 test.db2 + sqlite3_db_config_lookaside db2 0 0 0 + execsql $::sql2 db2 + } -body { + sqlite3changeset_apply db2 $::changeset xConflict + } -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + faultsim_integrity_check + if {$testrc==0} { compare_db db db2 } + } +} + +#------------------------------------------------------------------------- +# This test case is designed so that a malloc() failure occurs while +# resizing the session object hash-table from 256 to 512 buckets. This +# is not an error, just a sub-optimal condition. +# +do_faultsim_test 3 -faults oom-* -prep { + catch {db2 close} + catch {db close} + faultsim_restore_and_reopen + sqlite3 db2 test.db2 + + sqlite3session S db main + S attach t1 + execsql { BEGIN } + for {set i 0} {$i < 125} {incr i} { + execsql {INSERT INTO t1 VALUES(10+$i, 10+$i, 10+$i)} + } +} -body { + for {set i 125} {$i < 133} {incr i} { + execsql {INSERT INTO t1 VALUES(10+$i, 10+$i, 1-+$i)} + } + S changeset + set {} {} +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + sqlite3changeset_apply db2 [S changeset] xConflict + compare_db db db2 + } + catch { S delete } + faultsim_integrity_check +} + +catch { db close } +catch { db2 close } +forcedelete test.db2 test.db +sqlite3 db2 test.db2 +sqlite3 db test.db + +proc xConflict {op tbl type args} { + if { $type=="CONFLICT" || $type=="DATA" } { + return "REPLACE" + } + return "OMIT" +} + +do_test 4.0 { + execsql { + PRAGMA encoding = 'utf16'; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(5, 32); + } + execsql { + PRAGMA encoding = 'utf16'; + CREATE TABLE t1(a PRIMARY KEY, b NOT NULL); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(2, 4); + INSERT INTO t1 VALUES(4, 16); + } db2 +} {} + +faultsim_save_and_close +db2 close + +do_faultsim_test 4 -faults oom-* -prep { + catch {db2 close} + catch {db close} + faultsim_restore_and_reopen + sqlite3 db2 test.db2 + sqlite3session S db main + S attach t1 + execsql { + INSERT INTO t1 VALUES(1, 45); + INSERT INTO t1 VALUES(2, 55); + INSERT INTO t1 VALUES(3, 55); + UPDATE t1 SET a = 4 WHERE a = 5; + } +} -body { + sqlite3changeset_apply db2 [S changeset] xConflict +} -test { + catch { S delete } + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { compare_db db db2 } +} + +#------------------------------------------------------------------------- +# This block of tests verifies that OOM faults in the +# sqlite3changeset_invert() function are handled correctly. +# +catch {db close} +catch {db2 close} +forcedelete test.db +sqlite3 db test.db +execsql { + CREATE TABLE t1(a, b, PRIMARY KEY(b)); + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('string', 1); + INSERT INTO t1 VALUES(4, 2); + INSERT INTO t1 VALUES(X'FFAAFFAAFFAA', 3); +} +set changeset [changeset_from_sql { + INSERT INTO t1 VALUES('xxx', 'yyy'); + DELETE FROM t1 WHERE a = 'string'; + UPDATE t1 SET a = 20 WHERE b = 2; +}] +db close + +do_faultsim_test 5.1 -faults oom* -body { + set ::inverse [sqlite3changeset_invert $::changeset] + set {} {} +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + set x [list] + sqlite3session_foreach c $::inverse { lappend x $c } + foreach c { + {DELETE t1 0 .X {t xxx t yyy} {}} + {INSERT t1 0 .X {} {t string i 1}} + {UPDATE t1 0 .X {i 20 i 2} {i 4 {} {}}} + } { lappend y $c } + if {$x != $y} { error "changeset no good" } + } +} + +catch {db close} +catch {db2 close} +forcedelete test.db +sqlite3 db test.db +execsql { + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t2 VALUES(1, 'abc'); + INSERT INTO t2 VALUES(2, 'def'); +} +set changeset [changeset_from_sql { + UPDATE t2 SET b = (b || b || b || b); + UPDATE t2 SET b = (b || b || b || b); + UPDATE t2 SET b = (b || b || b || b); + UPDATE t2 SET b = (b || b || b || b); +}] +db close +set abc [string repeat abc 256] +set def [string repeat def 256] + +do_faultsim_test 5.2 -faults oom-tra* -body { + set ::inverse [sqlite3changeset_invert $::changeset] + set {} {} +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + set x [list] + sqlite3session_foreach c $::inverse { lappend x $c } + foreach c " + {UPDATE t2 0 X. {i 1 t $::abc} {{} {} t abc}} + {UPDATE t2 0 X. {i 2 t $::def} {{} {} t def}} + " { lappend y $c } + if {$x != $y} { error "changeset no good" } + } +} + +catch {db close} +catch {db2 close} +forcedelete test.db +sqlite3 db test.db +set abc [string repeat abc 256] +set def [string repeat def 256] +execsql " + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t2 VALUES(1, '$abc'); +" +set changeset [changeset_from_sql " + INSERT INTO t2 VALUES(2, '$def'); + DELETE FROM t2 WHERE a = 1; +"] +db close + +do_faultsim_test 5.3 -faults oom-tra* -body { + set ::inverse [sqlite3changeset_invert $::changeset] + set {} {} +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + set x [list] + sqlite3session_foreach c $::inverse { lappend x $c } + foreach c " + {INSERT t2 0 X. {} {i 1 t $::abc}} + {DELETE t2 0 X. {i 2 t $::def} {}} + " { lappend y $c } + if {$x != $y} { error "changeset no good" } + } +} + +#------------------------------------------------------------------------- +# Test that OOM errors in sqlite3changeset_concat() are handled correctly. +# +catch {db close} +forcedelete test.db +sqlite3 db test.db +do_execsql_test 5.prep1 { + CREATE TABLE t1(a, b, PRIMARY KEY(b)); + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('string', 1); + INSERT INTO t1 VALUES(4, 2); + INSERT INTO t1 VALUES(X'FFAAFFAAFFAA', 3); +} + +do_test 6.prep2 { + sqlite3session M db main + M attach * + set ::c2 [changeset_from_sql { + INSERT INTO t2 VALUES(randomblob(1000), randomblob(1000)); + INSERT INTO t2 VALUES('one', 'two'); + INSERT INTO t2 VALUES(1, NULL); + UPDATE t1 SET a = 5 WHERE a = 2; + }] + set ::c1 [changeset_from_sql { + DELETE FROM t2 WHERE a = 1; + UPDATE t1 SET a = 4 WHERE a = 2; + INSERT INTO t2 VALUES('x', 'y'); + }] + set ::total [changeset_to_list [M changeset]] + M delete +} {} + +do_faultsim_test 6 -faults oom-* -body { + set ::result [sqlite3changeset_concat $::c1 $::c2] + set {} {} +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + set v [changeset_to_list $::result] + if {$v != $::total} { error "result no good" } + } +} + +faultsim_delete_and_reopen +do_execsql_test 7.prep1 { + CREATE TABLE t1(a, b, PRIMARY KEY(a)); +} +faultsim_save_and_close + +set res [list] +for {set ::i 0} {$::i < 480} {incr ::i 4} { + lappend res "INSERT t1 0 X. {} {i $::i i $::i}" +} +set res [lsort $res] +do_faultsim_test 7 -faults oom-transient -prep { + catch { S delete } + faultsim_restore_and_reopen + sqlite3session S db main + S attach * +} -body { + for {set ::i 0} {$::i < 480} {incr ::i 4} { + execsql {INSERT INTO t1 VALUES($::i, $::i)} + } +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + set cres [list [catch {changeset_to_list [S changeset]} msg] $msg] + S delete + if {$cres != "1 SQLITE_NOMEM" && $cres != "0 {$::res}"} { + error "Expected {0 $::res} Got {$cres}" + } + } else { + catch { S changeset } + catch { S delete } + } +} + +faultsim_delete_and_reopen +do_test 8.prep { + sqlite3session S db main + S attach * + execsql { + CREATE TABLE t1(a, b, PRIMARY KEY(a)); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + } + set ::changeset [S changeset] + S delete +} {} + +set expected [normalize_list { + {INSERT t1 0 X. {} {i 1 i 2}} + {INSERT t1 0 X. {} {i 3 i 4}} + {INSERT t1 0 X. {} {i 5 i 6}} +}] +do_faultsim_test 8.1 -faults oom* -body { + set ::res [list] + sqlite3session_foreach -next v $::changeset { lappend ::res $v } + normalize_list $::res +} -test { + faultsim_test_result [list 0 $::expected] {1 SQLITE_NOMEM} +} +do_faultsim_test 8.2 -faults oom* -body { + set ::res [list] + sqlite3session_foreach v $::changeset { lappend ::res $v } + normalize_list $::res +} -test { + faultsim_test_result [list 0 $::expected] {1 SQLITE_NOMEM} +} + +faultsim_delete_and_reopen +do_test 9.1.prep { + execsql { + PRAGMA encoding = 'utf16'; + CREATE TABLE t1(a PRIMARY KEY, b); + } +} {} +faultsim_save_and_close + +set answers [list {0 {}} {1 SQLITE_NOMEM} \ + {1 {callback requested query abort}} \ + {1 {abort due to ROLLBACK}}] +do_faultsim_test 9.1 -faults oom-transient -prep { + catch { unset ::c } + faultsim_restore_and_reopen + sqlite3session S db main + S attach * +} -body { + execsql { + INSERT INTO t1 VALUES('abcdefghijklmnopqrstuv', 'ABCDEFGHIJKLMNOPQRSTUV'); + } + set ::c [S changeset] + set {} {} +} -test { + S delete + eval faultsim_test_result $::answers + if {[info exists ::c]} { + set expected [normalize_list { + {INSERT t1 0 X. {} {t abcdefghijklmnopqrstuv t ABCDEFGHIJKLMNOPQRSTUV}} + }] + if { [changeset_to_list $::c] != $expected } { + error "changeset mismatch" + } + } +} + +faultsim_delete_and_reopen +do_test 9.2.prep { + execsql { + PRAGMA encoding = 'utf16'; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('abcdefghij', 'ABCDEFGHIJKLMNOPQRSTUV'); + } +} {} +faultsim_save_and_close + +set answers [list {0 {}} {1 SQLITE_NOMEM} \ + {1 {callback requested query abort}} \ + {1 {abort due to ROLLBACK}}] +do_faultsim_test 9.2 -faults oom-transient -prep { + catch { unset ::c } + faultsim_restore_and_reopen + sqlite3session S db main + S attach * +} -body { + execsql { + UPDATE t1 SET b = 'xyz'; + } + set ::c [S changeset] + set {} {} +} -test { + S delete + eval faultsim_test_result $::answers + if {[info exists ::c]} { + set expected [normalize_list { + {UPDATE t1 0 X. {t abcdefghij t ABCDEFGHIJKLMNOPQRSTUV} {{} {} t xyz}} + }] + if { [changeset_to_list $::c] != $expected } { + error "changeset mismatch" + } + } +} + +#------------------------------------------------------------------------- +# Test that if a conflict-handler encounters an OOM in +# sqlite3_value_text() but goes on to return SQLITE_CHANGESET_REPLACE +# anyway, the OOM is picked up by the sessions module. +set bigstr [string repeat abcdefghij 100] +faultsim_delete_and_reopen +do_test 10.prep.1 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES($bigstr, $bigstr); + } + + sqlite3session S db main + S attach * + execsql { UPDATE t1 SET b = b||'x' } + set C [S changeset] + S delete + execsql { UPDATE t1 SET b = b||'xyz' } +} {} +faultsim_save_and_close + +faultsim_restore_and_reopen +do_test 10.prep.2 { + proc xConflict {args} { return "ABORT" } + list [catch { sqlite3changeset_apply db $C xConflict } msg] $msg +} {1 SQLITE_ABORT} +do_execsql_test 10.prep.3 { SELECT b=$bigstr||'x' FROM t1 } 0 +do_test 10.prep.4 { + proc xConflict {args} { return "REPLACE" } + list [catch { sqlite3changeset_apply db $C xConflict } msg] $msg +} {0 {}} +do_execsql_test 10.prep.5 { SELECT b=$bigstr||'x' FROM t1 } 1 +db close + +do_faultsim_test 10 -faults oom-tra* -prep { + faultsim_restore_and_reopen +} -body { + sqlite3changeset_apply_replace_all db $::C +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + if {[db one {SELECT b=$bigstr||'x' FROM t1}]==0} { + error "data does not look right" + } + } +} + +#------------------------------------------------------------------------- +# Test an OOM with an sqlite3changeset_apply() filter callback. +# +reset_db +do_test 11.prep { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(x PRIMARY KEY, y); + BEGIN; + } + + set ::cs [changeset_from_sql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES('x', 'y'); + }] + + execsql ROLLBACK + set {} {} +} {} + +proc filter {x} { return [string equal t1 $x] } +faultsim_save_and_close + +do_faultsim_test 11 -faults oom-tra* -prep { + faultsim_restore_and_reopen +} -body { + sqlite3changeset_apply db $::cs {} filter +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + if {$testrc==0} { + if {[db eval {SELECT * FROM t1 UNION ALL SELECT * FROM t2}] != "1 2"} { + error "data does not look right" + } + } +} + + +finish_test diff --git a/ext/session/sessionfault2.test b/ext/session/sessionfault2.test new file mode 100644 index 00000000..ffdc57b9 --- /dev/null +++ b/ext/session/sessionfault2.test @@ -0,0 +1,107 @@ +# 2016 March 31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionfault2 + +do_execsql_test 1.0.0 { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + + CREATE TABLE t2(a PRIMARY KEY, b UNIQUE); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t2 VALUES(3, 3); +} +faultsim_save_and_close + +faultsim_restore_and_reopen +do_test 1.0.1 { + set ::C [changeset_from_sql { + UPDATE t1 SET b=4 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=1; + UPDATE t2 SET b=0 WHERE a=1; + UPDATE t2 SET b=1 WHERE a=2; + UPDATE t2 SET b=2 WHERE a=3; + }] + set {} {} +} {} + +proc xConflict args { return "OMIT" } + +do_faultsim_test 1 -faults oom-p* -prep { + faultsim_restore_and_reopen +} -body { + sqlite3changeset_apply db $::C xConflict +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + faultsim_integrity_check + + catch { db eval ROLLBACK } + set res [db eval { + SELECT * FROM t1; + SELECT * FROM t2; + }] + + if {$testrc==0} { + if {$res != "1 2 2 3 3 4 1 0 2 1 3 2"} { error "data error" } + } else { + if { + $res != "1 2 2 3 3 4 1 0 2 1 3 2" + && $res != "1 1 2 2 3 3 1 1 2 2 3 3" + } { error "data error!! $res" } + } +} + +#------------------------------------------------------------------------- +# OOM when applying a changeset for which one of the tables has a name +# 99 bytes in size. This happens to cause an extra malloc in within the +# sessions_strm permutation. +# +reset_db +set nm [string repeat t 99] +do_execsql_test 2.0.0 [string map "%TBL% $nm" { + CREATE TABLE %TBL%(a PRIMARY KEY, b UNIQUE); +}] +faultsim_save_and_close + +faultsim_restore_and_reopen +do_test 1.0.1 { + set ::C [changeset_from_sql [string map "%TBL% $nm" { + INSERT INTO %TBL% VALUES(1, 2); + INSERT INTO %TBL% VALUES(3, 4); + }]] + set {} {} +} {} + +proc xConflict args { return "OMIT" } +do_faultsim_test 2 -faults oom-p* -prep { + faultsim_restore_and_reopen +} -body { + sqlite3changeset_apply db $::C xConflict +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} + faultsim_integrity_check +} + +finish_test + diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c new file mode 100644 index 00000000..70ca840d --- /dev/null +++ b/ext/session/sqlite3session.c @@ -0,0 +1,4641 @@ + +#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) +#include "sqlite3session.h" +#include +#include + +#ifndef SQLITE_AMALGAMATION +# include "sqliteInt.h" +# include "vdbeInt.h" +#endif + +typedef struct SessionTable SessionTable; +typedef struct SessionChange SessionChange; +typedef struct SessionBuffer SessionBuffer; +typedef struct SessionInput SessionInput; + +/* +** Minimum chunk size used by streaming versions of functions. +*/ +#ifndef SESSIONS_STRM_CHUNK_SIZE +# ifdef SQLITE_TEST +# define SESSIONS_STRM_CHUNK_SIZE 64 +# else +# define SESSIONS_STRM_CHUNK_SIZE 1024 +# endif +#endif + +typedef struct SessionHook SessionHook; +struct SessionHook { + void *pCtx; + int (*xOld)(void*,int,sqlite3_value**); + int (*xNew)(void*,int,sqlite3_value**); + int (*xCount)(void*); + int (*xDepth)(void*); +}; + +/* +** Session handle structure. +*/ +struct sqlite3_session { + sqlite3 *db; /* Database handle session is attached to */ + char *zDb; /* Name of database session is attached to */ + int bEnable; /* True if currently recording */ + int bIndirect; /* True if all changes are indirect */ + int bAutoAttach; /* True to auto-attach tables */ + int rc; /* Non-zero if an error has occurred */ + void *pFilterCtx; /* First argument to pass to xTableFilter */ + int (*xTableFilter)(void *pCtx, const char *zTab); + sqlite3_session *pNext; /* Next session object on same db. */ + SessionTable *pTable; /* List of attached tables */ + SessionHook hook; /* APIs to grab new and old data with */ +}; + +/* +** Instances of this structure are used to build strings or binary records. +*/ +struct SessionBuffer { + u8 *aBuf; /* Pointer to changeset buffer */ + int nBuf; /* Size of buffer aBuf */ + int nAlloc; /* Size of allocation containing aBuf */ +}; + +/* +** An object of this type is used internally as an abstraction for +** input data. Input data may be supplied either as a single large buffer +** (e.g. sqlite3changeset_start()) or using a stream function (e.g. +** sqlite3changeset_start_strm()). +*/ +struct SessionInput { + int bNoDiscard; /* If true, discard no data */ + int iCurrent; /* Offset in aData[] of current change */ + int iNext; /* Offset in aData[] of next change */ + u8 *aData; /* Pointer to buffer containing changeset */ + int nData; /* Number of bytes in aData */ + + SessionBuffer buf; /* Current read buffer */ + int (*xInput)(void*, void*, int*); /* Input stream call (or NULL) */ + void *pIn; /* First argument to xInput */ + int bEof; /* Set to true after xInput finished */ +}; + +/* +** Structure for changeset iterators. +*/ +struct sqlite3_changeset_iter { + SessionInput in; /* Input buffer or stream */ + SessionBuffer tblhdr; /* Buffer to hold apValue/zTab/abPK/ */ + int bPatchset; /* True if this is a patchset */ + int rc; /* Iterator error code */ + sqlite3_stmt *pConflict; /* Points to conflicting row, if any */ + char *zTab; /* Current table */ + int nCol; /* Number of columns in zTab */ + int op; /* Current operation */ + int bIndirect; /* True if current change was indirect */ + u8 *abPK; /* Primary key array */ + sqlite3_value **apValue; /* old.* and new.* values */ +}; + +/* +** Each session object maintains a set of the following structures, one +** for each table the session object is monitoring. The structures are +** stored in a linked list starting at sqlite3_session.pTable. +** +** The keys of the SessionTable.aChange[] hash table are all rows that have +** been modified in any way since the session object was attached to the +** table. +** +** The data associated with each hash-table entry is a structure containing +** a subset of the initial values that the modified row contained at the +** start of the session. Or no initial values if the row was inserted. +*/ +struct SessionTable { + SessionTable *pNext; + char *zName; /* Local name of table */ + int nCol; /* Number of columns in table zName */ + const char **azCol; /* Column names */ + u8 *abPK; /* Array of primary key flags */ + int nEntry; /* Total number of entries in hash table */ + int nChange; /* Size of apChange[] array */ + SessionChange **apChange; /* Hash table buckets */ +}; + +/* +** RECORD FORMAT: +** +** The following record format is similar to (but not compatible with) that +** used in SQLite database files. This format is used as part of the +** change-set binary format, and so must be architecture independent. +** +** Unlike the SQLite database record format, each field is self-contained - +** there is no separation of header and data. Each field begins with a +** single byte describing its type, as follows: +** +** 0x00: Undefined value. +** 0x01: Integer value. +** 0x02: Real value. +** 0x03: Text value. +** 0x04: Blob value. +** 0x05: SQL NULL value. +** +** Note that the above match the definitions of SQLITE_INTEGER, SQLITE_TEXT +** and so on in sqlite3.h. For undefined and NULL values, the field consists +** only of the single type byte. For other types of values, the type byte +** is followed by: +** +** Text values: +** A varint containing the number of bytes in the value (encoded using +** UTF-8). Followed by a buffer containing the UTF-8 representation +** of the text value. There is no nul terminator. +** +** Blob values: +** A varint containing the number of bytes in the value, followed by +** a buffer containing the value itself. +** +** Integer values: +** An 8-byte big-endian integer value. +** +** Real values: +** An 8-byte big-endian IEEE 754-2008 real value. +** +** Varint values are encoded in the same way as varints in the SQLite +** record format. +** +** CHANGESET FORMAT: +** +** A changeset is a collection of DELETE, UPDATE and INSERT operations on +** one or more tables. Operations on a single table are grouped together, +** but may occur in any order (i.e. deletes, updates and inserts are all +** mixed together). +** +** Each group of changes begins with a table header: +** +** 1 byte: Constant 0x54 (capital 'T') +** Varint: Number of columns in the table. +** nCol bytes: 0x01 for PK columns, 0x00 otherwise. +** N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +** +** Followed by one or more changes to the table. +** +** 1 byte: Either SQLITE_INSERT (0x12), UPDATE (0x17) or DELETE (0x09). +** 1 byte: The "indirect-change" flag. +** old.* record: (delete and update only) +** new.* record: (insert and update only) +** +** The "old.*" and "new.*" records, if present, are N field records in the +** format described above under "RECORD FORMAT", where N is the number of +** columns in the table. The i'th field of each record is associated with +** the i'th column of the table, counting from left to right in the order +** in which columns were declared in the CREATE TABLE statement. +** +** The new.* record that is part of each INSERT change contains the values +** that make up the new row. Similarly, the old.* record that is part of each +** DELETE change contains the values that made up the row that was deleted +** from the database. In the changeset format, the records that are part +** of INSERT or DELETE changes never contain any undefined (type byte 0x00) +** fields. +** +** Within the old.* record associated with an UPDATE change, all fields +** associated with table columns that are not PRIMARY KEY columns and are +** not modified by the UPDATE change are set to "undefined". Other fields +** are set to the values that made up the row before the UPDATE that the +** change records took place. Within the new.* record, fields associated +** with table columns modified by the UPDATE change contain the new +** values. Fields associated with table columns that are not modified +** are set to "undefined". +** +** PATCHSET FORMAT: +** +** A patchset is also a collection of changes. It is similar to a changeset, +** but leaves undefined those fields that are not useful if no conflict +** resolution is required when applying the changeset. +** +** Each group of changes begins with a table header: +** +** 1 byte: Constant 0x50 (capital 'P') +** Varint: Number of columns in the table. +** nCol bytes: 0x01 for PK columns, 0x00 otherwise. +** N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +** +** Followed by one or more changes to the table. +** +** 1 byte: Either SQLITE_INSERT (0x12), UPDATE (0x17) or DELETE (0x09). +** 1 byte: The "indirect-change" flag. +** single record: (PK fields for DELETE, PK and modified fields for UPDATE, +** full record for INSERT). +** +** As in the changeset format, each field of the single record that is part +** of a patchset change is associated with the correspondingly positioned +** table column, counting from left to right within the CREATE TABLE +** statement. +** +** For a DELETE change, all fields within the record except those associated +** with PRIMARY KEY columns are set to "undefined". The PRIMARY KEY fields +** contain the values identifying the row to delete. +** +** For an UPDATE change, all fields except those associated with PRIMARY KEY +** columns and columns that are modified by the UPDATE are set to "undefined". +** PRIMARY KEY fields contain the values identifying the table row to update, +** and fields associated with modified columns contain the new column values. +** +** The records associated with INSERT changes are in the same format as for +** changesets. It is not possible for a record associated with an INSERT +** change to contain a field set to "undefined". +*/ + +/* +** For each row modified during a session, there exists a single instance of +** this structure stored in a SessionTable.aChange[] hash table. +*/ +struct SessionChange { + int op; /* One of UPDATE, DELETE, INSERT */ + int bIndirect; /* True if this change is "indirect" */ + int nRecord; /* Number of bytes in buffer aRecord[] */ + u8 *aRecord; /* Buffer containing old.* record */ + SessionChange *pNext; /* For hash-table collisions */ +}; + +/* +** Write a varint with value iVal into the buffer at aBuf. Return the +** number of bytes written. +*/ +static int sessionVarintPut(u8 *aBuf, int iVal){ + return putVarint32(aBuf, iVal); +} + +/* +** Return the number of bytes required to store value iVal as a varint. +*/ +static int sessionVarintLen(int iVal){ + return sqlite3VarintLen(iVal); +} + +/* +** Read a varint value from aBuf[] into *piVal. Return the number of +** bytes read. +*/ +static int sessionVarintGet(u8 *aBuf, int *piVal){ + return getVarint32(aBuf, *piVal); +} + +/* Load an unaligned and unsigned 32-bit integer */ +#define SESSION_UINT32(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) + +/* +** Read a 64-bit big-endian integer value from buffer aRec[]. Return +** the value read. +*/ +static sqlite3_int64 sessionGetI64(u8 *aRec){ + u64 x = SESSION_UINT32(aRec); + u32 y = SESSION_UINT32(aRec+4); + x = (x<<32) + y; + return (sqlite3_int64)x; +} + +/* +** Write a 64-bit big-endian integer value to the buffer aBuf[]. +*/ +static void sessionPutI64(u8 *aBuf, sqlite3_int64 i){ + aBuf[0] = (i>>56) & 0xFF; + aBuf[1] = (i>>48) & 0xFF; + aBuf[2] = (i>>40) & 0xFF; + aBuf[3] = (i>>32) & 0xFF; + aBuf[4] = (i>>24) & 0xFF; + aBuf[5] = (i>>16) & 0xFF; + aBuf[6] = (i>> 8) & 0xFF; + aBuf[7] = (i>> 0) & 0xFF; +} + +/* +** This function is used to serialize the contents of value pValue (see +** comment titled "RECORD FORMAT" above). +** +** If it is non-NULL, the serialized form of the value is written to +** buffer aBuf. *pnWrite is set to the number of bytes written before +** returning. Or, if aBuf is NULL, the only thing this function does is +** set *pnWrite. +** +** If no error occurs, SQLITE_OK is returned. Or, if an OOM error occurs +** within a call to sqlite3_value_text() (may fail if the db is utf-16)) +** SQLITE_NOMEM is returned. +*/ +static int sessionSerializeValue( + u8 *aBuf, /* If non-NULL, write serialized value here */ + sqlite3_value *pValue, /* Value to serialize */ + int *pnWrite /* IN/OUT: Increment by bytes written */ +){ + int nByte; /* Size of serialized value in bytes */ + + if( pValue ){ + int eType; /* Value type (SQLITE_NULL, TEXT etc.) */ + + eType = sqlite3_value_type(pValue); + if( aBuf ) aBuf[0] = eType; + + switch( eType ){ + case SQLITE_NULL: + nByte = 1; + break; + + case SQLITE_INTEGER: + case SQLITE_FLOAT: + if( aBuf ){ + /* TODO: SQLite does something special to deal with mixed-endian + ** floating point values (e.g. ARM7). This code probably should + ** too. */ + u64 i; + if( eType==SQLITE_INTEGER ){ + i = (u64)sqlite3_value_int64(pValue); + }else{ + double r; + assert( sizeof(double)==8 && sizeof(u64)==8 ); + r = sqlite3_value_double(pValue); + memcpy(&i, &r, 8); + } + sessionPutI64(&aBuf[1], i); + } + nByte = 9; + break; + + default: { + u8 *z; + int n; + int nVarint; + + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + if( eType==SQLITE_TEXT ){ + z = (u8 *)sqlite3_value_text(pValue); + }else{ + z = (u8 *)sqlite3_value_blob(pValue); + } + n = sqlite3_value_bytes(pValue); + if( z==0 && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + nVarint = sessionVarintLen(n); + + if( aBuf ){ + sessionVarintPut(&aBuf[1], n); + memcpy(&aBuf[nVarint + 1], eType==SQLITE_TEXT ? + sqlite3_value_text(pValue) : sqlite3_value_blob(pValue), n + ); + } + + nByte = 1 + nVarint + n; + break; + } + } + }else{ + nByte = 1; + if( aBuf ) aBuf[0] = '\0'; + } + + if( pnWrite ) *pnWrite += nByte; + return SQLITE_OK; +} + + +/* +** This macro is used to calculate hash key values for data structures. In +** order to use this macro, the entire data structure must be represented +** as a series of unsigned integers. In order to calculate a hash-key value +** for a data structure represented as three such integers, the macro may +** then be used as follows: +** +** int hash_key_value; +** hash_key_value = HASH_APPEND(0, ); +** hash_key_value = HASH_APPEND(hash_key_value, ); +** hash_key_value = HASH_APPEND(hash_key_value, ); +** +** In practice, the data structures this macro is used for are the primary +** key values of modified rows. +*/ +#define HASH_APPEND(hash, add) ((hash) << 3) ^ (hash) ^ (unsigned int)(add) + +/* +** Append the hash of the 64-bit integer passed as the second argument to the +** hash-key value passed as the first. Return the new hash-key value. +*/ +static unsigned int sessionHashAppendI64(unsigned int h, i64 i){ + h = HASH_APPEND(h, i & 0xFFFFFFFF); + return HASH_APPEND(h, (i>>32)&0xFFFFFFFF); +} + +/* +** Append the hash of the blob passed via the second and third arguments to +** the hash-key value passed as the first. Return the new hash-key value. +*/ +static unsigned int sessionHashAppendBlob(unsigned int h, int n, const u8 *z){ + int i; + for(i=0; inCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; + + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); + }else{ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); + } + if( rc!=SQLITE_OK ) return rc; + + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); + }else{ + assert( eType==SQLITE_NULL ); + *pbNullPK = 1; + } + } + } + + *piHash = (h % pTab->nChange); + return SQLITE_OK; +} + +/* +** The buffer that the argument points to contains a serialized SQL value. +** Return the number of bytes of space occupied by the value (including +** the type byte). +*/ +static int sessionSerialLen(u8 *a){ + int e = *a; + int n; + if( e==0 ) return 1; + if( e==SQLITE_NULL ) return 1; + if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; + return sessionVarintGet(&a[1], &n) + 1 + n; +} + +/* +** Based on the primary key values stored in change aRecord, calculate a +** hash key. Assume the has table has nBucket buckets. The hash keys +** calculated by this function are compatible with those calculated by +** sessionPreupdateHash(). +** +** The bPkOnly argument is non-zero if the record at aRecord[] is from +** a patchset DELETE. In this case the non-PK fields are omitted entirely. +*/ +static unsigned int sessionChangeHash( + SessionTable *pTab, /* Table handle */ + int bPkOnly, /* Record consists of PK fields only */ + u8 *aRecord, /* Change record */ + int nBucket /* Assume this many buckets in hash table */ +){ + unsigned int h = 0; /* Value to return */ + int i; /* Used to iterate through columns */ + u8 *a = aRecord; /* Used to iterate through change record */ + + for(i=0; inCol; i++){ + int eType = *a; + int isPK = pTab->abPK[i]; + if( bPkOnly && isPK==0 ) continue; + + /* It is not possible for eType to be SQLITE_NULL here. The session + ** module does not record changes for rows with NULL values stored in + ** primary key columns. */ + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_TEXT || eType==SQLITE_BLOB + || eType==SQLITE_NULL || eType==0 + ); + assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); + + if( isPK ){ + a++; + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + h = sessionHashAppendI64(h, sessionGetI64(a)); + a += 8; + }else{ + int n; + a += sessionVarintGet(a, &n); + h = sessionHashAppendBlob(h, n, a); + a += n; + } + }else{ + a += sessionSerialLen(a); + } + } + return (h % nBucket); +} + +/* +** Arguments aLeft and aRight are pointers to change records for table pTab. +** This function returns true if the two records apply to the same row (i.e. +** have the same values stored in the primary key columns), or false +** otherwise. +*/ +static int sessionChangeEqual( + SessionTable *pTab, /* Table used for PK definition */ + int bLeftPkOnly, /* True if aLeft[] contains PK fields only */ + u8 *aLeft, /* Change record */ + int bRightPkOnly, /* True if aRight[] contains PK fields only */ + u8 *aRight /* Change record */ +){ + u8 *a1 = aLeft; /* Cursor to iterate through aLeft */ + u8 *a2 = aRight; /* Cursor to iterate through aRight */ + int iCol; /* Used to iterate through table columns */ + + for(iCol=0; iColnCol; iCol++){ + if( pTab->abPK[iCol] ){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); + + if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ + return 0; + } + a1 += n1; + a2 += n2; + }else{ + if( bLeftPkOnly==0 ) a1 += sessionSerialLen(a1); + if( bRightPkOnly==0 ) a2 += sessionSerialLen(a2); + } + } + + return 1; +} + +/* +** Arguments aLeft and aRight both point to buffers containing change +** records with nCol columns. This function "merges" the two records into +** a single records which is written to the buffer at *paOut. *paOut is +** then set to point to one byte after the last byte written before +** returning. +** +** The merging of records is done as follows: For each column, if the +** aRight record contains a value for the column, copy the value from +** their. Otherwise, if aLeft contains a value, copy it. If neither +** record contains a value for a given column, then neither does the +** output record. +*/ +static void sessionMergeRecord( + u8 **paOut, + int nCol, + u8 *aLeft, + u8 *aRight +){ + u8 *a1 = aLeft; /* Cursor used to iterate through aLeft */ + u8 *a2 = aRight; /* Cursor used to iterate through aRight */ + u8 *aOut = *paOut; /* Output cursor */ + int iCol; /* Used to iterate from 0 to nCol */ + + for(iCol=0; iColnCol; i++){ + int nOld; + u8 *aOld; + int nNew; + u8 *aNew; + + aOld = sessionMergeValue(&aOld1, &aOld2, &nOld); + aNew = sessionMergeValue(&aNew1, &aNew2, &nNew); + if( pTab->abPK[i] || nOld!=nNew || memcmp(aOld, aNew, nNew) ){ + if( pTab->abPK[i]==0 ) bRequired = 1; + memcpy(aOut, aOld, nOld); + aOut += nOld; + }else{ + *(aOut++) = '\0'; + } + } + + if( !bRequired ) return 0; + } + + /* Write the new.* vector */ + aOld1 = aOldRecord1; + aOld2 = aOldRecord2; + aNew1 = aNewRecord1; + aNew2 = aNewRecord2; + for(i=0; inCol; i++){ + int nOld; + u8 *aOld; + int nNew; + u8 *aNew; + + aOld = sessionMergeValue(&aOld1, &aOld2, &nOld); + aNew = sessionMergeValue(&aNew1, &aNew2, &nNew); + if( bPatchset==0 + && (pTab->abPK[i] || (nOld==nNew && 0==memcmp(aOld, aNew, nNew))) + ){ + *(aOut++) = '\0'; + }else{ + memcpy(aOut, aNew, nNew); + aOut += nNew; + } + } + + *paOut = aOut; + return 1; +} + +/* +** This function is only called from within a pre-update-hook callback. +** It determines if the current pre-update-hook change affects the same row +** as the change stored in argument pChange. If so, it returns true. Otherwise +** if the pre-update-hook does not affect the same row as pChange, it returns +** false. +*/ +static int sessionPreupdateEqual( + sqlite3_session *pSession, /* Session object that owns SessionTable */ + SessionTable *pTab, /* Table associated with change */ + SessionChange *pChange, /* Change to compare to */ + int op /* Current pre-update operation */ +){ + int iCol; /* Used to iterate through columns */ + u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); + for(iCol=0; iColnCol; iCol++){ + if( !pTab->abPK[iCol] ){ + a += sessionSerialLen(a); + }else{ + sqlite3_value *pVal; /* Value returned by preupdate_new/old */ + int rc; /* Error code from preupdate_new/old */ + int eType = *a++; /* Type of value from change record */ + + /* The following calls to preupdate_new() and preupdate_old() can not + ** fail. This is because they cache their return values, and by the + ** time control flows to here they have already been called once from + ** within sessionPreupdateHash(). The first two asserts below verify + ** this (that the method has already been called). */ + if( op==SQLITE_INSERT ){ + /* assert( db->pPreUpdate->pNewUnpacked || db->pPreUpdate->aNew ); */ + rc = pSession->hook.xNew(pSession->hook.pCtx, iCol, &pVal); + }else{ + /* assert( db->pPreUpdate->pUnpacked ); */ + rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); + } + assert( rc==SQLITE_OK ); + if( sqlite3_value_type(pVal)!=eType ) return 0; + + /* A SessionChange object never has a NULL value in a PK column */ + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_BLOB || eType==SQLITE_TEXT + ); + + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal = sessionGetI64(a); + a += 8; + if( eType==SQLITE_INTEGER ){ + if( sqlite3_value_int64(pVal)!=iVal ) return 0; + }else{ + double rVal; + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&rVal, &iVal, 8); + if( sqlite3_value_double(pVal)!=rVal ) return 0; + } + }else{ + int n; + const u8 *z; + a += sessionVarintGet(a, &n); + if( sqlite3_value_bytes(pVal)!=n ) return 0; + if( eType==SQLITE_TEXT ){ + z = sqlite3_value_text(pVal); + }else{ + z = sqlite3_value_blob(pVal); + } + if( memcmp(a, z, n) ) return 0; + a += n; + break; + } + } + } + + return 1; +} + +/* +** If required, grow the hash table used to store changes on table pTab +** (part of the session pSession). If a fatal OOM error occurs, set the +** session object to failed and return SQLITE_ERROR. Otherwise, return +** SQLITE_OK. +** +** It is possible that a non-fatal OOM error occurs in this function. In +** that case the hash-table does not grow, but SQLITE_OK is returned anyway. +** Growing the hash table in this case is a performance optimization only, +** it is not required for correct operation. +*/ +static int sessionGrowHash(int bPatchset, SessionTable *pTab){ + if( pTab->nChange==0 || pTab->nEntry>=(pTab->nChange/2) ){ + int i; + SessionChange **apNew; + int nNew = (pTab->nChange ? pTab->nChange : 128) * 2; + + apNew = (SessionChange **)sqlite3_malloc(sizeof(SessionChange *) * nNew); + if( apNew==0 ){ + if( pTab->nChange==0 ){ + return SQLITE_ERROR; + } + return SQLITE_OK; + } + memset(apNew, 0, sizeof(SessionChange *) * nNew); + + for(i=0; inChange; i++){ + SessionChange *p; + SessionChange *pNext; + for(p=pTab->apChange[i]; p; p=pNext){ + int bPkOnly = (p->op==SQLITE_DELETE && bPatchset); + int iHash = sessionChangeHash(pTab, bPkOnly, p->aRecord, nNew); + pNext = p->pNext; + p->pNext = apNew[iHash]; + apNew[iHash] = p; + } + } + + sqlite3_free(pTab->apChange); + pTab->nChange = nNew; + pTab->apChange = apNew; + } + + return SQLITE_OK; +} + +/* +** This function queries the database for the names of the columns of table +** zThis, in schema zDb. It is expected that the table has nCol columns. If +** not, SQLITE_SCHEMA is returned and none of the output variables are +** populated. +** +** Otherwise, if they are not NULL, variable *pnCol is set to the number +** of columns in the database table and variable *pzTab is set to point to a +** nul-terminated copy of the table name. *pazCol (if not NULL) is set to +** point to an array of pointers to column names. And *pabPK (again, if not +** NULL) is set to point to an array of booleans - true if the corresponding +** column is part of the primary key. +** +** For example, if the table is declared as: +** +** CREATE TABLE tbl1(w, x, y, z, PRIMARY KEY(w, z)); +** +** Then the four output variables are populated as follows: +** +** *pnCol = 4 +** *pzTab = "tbl1" +** *pazCol = {"w", "x", "y", "z"} +** *pabPK = {1, 0, 0, 1} +** +** All returned buffers are part of the same single allocation, which must +** be freed using sqlite3_free() by the caller. If pazCol was not NULL, then +** pointer *pazCol should be freed to release all memory. Otherwise, pointer +** *pabPK. It is illegal for both pazCol and pabPK to be NULL. +*/ +static int sessionTableInfo( + sqlite3 *db, /* Database connection */ + const char *zDb, /* Name of attached database (e.g. "main") */ + const char *zThis, /* Table name */ + int *pnCol, /* OUT: number of columns */ + const char **pzTab, /* OUT: Copy of zThis */ + const char ***pazCol, /* OUT: Array of column names for table */ + u8 **pabPK /* OUT: Array of booleans - true for PK col */ +){ + char *zPragma; + sqlite3_stmt *pStmt; + int rc; + int nByte; + int nDbCol = 0; + int nThis; + int i; + u8 *pAlloc = 0; + char **azCol = 0; + u8 *abPK = 0; + + assert( pazCol && pabPK ); + + nThis = sqlite3Strlen30(zThis); + zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); + if( !zPragma ) return SQLITE_NOMEM; + + rc = sqlite3_prepare_v2(db, zPragma, -1, &pStmt, 0); + sqlite3_free(zPragma); + if( rc!=SQLITE_OK ) return rc; + + nByte = nThis + 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nByte += sqlite3_column_bytes(pStmt, 1); + nDbCol++; + } + rc = sqlite3_reset(pStmt); + + if( rc==SQLITE_OK ){ + nByte += nDbCol * (sizeof(const char *) + sizeof(u8) + 1); + pAlloc = sqlite3_malloc(nByte); + if( pAlloc==0 ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + azCol = (char **)pAlloc; + pAlloc = (u8 *)&azCol[nDbCol]; + abPK = (u8 *)pAlloc; + pAlloc = &abPK[nDbCol]; + if( pzTab ){ + memcpy(pAlloc, zThis, nThis+1); + *pzTab = (char *)pAlloc; + pAlloc += nThis+1; + } + + i = 0; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nName = sqlite3_column_bytes(pStmt, 1); + const unsigned char *zName = sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; + memcpy(pAlloc, zName, nName+1); + azCol[i] = (char *)pAlloc; + pAlloc += nName+1; + abPK[i] = sqlite3_column_int(pStmt, 5); + i++; + } + rc = sqlite3_reset(pStmt); + + } + + /* If successful, populate the output variables. Otherwise, zero them and + ** free any allocation made. An error code will be returned in this case. + */ + if( rc==SQLITE_OK ){ + *pazCol = (const char **)azCol; + *pabPK = abPK; + *pnCol = nDbCol; + }else{ + *pazCol = 0; + *pabPK = 0; + *pnCol = 0; + if( pzTab ) *pzTab = 0; + sqlite3_free(azCol); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** This function is only called from within a pre-update handler for a +** write to table pTab, part of session pSession. If this is the first +** write to this table, initalize the SessionTable.nCol, azCol[] and +** abPK[] arrays accordingly. +** +** If an error occurs, an error code is stored in sqlite3_session.rc and +** non-zero returned. Or, if no error occurs but the table has no primary +** key, sqlite3_session.rc is left set to SQLITE_OK and non-zero returned to +** indicate that updates on this table should be ignored. SessionTable.abPK +** is set to NULL in this case. +*/ +static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ + if( pTab->nCol==0 ){ + u8 *abPK; + assert( pTab->azCol==0 || pTab->abPK==0 ); + pSession->rc = sessionTableInfo(pSession->db, pSession->zDb, + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + ); + if( pSession->rc==SQLITE_OK ){ + int i; + for(i=0; inCol; i++){ + if( abPK[i] ){ + pTab->abPK = abPK; + break; + } + } + } + } + return (pSession->rc || pTab->abPK==0); +} + +/* +** This function is only called from with a pre-update-hook reporting a +** change on table pTab (attached to session pSession). The type of change +** (UPDATE, INSERT, DELETE) is specified by the first argument. +** +** Unless one is already present or an error occurs, an entry is added +** to the changed-rows hash table associated with table pTab. +*/ +static void sessionPreupdateOneChange( + int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + sqlite3_session *pSession, /* Session object pTab is attached to */ + SessionTable *pTab /* Table that change applies to */ +){ + int iHash; + int bNull = 0; + int rc = SQLITE_OK; + + if( pSession->rc ) return; + + /* Load table details if required */ + if( sessionInitTable(pSession, pTab) ) return; + + /* Check the number of columns in this xPreUpdate call matches the + ** number of columns in the table. */ + if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + pSession->rc = SQLITE_SCHEMA; + return; + } + + /* Grow the hash table if required */ + if( sessionGrowHash(0, pTab) ){ + pSession->rc = SQLITE_NOMEM; + return; + } + + /* Calculate the hash-key for this change. If the primary key of the row + ** includes a NULL value, exit early. Such changes are ignored by the + ** session module. */ + rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + if( rc!=SQLITE_OK ) goto error_out; + + if( bNull==0 ){ + /* Search the hash table for an existing record for this row. */ + SessionChange *pC; + for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ + if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + } + + if( pC==0 ){ + /* Create a new change object containing all the old values (if + ** this is an SQLITE_UPDATE or SQLITE_DELETE), or just the PK + ** values (if this is an INSERT). */ + SessionChange *pChange; /* New change object */ + int nByte; /* Number of bytes to allocate */ + int i; /* Used to iterate through columns */ + + assert( rc==SQLITE_OK ); + pTab->nEntry++; + + /* Figure out how large an allocation is required */ + nByte = sizeof(SessionChange); + for(i=0; inCol; i++){ + sqlite3_value *p = 0; + if( op!=SQLITE_INSERT ){ + TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); + assert( trc==SQLITE_OK ); + }else if( pTab->abPK[i] ){ + TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p); + assert( trc==SQLITE_OK ); + } + + /* This may fail if SQLite value p contains a utf-16 string that must + ** be converted to utf-8 and an OOM error occurs while doing so. */ + rc = sessionSerializeValue(0, p, &nByte); + if( rc!=SQLITE_OK ) goto error_out; + } + + /* Allocate the change object */ + pChange = (SessionChange *)sqlite3_malloc(nByte); + if( !pChange ){ + rc = SQLITE_NOMEM; + goto error_out; + }else{ + memset(pChange, 0, sizeof(SessionChange)); + pChange->aRecord = (u8 *)&pChange[1]; + } + + /* Populate the change object. None of the preupdate_old(), + ** preupdate_new() or SerializeValue() calls below may fail as all + ** required values and encodings have already been cached in memory. + ** It is not possible for an OOM to occur in this block. */ + nByte = 0; + for(i=0; inCol; i++){ + sqlite3_value *p = 0; + if( op!=SQLITE_INSERT ){ + pSession->hook.xOld(pSession->hook.pCtx, i, &p); + }else if( pTab->abPK[i] ){ + pSession->hook.xNew(pSession->hook.pCtx, i, &p); + } + sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte); + } + + /* Add the change to the hash-table */ + if( pSession->bIndirect || pSession->hook.xDepth(pSession->hook.pCtx) ){ + pChange->bIndirect = 1; + } + pChange->nRecord = nByte; + pChange->op = op; + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + + }else if( pC->bIndirect ){ + /* If the existing change is considered "indirect", but this current + ** change is "direct", mark the change object as direct. */ + if( pSession->hook.xDepth(pSession->hook.pCtx)==0 + && pSession->bIndirect==0 + ){ + pC->bIndirect = 0; + } + } + } + + /* If an error has occurred, mark the session object as failed. */ + error_out: + if( rc!=SQLITE_OK ){ + pSession->rc = rc; + } +} + +static int sessionFindTable( + sqlite3_session *pSession, + const char *zName, + SessionTable **ppTab +){ + int rc = SQLITE_OK; + int nName = sqlite3Strlen30(zName); + SessionTable *pRet; + + /* Search for an existing table */ + for(pRet=pSession->pTable; pRet; pRet=pRet->pNext){ + if( 0==sqlite3_strnicmp(pRet->zName, zName, nName+1) ) break; + } + + if( pRet==0 && pSession->bAutoAttach ){ + /* If there is a table-filter configured, invoke it. If it returns 0, + ** do not automatically add the new table. */ + if( pSession->xTableFilter==0 + || pSession->xTableFilter(pSession->pFilterCtx, zName) + ){ + rc = sqlite3session_attach(pSession, zName); + if( rc==SQLITE_OK ){ + for(pRet=pSession->pTable; pRet->pNext; pRet=pRet->pNext); + assert( 0==sqlite3_strnicmp(pRet->zName, zName, nName+1) ); + } + } + } + + assert( rc==SQLITE_OK || pRet==0 ); + *ppTab = pRet; + return rc; +} + +/* +** The 'pre-update' hook registered by this module with SQLite databases. +*/ +static void xPreUpdate( + void *pCtx, /* Copy of third arg to preupdate_hook() */ + sqlite3 *db, /* Database handle */ + int op, /* SQLITE_UPDATE, DELETE or INSERT */ + char const *zDb, /* Database name */ + char const *zName, /* Table name */ + sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ + sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ +){ + sqlite3_session *pSession; + int nDb = sqlite3Strlen30(zDb); + + assert( sqlite3_mutex_held(db->mutex) ); + + for(pSession=(sqlite3_session *)pCtx; pSession; pSession=pSession->pNext){ + SessionTable *pTab; + + /* If this session is attached to a different database ("main", "temp" + ** etc.), or if it is not currently enabled, there is nothing to do. Skip + ** to the next session object attached to this database. */ + if( pSession->bEnable==0 ) continue; + if( pSession->rc ) continue; + if( sqlite3_strnicmp(zDb, pSession->zDb, nDb+1) ) continue; + + pSession->rc = sessionFindTable(pSession, zName, &pTab); + if( pTab ){ + assert( pSession->rc==SQLITE_OK ); + sessionPreupdateOneChange(op, pSession, pTab); + if( op==SQLITE_UPDATE ){ + sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + } + } + } +} + +/* +** The pre-update hook implementations. +*/ +static int sessionPreupdateOld(void *pCtx, int iVal, sqlite3_value **ppVal){ + return sqlite3_preupdate_old((sqlite3*)pCtx, iVal, ppVal); +} +static int sessionPreupdateNew(void *pCtx, int iVal, sqlite3_value **ppVal){ + return sqlite3_preupdate_new((sqlite3*)pCtx, iVal, ppVal); +} +static int sessionPreupdateCount(void *pCtx){ + return sqlite3_preupdate_count((sqlite3*)pCtx); +} +static int sessionPreupdateDepth(void *pCtx){ + return sqlite3_preupdate_depth((sqlite3*)pCtx); +} + +/* +** Install the pre-update hooks on the session object passed as the only +** argument. +*/ +static void sessionPreupdateHooks( + sqlite3_session *pSession +){ + pSession->hook.pCtx = (void*)pSession->db; + pSession->hook.xOld = sessionPreupdateOld; + pSession->hook.xNew = sessionPreupdateNew; + pSession->hook.xCount = sessionPreupdateCount; + pSession->hook.xDepth = sessionPreupdateDepth; +} + +typedef struct SessionDiffCtx SessionDiffCtx; +struct SessionDiffCtx { + sqlite3_stmt *pStmt; + int nOldOff; +}; + +/* +** The diff hook implementations. +*/ +static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ + SessionDiffCtx *p = (SessionDiffCtx*)pCtx; + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + return SQLITE_OK; +} +static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ + SessionDiffCtx *p = (SessionDiffCtx*)pCtx; + *ppVal = sqlite3_column_value(p->pStmt, iVal); + return SQLITE_OK; +} +static int sessionDiffCount(void *pCtx){ + SessionDiffCtx *p = (SessionDiffCtx*)pCtx; + return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); +} +static int sessionDiffDepth(void *pCtx){ + return 0; +} + +/* +** Install the diff hooks on the session object passed as the only +** argument. +*/ +static void sessionDiffHooks( + sqlite3_session *pSession, + SessionDiffCtx *pDiffCtx +){ + pSession->hook.pCtx = (void*)pDiffCtx; + pSession->hook.xOld = sessionDiffOld; + pSession->hook.xNew = sessionDiffNew; + pSession->hook.xCount = sessionDiffCount; + pSession->hook.xDepth = sessionDiffDepth; +} + +static char *sessionExprComparePK( + int nCol, + const char *zDb1, const char *zDb2, + const char *zTab, + const char **azCol, u8 *abPK +){ + int i; + const char *zSep = ""; + char *zRet = 0; + + for(i=0; inCol, zDb1, zDb2, pTab->zName,zExpr); + + if( zStmt==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare(pSession->db, zStmt, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; + pDiffCtx->pStmt = pStmt; + pDiffCtx->nOldOff = 0; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + sessionPreupdateOneChange(op, pSession, pTab); + } + rc = sqlite3_finalize(pStmt); + } + sqlite3_free(zStmt); + } + + return rc; +} + +static int sessionDiffFindModified( + sqlite3_session *pSession, + SessionTable *pTab, + const char *zFrom, + const char *zExpr +){ + int rc = SQLITE_OK; + + char *zExpr2 = sessionExprCompareOther(pTab->nCol, + pSession->zDb, zFrom, pTab->zName, pTab->azCol, pTab->abPK + ); + if( zExpr2==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zStmt = sqlite3_mprintf( + "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + ); + if( zStmt==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare(pSession->db, zStmt, -1, &pStmt, 0); + + if( rc==SQLITE_OK ){ + SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; + pDiffCtx->pStmt = pStmt; + pDiffCtx->nOldOff = pTab->nCol; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + } + rc = sqlite3_finalize(pStmt); + } + sqlite3_free(zStmt); + } + } + + return rc; +} + +int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFrom, + const char *zTbl, + char **pzErrMsg +){ + const char *zDb = pSession->zDb; + int rc = pSession->rc; + SessionDiffCtx d; + + memset(&d, 0, sizeof(d)); + sessionDiffHooks(pSession, &d); + + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + if( pzErrMsg ) *pzErrMsg = 0; + if( rc==SQLITE_OK ){ + char *zExpr = 0; + sqlite3 *db = pSession->db; + SessionTable *pTo; /* Table zTbl */ + + /* Locate and if necessary initialize the target table object */ + rc = sessionFindTable(pSession, zTbl, &pTo); + if( pTo==0 ) goto diff_out; + if( sessionInitTable(pSession, pTo) ){ + rc = pSession->rc; + goto diff_out; + } + + /* Check the table schemas match */ + if( rc==SQLITE_OK ){ + int bHasPk = 0; + int bMismatch = 0; + int nCol; /* Columns in zFrom.zTbl */ + u8 *abPK; + const char **azCol = 0; + rc = sessionTableInfo(db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + if( rc==SQLITE_OK ){ + if( pTo->nCol!=nCol ){ + bMismatch = 1; + }else{ + int i; + for(i=0; iabPK[i]!=abPK[i] ) bMismatch = 1; + if( sqlite3_stricmp(azCol[i], pTo->azCol[i]) ) bMismatch = 1; + if( abPK[i] ) bHasPk = 1; + } + } + + } + sqlite3_free((char*)azCol); + if( bMismatch ){ + *pzErrMsg = sqlite3_mprintf("table schemas do not match"); + rc = SQLITE_SCHEMA; + } + if( bHasPk==0 ){ + /* Ignore tables with no primary keys */ + goto diff_out; + } + } + + if( rc==SQLITE_OK ){ + zExpr = sessionExprComparePK(pTo->nCol, + zDb, zFrom, pTo->zName, pTo->azCol, pTo->abPK + ); + } + + /* Find new rows */ + if( rc==SQLITE_OK ){ + rc = sessionDiffFindNew(SQLITE_INSERT, pSession, pTo, zDb, zFrom, zExpr); + } + + /* Find old rows */ + if( rc==SQLITE_OK ){ + rc = sessionDiffFindNew(SQLITE_DELETE, pSession, pTo, zFrom, zDb, zExpr); + } + + /* Find modified rows */ + if( rc==SQLITE_OK ){ + rc = sessionDiffFindModified(pSession, pTo, zFrom, zExpr); + } + + sqlite3_free(zExpr); + } + + diff_out: + sessionPreupdateHooks(pSession); + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return rc; +} + +/* +** Create a session object. This session object will record changes to +** database zDb attached to connection db. +*/ +int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +){ + sqlite3_session *pNew; /* Newly allocated session object */ + sqlite3_session *pOld; /* Session object already attached to db */ + int nDb = sqlite3Strlen30(zDb); /* Length of zDb in bytes */ + + /* Zero the output value in case an error occurs. */ + *ppSession = 0; + + /* Allocate and populate the new session object. */ + pNew = (sqlite3_session *)sqlite3_malloc(sizeof(sqlite3_session) + nDb + 1); + if( !pNew ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(sqlite3_session)); + pNew->db = db; + pNew->zDb = (char *)&pNew[1]; + pNew->bEnable = 1; + memcpy(pNew->zDb, zDb, nDb+1); + sessionPreupdateHooks(pNew); + + /* Add the new session object to the linked list of session objects + ** attached to database handle $db. Do this under the cover of the db + ** handle mutex. */ + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + pOld = (sqlite3_session*)sqlite3_preupdate_hook(db, xPreUpdate, (void*)pNew); + pNew->pNext = pOld; + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + + *ppSession = pNew; + return SQLITE_OK; +} + +/* +** Free the list of table objects passed as the first argument. The contents +** of the changed-rows hash tables are also deleted. +*/ +static void sessionDeleteTable(SessionTable *pList){ + SessionTable *pNext; + SessionTable *pTab; + + for(pTab=pList; pTab; pTab=pNext){ + int i; + pNext = pTab->pNext; + for(i=0; inChange; i++){ + SessionChange *p; + SessionChange *pNextChange; + for(p=pTab->apChange[i]; p; p=pNextChange){ + pNextChange = p->pNext; + sqlite3_free(p); + } + } + sqlite3_free((char*)pTab->azCol); /* cast works around VC++ bug */ + sqlite3_free(pTab->apChange); + sqlite3_free(pTab); + } +} + +/* +** Delete a session object previously allocated using sqlite3session_create(). +*/ +void sqlite3session_delete(sqlite3_session *pSession){ + sqlite3 *db = pSession->db; + sqlite3_session *pHead; + sqlite3_session **pp; + + /* Unlink the session from the linked list of sessions attached to the + ** database handle. Hold the db mutex while doing so. */ + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + pHead = (sqlite3_session*)sqlite3_preupdate_hook(db, 0, 0); + for(pp=&pHead; ALWAYS((*pp)!=0); pp=&((*pp)->pNext)){ + if( (*pp)==pSession ){ + *pp = (*pp)->pNext; + if( pHead ) sqlite3_preupdate_hook(db, xPreUpdate, (void*)pHead); + break; + } + } + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + + /* Delete all attached table objects. And the contents of their + ** associated hash-tables. */ + sessionDeleteTable(pSession->pTable); + + /* Free the session object itself. */ + sqlite3_free(pSession); +} + +/* +** Set a table filter on a Session Object. +*/ +void sqlite3session_table_filter( + sqlite3_session *pSession, + int(*xFilter)(void*, const char*), + void *pCtx /* First argument passed to xFilter */ +){ + pSession->bAutoAttach = 1; + pSession->pFilterCtx = pCtx; + pSession->xTableFilter = xFilter; +} + +/* +** Attach a table to a session. All subsequent changes made to the table +** while the session object is enabled will be recorded. +** +** Only tables that have a PRIMARY KEY defined may be attached. It does +** not matter if the PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) +** or not. +*/ +int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zName /* Table name */ +){ + int rc = SQLITE_OK; + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + + if( !zName ){ + pSession->bAutoAttach = 1; + }else{ + SessionTable *pTab; /* New table object (if required) */ + int nName; /* Number of bytes in string zName */ + + /* First search for an existing entry. If one is found, this call is + ** a no-op. Return early. */ + nName = sqlite3Strlen30(zName); + for(pTab=pSession->pTable; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zName, nName+1) ) break; + } + + if( !pTab ){ + /* Allocate new SessionTable object. */ + pTab = (SessionTable *)sqlite3_malloc(sizeof(SessionTable) + nName + 1); + if( !pTab ){ + rc = SQLITE_NOMEM; + }else{ + /* Populate the new SessionTable object and link it into the list. + ** The new object must be linked onto the end of the list, not + ** simply added to the start of it in order to ensure that tables + ** appear in the correct order when a changeset or patchset is + ** eventually generated. */ + SessionTable **ppTab; + memset(pTab, 0, sizeof(SessionTable)); + pTab->zName = (char *)&pTab[1]; + memcpy(pTab->zName, zName, nName+1); + for(ppTab=&pSession->pTable; *ppTab; ppTab=&(*ppTab)->pNext); + *ppTab = pTab; + } + } + } + + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return rc; +} + +/* +** Ensure that there is room in the buffer to append nByte bytes of data. +** If not, use sqlite3_realloc() to grow the buffer so that there is. +** +** If successful, return zero. Otherwise, if an OOM condition is encountered, +** set *pRc to SQLITE_NOMEM and return non-zero. +*/ +static int sessionBufferGrow(SessionBuffer *p, int nByte, int *pRc){ + if( *pRc==SQLITE_OK && p->nAlloc-p->nBufnAlloc ? p->nAlloc : 128; + do { + nNew = nNew*2; + }while( nNew<(p->nBuf+nByte) ); + + aNew = (u8 *)sqlite3_realloc(p->aBuf, nNew); + if( 0==aNew ){ + *pRc = SQLITE_NOMEM; + }else{ + p->aBuf = aNew; + p->nAlloc = nNew; + } + } + return (*pRc!=SQLITE_OK); +} + +/* +** Append the value passed as the second argument to the buffer passed +** as the first. +** +** This function is a no-op if *pRc is non-zero when it is called. +** Otherwise, if an error occurs, *pRc is set to an SQLite error code +** before returning. +*/ +static void sessionAppendValue(SessionBuffer *p, sqlite3_value *pVal, int *pRc){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + int nByte = 0; + rc = sessionSerializeValue(0, pVal, &nByte); + sessionBufferGrow(p, nByte, &rc); + if( rc==SQLITE_OK ){ + rc = sessionSerializeValue(&p->aBuf[p->nBuf], pVal, 0); + p->nBuf += nByte; + }else{ + *pRc = rc; + } + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a single byte to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendByte(SessionBuffer *p, u8 v, int *pRc){ + if( 0==sessionBufferGrow(p, 1, pRc) ){ + p->aBuf[p->nBuf++] = v; + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a single varint to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendVarint(SessionBuffer *p, int v, int *pRc){ + if( 0==sessionBufferGrow(p, 9, pRc) ){ + p->nBuf += sessionVarintPut(&p->aBuf[p->nBuf], v); + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a blob of data to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendBlob( + SessionBuffer *p, + const u8 *aBlob, + int nBlob, + int *pRc +){ + if( 0==sessionBufferGrow(p, nBlob, pRc) ){ + memcpy(&p->aBuf[p->nBuf], aBlob, nBlob); + p->nBuf += nBlob; + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a string to the buffer. All bytes in the string +** up to (but not including) the nul-terminator are written to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendStr( + SessionBuffer *p, + const char *zStr, + int *pRc +){ + int nStr = sqlite3Strlen30(zStr); + if( 0==sessionBufferGrow(p, nStr, pRc) ){ + memcpy(&p->aBuf[p->nBuf], zStr, nStr); + p->nBuf += nStr; + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append the string representation of integer iVal +** to the buffer. No nul-terminator is written. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendInteger( + SessionBuffer *p, /* Buffer to append to */ + int iVal, /* Value to write the string rep. of */ + int *pRc /* IN/OUT: Error code */ +){ + char aBuf[24]; + sqlite3_snprintf(sizeof(aBuf)-1, aBuf, "%d", iVal); + sessionAppendStr(p, aBuf, pRc); +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append the string zStr enclosed in quotes (") and +** with any embedded quote characters escaped to the buffer. No +** nul-terminator byte is written. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendIdent( + SessionBuffer *p, /* Buffer to a append to */ + const char *zStr, /* String to quote, escape and append */ + int *pRc /* IN/OUT: Error code */ +){ + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + if( 0==sessionBufferGrow(p, nStr, pRc) ){ + char *zOut = (char *)&p->aBuf[p->nBuf]; + const char *zIn = zStr; + *zOut++ = '"'; + while( *zIn ){ + if( *zIn=='"' ) *zOut++ = '"'; + *zOut++ = *(zIn++); + } + *zOut++ = '"'; + p->nBuf = (int)((u8 *)zOut - p->aBuf); + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwse, it appends the serialized version of the value stored +** in column iCol of the row that SQL statement pStmt currently points +** to to the buffer. +*/ +static void sessionAppendCol( + SessionBuffer *p, /* Buffer to append to */ + sqlite3_stmt *pStmt, /* Handle pointing to row containing value */ + int iCol, /* Column to read value from */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int eType = sqlite3_column_type(pStmt, iCol); + sessionAppendByte(p, (u8)eType, pRc); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + sqlite3_int64 i; + u8 aBuf[8]; + if( eType==SQLITE_INTEGER ){ + i = sqlite3_column_int64(pStmt, iCol); + }else{ + double r = sqlite3_column_double(pStmt, iCol); + memcpy(&i, &r, 8); + } + sessionPutI64(aBuf, i); + sessionAppendBlob(p, aBuf, 8, pRc); + } + if( eType==SQLITE_BLOB || eType==SQLITE_TEXT ){ + u8 *z; + int nByte; + if( eType==SQLITE_BLOB ){ + z = (u8 *)sqlite3_column_blob(pStmt, iCol); + }else{ + z = (u8 *)sqlite3_column_text(pStmt, iCol); + } + nByte = sqlite3_column_bytes(pStmt, iCol); + if( z || (eType==SQLITE_BLOB && nByte==0) ){ + sessionAppendVarint(p, nByte, pRc); + sessionAppendBlob(p, z, nByte, pRc); + }else{ + *pRc = SQLITE_NOMEM; + } + } + } +} + +/* +** +** This function appends an update change to the buffer (see the comments +** under "CHANGESET FORMAT" at the top of the file). An update change +** consists of: +** +** 1 byte: SQLITE_UPDATE (0x17) +** n bytes: old.* record (see RECORD FORMAT) +** m bytes: new.* record (see RECORD FORMAT) +** +** The SessionChange object passed as the third argument contains the +** values that were stored in the row when the session began (the old.* +** values). The statement handle passed as the second argument points +** at the current version of the row (the new.* values). +** +** If all of the old.* values are equal to their corresponding new.* value +** (i.e. nothing has changed), then no data at all is appended to the buffer. +** +** Otherwise, the old.* record contains all primary key values and the +** original values of any fields that have been modified. The new.* record +** contains the new values of only those fields that have been modified. +*/ +static int sessionAppendUpdate( + SessionBuffer *pBuf, /* Buffer to append to */ + int bPatchset, /* True for "patchset", 0 for "changeset" */ + sqlite3_stmt *pStmt, /* Statement handle pointing at new row */ + SessionChange *p, /* Object containing old values */ + u8 *abPK /* Boolean array - true for PK columns */ +){ + int rc = SQLITE_OK; + SessionBuffer buf2 = {0,0,0}; /* Buffer to accumulate new.* record in */ + int bNoop = 1; /* Set to zero if any values are modified */ + int nRewind = pBuf->nBuf; /* Set to zero if any values are modified */ + int i; /* Used to iterate through columns */ + u8 *pCsr = p->aRecord; /* Used to iterate through old.* values */ + + sessionAppendByte(pBuf, SQLITE_UPDATE, &rc); + sessionAppendByte(pBuf, p->bIndirect, &rc); + for(i=0; inBuf = nRewind; + }else{ + sessionAppendBlob(pBuf, buf2.aBuf, buf2.nBuf, &rc); + } + sqlite3_free(buf2.aBuf); + + return rc; +} + +/* +** Append a DELETE change to the buffer passed as the first argument. Use +** the changeset format if argument bPatchset is zero, or the patchset +** format otherwise. +*/ +static int sessionAppendDelete( + SessionBuffer *pBuf, /* Buffer to append to */ + int bPatchset, /* True for "patchset", 0 for "changeset" */ + SessionChange *p, /* Object containing old values */ + int nCol, /* Number of columns in table */ + u8 *abPK /* Boolean array - true for PK columns */ +){ + int rc = SQLITE_OK; + + sessionAppendByte(pBuf, SQLITE_DELETE, &rc); + sessionAppendByte(pBuf, p->bIndirect, &rc); + + if( bPatchset==0 ){ + sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc); + }else{ + int i; + u8 *a = p->aRecord; + for(i=0; iaRecord)==p->nRecord ); + } + + return rc; +} + +/* +** Formulate and prepare a SELECT statement to retrieve a row from table +** zTab in database zDb based on its primary key. i.e. +** +** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +*/ +static int sessionSelectStmt( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name */ + const char *zTab, /* Table name */ + int nCol, /* Number of columns in table */ + const char **azCol, /* Names of table columns */ + u8 *abPK, /* PRIMARY KEY array */ + sqlite3_stmt **ppStmt /* OUT: Prepared SELECT statement */ +){ + int rc = SQLITE_OK; + int i; + const char *zSep = ""; + SessionBuffer buf = {0, 0, 0}; + + sessionAppendStr(&buf, "SELECT * FROM ", &rc); + sessionAppendIdent(&buf, zDb, &rc); + sessionAppendStr(&buf, ".", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " WHERE ", &rc); + for(i=0; iaRecord; + + for(i=0; inCol, pRc); + sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc); + sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc); +} + +/* +** Generate either a changeset (if argument bPatchset is zero) or a patchset +** (if it is non-zero) based on the current contents of the session object +** passed as the first argument. +** +** If no error occurs, SQLITE_OK is returned and the new changeset/patchset +** stored in output variables *pnChangeset and *ppChangeset. Or, if an error +** occurs, an SQLite error code is returned and both output variables set +** to 0. +*/ +static int sessionGenerateChangeset( + sqlite3_session *pSession, /* Session object */ + int bPatchset, /* True for patchset, false for changeset */ + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, /* First argument for xOutput */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +){ + sqlite3 *db = pSession->db; /* Source database handle */ + SessionTable *pTab; /* Used to iterate through attached tables */ + SessionBuffer buf = {0,0,0}; /* Buffer in which to accumlate changeset */ + int rc; /* Return code */ + + assert( xOutput==0 || (pnChangeset==0 && ppChangeset==0 ) ); + + /* Zero the output variables in case an error occurs. If this session + ** object is already in the error state (sqlite3_session.rc != SQLITE_OK), + ** this call will be a no-op. */ + if( xOutput==0 ){ + *pnChangeset = 0; + *ppChangeset = 0; + } + + if( pSession->rc ) return pSession->rc; + rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + + for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ + if( pTab->nEntry ){ + const char *zName = pTab->zName; + int nCol; /* Number of columns in table */ + u8 *abPK; /* Primary key array */ + const char **azCol = 0; /* Table columns */ + int i; /* Used to iterate through hash buckets */ + sqlite3_stmt *pSel = 0; /* SELECT statement to query table pTab */ + int nRewind = buf.nBuf; /* Initial size of write buffer */ + int nNoop; /* Size of buffer after writing tbl header */ + + /* Check the table schema is still Ok. */ + rc = sessionTableInfo(db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK); + if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ + rc = SQLITE_SCHEMA; + } + + /* Write a table header */ + sessionAppendTableHdr(&buf, bPatchset, pTab, &rc); + + /* Build and compile a statement to execute: */ + if( rc==SQLITE_OK ){ + rc = sessionSelectStmt( + db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + } + + nNoop = buf.nBuf; + for(i=0; inChange && rc==SQLITE_OK; i++){ + SessionChange *p; /* Used to iterate through changes */ + + for(p=pTab->apChange[i]; rc==SQLITE_OK && p; p=p->pNext){ + rc = sessionSelectBind(pSel, nCol, abPK, p); + if( rc!=SQLITE_OK ) continue; + if( sqlite3_step(pSel)==SQLITE_ROW ){ + if( p->op==SQLITE_INSERT ){ + int iCol; + sessionAppendByte(&buf, SQLITE_INSERT, &rc); + sessionAppendByte(&buf, p->bIndirect, &rc); + for(iCol=0; iColop!=SQLITE_INSERT ){ + rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pSel); + } + + /* If the buffer is now larger than SESSIONS_STRM_CHUNK_SIZE, pass + ** its contents to the xOutput() callback. */ + if( xOutput + && rc==SQLITE_OK + && buf.nBuf>nNoop + && buf.nBuf>SESSIONS_STRM_CHUNK_SIZE + ){ + rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf); + nNoop = -1; + buf.nBuf = 0; + } + + } + } + + sqlite3_finalize(pSel); + if( buf.nBuf==nNoop ){ + buf.nBuf = nRewind; + } + sqlite3_free((char*)azCol); /* cast works around VC++ bug */ + } + } + + if( rc==SQLITE_OK ){ + if( xOutput==0 ){ + *pnChangeset = buf.nBuf; + *ppChangeset = buf.aBuf; + buf.aBuf = 0; + }else if( buf.nBuf>0 ){ + rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf); + } + } + + sqlite3_free(buf.aBuf); + sqlite3_exec(db, "RELEASE changeset", 0, 0, 0); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; +} + +/* +** Obtain a changeset object containing all changes recorded by the +** session object passed as the first argument. +** +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +*/ +int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +){ + return sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); +} + +/* +** Streaming version of sqlite3session_changeset(). +*/ +int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0); +} + +/* +** Streaming version of sqlite3session_patchset(). +*/ +int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0); +} + +/* +** Obtain a patchset object containing all changes recorded by the +** session object passed as the first argument. +** +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +*/ +int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ + void **ppPatchset /* OUT: Buffer containing changeset */ +){ + return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset); +} + +/* +** Enable or disable the session object passed as the first argument. +*/ +int sqlite3session_enable(sqlite3_session *pSession, int bEnable){ + int ret; + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + if( bEnable>=0 ){ + pSession->bEnable = bEnable; + } + ret = pSession->bEnable; + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return ret; +} + +/* +** Enable or disable the session object passed as the first argument. +*/ +int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect){ + int ret; + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + if( bIndirect>=0 ){ + pSession->bIndirect = bIndirect; + } + ret = pSession->bIndirect; + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return ret; +} + +/* +** Return true if there have been no changes to monitored tables recorded +** by the session object passed as the only argument. +*/ +int sqlite3session_isempty(sqlite3_session *pSession){ + int ret = 0; + SessionTable *pTab; + + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + for(pTab=pSession->pTable; pTab && ret==0; pTab=pTab->pNext){ + ret = (pTab->nEntry>0); + } + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + + return (ret==0); +} + +/* +** Do the work for either sqlite3changeset_start() or start_strm(). +*/ +static int sessionChangesetStart( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int nChangeset, /* Size of buffer pChangeset in bytes */ + void *pChangeset /* Pointer to buffer containing changeset */ +){ + sqlite3_changeset_iter *pRet; /* Iterator to return */ + int nByte; /* Number of bytes to allocate for iterator */ + + assert( xInput==0 || (pChangeset==0 && nChangeset==0) ); + + /* Zero the output variable in case an error occurs. */ + *pp = 0; + + /* Allocate and initialize the iterator structure. */ + nByte = sizeof(sqlite3_changeset_iter); + pRet = (sqlite3_changeset_iter *)sqlite3_malloc(nByte); + if( !pRet ) return SQLITE_NOMEM; + memset(pRet, 0, sizeof(sqlite3_changeset_iter)); + pRet->in.aData = (u8 *)pChangeset; + pRet->in.nData = nChangeset; + pRet->in.xInput = xInput; + pRet->in.pIn = pIn; + pRet->in.bEof = (xInput ? 0 : 1); + + /* Populate the output variable and return success. */ + *pp = pRet; + return SQLITE_OK; +} + +/* +** Create an iterator used to iterate through the contents of a changeset. +*/ +int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int nChangeset, /* Size of buffer pChangeset in bytes */ + void *pChangeset /* Pointer to buffer containing changeset */ +){ + return sessionChangesetStart(pp, 0, 0, nChangeset, pChangeset); +} + +/* +** Streaming version of sqlite3changeset_start(). +*/ +int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +){ + return sessionChangesetStart(pp, xInput, pIn, 0, 0); +} + +/* +** If the SessionInput object passed as the only argument is a streaming +** object and the buffer is full, discard some data to free up space. +*/ +static void sessionDiscardData(SessionInput *pIn){ + if( pIn->bEof && pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){ + int nMove = pIn->buf.nBuf - pIn->iNext; + assert( nMove>=0 ); + if( nMove>0 ){ + memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove); + } + pIn->buf.nBuf -= pIn->iNext; + pIn->iNext = 0; + pIn->nData = pIn->buf.nBuf; + } +} + +/* +** Ensure that there are at least nByte bytes available in the buffer. Or, +** if there are not nByte bytes remaining in the input, that all available +** data is in the buffer. +** +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +static int sessionInputBuffer(SessionInput *pIn, int nByte){ + int rc = SQLITE_OK; + if( pIn->xInput ){ + while( !pIn->bEof && (pIn->iNext+nByte)>=pIn->nData && rc==SQLITE_OK ){ + int nNew = SESSIONS_STRM_CHUNK_SIZE; + + if( pIn->bNoDiscard==0 ) sessionDiscardData(pIn); + if( SQLITE_OK==sessionBufferGrow(&pIn->buf, nNew, &rc) ){ + rc = pIn->xInput(pIn->pIn, &pIn->buf.aBuf[pIn->buf.nBuf], &nNew); + if( nNew==0 ){ + pIn->bEof = 1; + }else{ + pIn->buf.nBuf += nNew; + } + } + + pIn->aData = pIn->buf.aBuf; + pIn->nData = pIn->buf.nBuf; + } + } + return rc; +} + +/* +** When this function is called, *ppRec points to the start of a record +** that contains nCol values. This function advances the pointer *ppRec +** until it points to the byte immediately following that record. +*/ +static void sessionSkipRecord( + u8 **ppRec, /* IN/OUT: Record pointer */ + int nCol /* Number of values in record */ +){ + u8 *aRec = *ppRec; + int i; + for(i=0; iaData[pIn->iNext++]; + } + + assert( apOut[i]==0 ); + if( eType ){ + apOut[i] = sqlite3ValueNew(0); + if( !apOut[i] ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + u8 *aVal = &pIn->aData[pIn->iNext]; + if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + int nByte; + pIn->iNext += sessionVarintGet(aVal, &nByte); + rc = sessionInputBuffer(pIn, nByte); + if( rc==SQLITE_OK ){ + u8 enc = (eType==SQLITE_TEXT ? SQLITE_UTF8 : 0); + rc = sessionValueSetStr(apOut[i],&pIn->aData[pIn->iNext],nByte,enc); + } + pIn->iNext += nByte; + } + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + sqlite3_int64 v = sessionGetI64(aVal); + if( eType==SQLITE_INTEGER ){ + sqlite3VdbeMemSetInt64(apOut[i], v); + }else{ + double d; + memcpy(&d, &v, 8); + sqlite3VdbeMemSetDouble(apOut[i], d); + } + pIn->iNext += 8; + } + } + } + + return rc; +} + +/* +** The input pointer currently points to the second byte of a table-header. +** Specifically, to the following: +** +** + number of columns in table (varint) +** + array of PK flags (1 byte per column), +** + table name (nul terminated). +** +** This function ensures that all of the above is present in the input +** buffer (i.e. that it can be accessed without any calls to xInput()). +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +** The input pointer is not moved. +*/ +static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ + int rc = SQLITE_OK; + int nCol = 0; + int nRead = 0; + + rc = sessionInputBuffer(pIn, 9); + if( rc==SQLITE_OK ){ + nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); + rc = sessionInputBuffer(pIn, nRead+nCol+100); + nRead += nCol; + } + + while( rc==SQLITE_OK ){ + while( (pIn->iNext + nRead)nData && pIn->aData[pIn->iNext + nRead] ){ + nRead++; + } + if( (pIn->iNext + nRead)nData ) break; + rc = sessionInputBuffer(pIn, nRead + 100); + } + *pnByte = nRead+1; + return rc; +} + +/* +** The input pointer currently points to the first byte of the first field +** of a record consisting of nCol columns. This function ensures the entire +** record is buffered. It does not move the input pointer. +** +** If successful, SQLITE_OK is returned and *pnByte is set to the size of +** the record in bytes. Otherwise, an SQLite error code is returned. The +** final value of *pnByte is undefined in this case. +*/ +static int sessionChangesetBufferRecord( + SessionInput *pIn, /* Input data */ + int nCol, /* Number of columns in record */ + int *pnByte /* OUT: Size of record in bytes */ +){ + int rc = SQLITE_OK; + int nByte = 0; + int i; + for(i=0; rc==SQLITE_OK && iaData[pIn->iNext + nByte++]; + if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + int n; + nByte += sessionVarintGet(&pIn->aData[pIn->iNext+nByte], &n); + nByte += n; + rc = sessionInputBuffer(pIn, nByte); + }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + nByte += 8; + } + } + } + *pnByte = nByte; + return rc; +} + +/* +** The input pointer currently points to the second byte of a table-header. +** Specifically, to the following: +** +** + number of columns in table (varint) +** + array of PK flags (1 byte per column), +** + table name (nul terminated). +** +** This function decodes the table-header and populates the p->nCol, +** p->zTab and p->abPK[] variables accordingly. The p->apValue[] array is +** also allocated or resized according to the new value of p->nCol. The +** input pointer is left pointing to the byte following the table header. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code +** is returned and the final values of the various fields enumerated above +** are undefined. +*/ +static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){ + int rc; + int nCopy; + assert( p->rc==SQLITE_OK ); + + rc = sessionChangesetBufferTblhdr(&p->in, &nCopy); + if( rc==SQLITE_OK ){ + int nByte; + int nVarint; + nVarint = sessionVarintGet(&p->in.aData[p->in.iNext], &p->nCol); + nCopy -= nVarint; + p->in.iNext += nVarint; + nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; + p->tblhdr.nBuf = 0; + sessionBufferGrow(&p->tblhdr, nByte, &rc); + } + + if( rc==SQLITE_OK ){ + int iPK = sizeof(sqlite3_value*)*p->nCol*2; + memset(p->tblhdr.aBuf, 0, iPK); + memcpy(&p->tblhdr.aBuf[iPK], &p->in.aData[p->in.iNext], nCopy); + p->in.iNext += nCopy; + } + + p->apValue = (sqlite3_value**)p->tblhdr.aBuf; + p->abPK = (u8*)&p->apValue[p->nCol*2]; + p->zTab = (char*)&p->abPK[p->nCol]; + return (p->rc = rc); +} + +/* +** Advance the changeset iterator to the next change. +** +** If both paRec and pnRec are NULL, then this function works like the public +** API sqlite3changeset_next(). If SQLITE_ROW is returned, then the +** sqlite3changeset_new() and old() APIs may be used to query for values. +** +** Otherwise, if paRec and pnRec are not NULL, then a pointer to the change +** record is written to *paRec before returning and the number of bytes in +** the record to *pnRec. +** +** Either way, this function returns SQLITE_ROW if the iterator is +** successfully advanced to the next change in the changeset, an SQLite +** error code if an error occurs, or SQLITE_DONE if there are no further +** changes in the changeset. +*/ +static int sessionChangesetNext( + sqlite3_changeset_iter *p, /* Changeset iterator */ + u8 **paRec, /* If non-NULL, store record pointer here */ + int *pnRec /* If non-NULL, store size of record here */ +){ + int i; + u8 op; + + assert( (paRec==0 && pnRec==0) || (paRec && pnRec) ); + + /* If the iterator is in the error-state, return immediately. */ + if( p->rc!=SQLITE_OK ) return p->rc; + + /* Free the current contents of p->apValue[], if any. */ + if( p->apValue ){ + for(i=0; inCol*2; i++){ + sqlite3ValueFree(p->apValue[i]); + } + memset(p->apValue, 0, sizeof(sqlite3_value*)*p->nCol*2); + } + + /* Make sure the buffer contains at least 10 bytes of input data, or all + ** remaining data if there are less than 10 bytes available. This is + ** sufficient either for the 'T' or 'P' byte and the varint that follows + ** it, or for the two single byte values otherwise. */ + p->rc = sessionInputBuffer(&p->in, 2); + if( p->rc!=SQLITE_OK ) return p->rc; + + /* If the iterator is already at the end of the changeset, return DONE. */ + if( p->in.iNext>=p->in.nData ){ + return SQLITE_DONE; + } + + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + + op = p->in.aData[p->in.iNext++]; + if( op=='T' || op=='P' ){ + p->bPatchset = (op=='P'); + if( sessionChangesetReadTblhdr(p) ) return p->rc; + if( (p->rc = sessionInputBuffer(&p->in, 2)) ) return p->rc; + p->in.iCurrent = p->in.iNext; + op = p->in.aData[p->in.iNext++]; + } + + p->op = op; + p->bIndirect = p->in.aData[p->in.iNext++]; + if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ + return (p->rc = SQLITE_CORRUPT_BKPT); + } + + if( paRec ){ + int nVal; /* Number of values to buffer */ + if( p->bPatchset==0 && op==SQLITE_UPDATE ){ + nVal = p->nCol * 2; + }else if( p->bPatchset && op==SQLITE_DELETE ){ + nVal = 0; + for(i=0; inCol; i++) if( p->abPK[i] ) nVal++; + }else{ + nVal = p->nCol; + } + p->rc = sessionChangesetBufferRecord(&p->in, nVal, pnRec); + if( p->rc!=SQLITE_OK ) return p->rc; + *paRec = &p->in.aData[p->in.iNext]; + p->in.iNext += *pnRec; + }else{ + + /* If this is an UPDATE or DELETE, read the old.* record. */ + if( p->op!=SQLITE_INSERT && (p->bPatchset==0 || p->op==SQLITE_DELETE) ){ + u8 *abPK = p->bPatchset ? p->abPK : 0; + p->rc = sessionReadRecord(&p->in, p->nCol, abPK, p->apValue); + if( p->rc!=SQLITE_OK ) return p->rc; + } + + /* If this is an INSERT or UPDATE, read the new.* record. */ + if( p->op!=SQLITE_DELETE ){ + p->rc = sessionReadRecord(&p->in, p->nCol, 0, &p->apValue[p->nCol]); + if( p->rc!=SQLITE_OK ) return p->rc; + } + + if( p->bPatchset && p->op==SQLITE_UPDATE ){ + /* If this is an UPDATE that is part of a patchset, then all PK and + ** modified fields are present in the new.* record. The old.* record + ** is currently completely empty. This block shifts the PK fields from + ** new.* to old.*, to accommodate the code that reads these arrays. */ + for(i=0; inCol; i++){ + assert( p->apValue[i]==0 ); + assert( p->abPK[i]==0 || p->apValue[i+p->nCol] ); + if( p->abPK[i] ){ + p->apValue[i] = p->apValue[i+p->nCol]; + p->apValue[i+p->nCol] = 0; + } + } + } + } + + return SQLITE_ROW; +} + +/* +** Advance an iterator created by sqlite3changeset_start() to the next +** change in the changeset. This function may return SQLITE_ROW, SQLITE_DONE +** or SQLITE_CORRUPT. +** +** This function may not be called on iterators passed to a conflict handler +** callback by changeset_apply(). +*/ +int sqlite3changeset_next(sqlite3_changeset_iter *p){ + return sessionChangesetNext(p, 0, 0); +} + +/* +** The following function extracts information on the current change +** from a changeset iterator. It may only be called after changeset_next() +** has returned SQLITE_ROW. +*/ +int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator handle */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True if change is indirect */ +){ + *pOp = pIter->op; + *pnCol = pIter->nCol; + *pzTab = pIter->zTab; + if( pbIndirect ) *pbIndirect = pIter->bIndirect; + return SQLITE_OK; +} + +/* +** Return information regarding the PRIMARY KEY and number of columns in +** the database table affected by the change that pIter currently points +** to. This function may only be called after changeset_next() returns +** SQLITE_ROW. +*/ +int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +){ + *pabPK = pIter->abPK; + if( pnCol ) *pnCol = pIter->nCol; + return SQLITE_OK; +} + +/* +** This function may only be called while the iterator is pointing to an +** SQLITE_UPDATE or SQLITE_DELETE change (see sqlite3changeset_op()). +** Otherwise, SQLITE_MISUSE is returned. +** +** It sets *ppValue to point to an sqlite3_value structure containing the +** iVal'th value in the old.* record. Or, if that particular value is not +** included in the record (because the change is an UPDATE and the field +** was not modified and is not a PK column), set *ppValue to NULL. +** +** If value iVal is out-of-range, SQLITE_RANGE is returned and *ppValue is +** not modified. Otherwise, SQLITE_OK. +*/ +int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Index of old.* value to retrieve */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +){ + if( pIter->op!=SQLITE_UPDATE && pIter->op!=SQLITE_DELETE ){ + return SQLITE_MISUSE; + } + if( iVal<0 || iVal>=pIter->nCol ){ + return SQLITE_RANGE; + } + *ppValue = pIter->apValue[iVal]; + return SQLITE_OK; +} + +/* +** This function may only be called while the iterator is pointing to an +** SQLITE_UPDATE or SQLITE_INSERT change (see sqlite3changeset_op()). +** Otherwise, SQLITE_MISUSE is returned. +** +** It sets *ppValue to point to an sqlite3_value structure containing the +** iVal'th value in the new.* record. Or, if that particular value is not +** included in the record (because the change is an UPDATE and the field +** was not modified), set *ppValue to NULL. +** +** If value iVal is out-of-range, SQLITE_RANGE is returned and *ppValue is +** not modified. Otherwise, SQLITE_OK. +*/ +int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Index of new.* value to retrieve */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +){ + if( pIter->op!=SQLITE_UPDATE && pIter->op!=SQLITE_INSERT ){ + return SQLITE_MISUSE; + } + if( iVal<0 || iVal>=pIter->nCol ){ + return SQLITE_RANGE; + } + *ppValue = pIter->apValue[pIter->nCol+iVal]; + return SQLITE_OK; +} + +/* +** The following two macros are used internally. They are similar to the +** sqlite3changeset_new() and sqlite3changeset_old() functions, except that +** they omit all error checking and return a pointer to the requested value. +*/ +#define sessionChangesetNew(pIter, iVal) (pIter)->apValue[(pIter)->nCol+(iVal)] +#define sessionChangesetOld(pIter, iVal) (pIter)->apValue[(iVal)] + +/* +** This function may only be called with a changeset iterator that has been +** passed to an SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT +** conflict-handler function. Otherwise, SQLITE_MISUSE is returned. +** +** If successful, *ppValue is set to point to an sqlite3_value structure +** containing the iVal'th value of the conflicting record. +** +** If value iVal is out-of-range or some other error occurs, an SQLite error +** code is returned. Otherwise, SQLITE_OK. +*/ +int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Index of conflict record value to fetch */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +){ + if( !pIter->pConflict ){ + return SQLITE_MISUSE; + } + if( iVal<0 || iVal>=sqlite3_column_count(pIter->pConflict) ){ + return SQLITE_RANGE; + } + *ppValue = sqlite3_column_value(pIter->pConflict, iVal); + return SQLITE_OK; +} + +/* +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +){ + if( pIter->pConflict || pIter->apValue ){ + return SQLITE_MISUSE; + } + *pnOut = pIter->nCol; + return SQLITE_OK; +} + + +/* +** Finalize an iterator allocated with sqlite3changeset_start(). +** +** This function may not be called on iterators passed to a conflict handler +** callback by changeset_apply(). +*/ +int sqlite3changeset_finalize(sqlite3_changeset_iter *p){ + int rc = SQLITE_OK; + if( p ){ + int i; /* Used to iterate through p->apValue[] */ + rc = p->rc; + if( p->apValue ){ + for(i=0; inCol*2; i++) sqlite3ValueFree(p->apValue[i]); + } + sqlite3_free(p->tblhdr.aBuf); + sqlite3_free(p->in.buf.aBuf); + sqlite3_free(p); + } + return rc; +} + +static int sessionChangesetInvert( + SessionInput *pInput, /* Input changeset */ + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, + int *pnInverted, /* OUT: Number of bytes in output changeset */ + void **ppInverted /* OUT: Inverse of pChangeset */ +){ + int rc = SQLITE_OK; /* Return value */ + SessionBuffer sOut; /* Output buffer */ + int nCol = 0; /* Number of cols in current table */ + u8 *abPK = 0; /* PK array for current table */ + sqlite3_value **apVal = 0; /* Space for values for UPDATE inversion */ + SessionBuffer sPK = {0, 0, 0}; /* PK array for current table */ + + /* Initialize the output buffer */ + memset(&sOut, 0, sizeof(SessionBuffer)); + + /* Zero the output variables in case an error occurs. */ + if( ppInverted ){ + *ppInverted = 0; + *pnInverted = 0; + } + + while( 1 ){ + u8 eType; + + /* Test for EOF. */ + if( (rc = sessionInputBuffer(pInput, 2)) ) goto finished_invert; + if( pInput->iNext>=pInput->nData ) break; + eType = pInput->aData[pInput->iNext]; + + switch( eType ){ + case 'T': { + /* A 'table' record consists of: + ** + ** * A constant 'T' character, + ** * Number of columns in said table (a varint), + ** * An array of nCol bytes (sPK), + ** * A nul-terminated table name. + */ + int nByte; + int nVar; + pInput->iNext++; + if( (rc = sessionChangesetBufferTblhdr(pInput, &nByte)) ){ + goto finished_invert; + } + nVar = sessionVarintGet(&pInput->aData[pInput->iNext], &nCol); + sPK.nBuf = 0; + sessionAppendBlob(&sPK, &pInput->aData[pInput->iNext+nVar], nCol, &rc); + sessionAppendByte(&sOut, eType, &rc); + sessionAppendBlob(&sOut, &pInput->aData[pInput->iNext], nByte, &rc); + if( rc ) goto finished_invert; + + pInput->iNext += nByte; + sqlite3_free(apVal); + apVal = 0; + abPK = sPK.aBuf; + break; + } + + case SQLITE_INSERT: + case SQLITE_DELETE: { + int nByte; + int bIndirect = pInput->aData[pInput->iNext+1]; + int eType2 = (eType==SQLITE_DELETE ? SQLITE_INSERT : SQLITE_DELETE); + pInput->iNext += 2; + assert( rc==SQLITE_OK ); + rc = sessionChangesetBufferRecord(pInput, nCol, &nByte); + sessionAppendByte(&sOut, eType2, &rc); + sessionAppendByte(&sOut, bIndirect, &rc); + sessionAppendBlob(&sOut, &pInput->aData[pInput->iNext], nByte, &rc); + pInput->iNext += nByte; + if( rc ) goto finished_invert; + break; + } + + case SQLITE_UPDATE: { + int iCol; + + if( 0==apVal ){ + apVal = (sqlite3_value **)sqlite3_malloc(sizeof(apVal[0])*nCol*2); + if( 0==apVal ){ + rc = SQLITE_NOMEM; + goto finished_invert; + } + memset(apVal, 0, sizeof(apVal[0])*nCol*2); + } + + /* Write the header for the new UPDATE change. Same as the original. */ + sessionAppendByte(&sOut, eType, &rc); + sessionAppendByte(&sOut, pInput->aData[pInput->iNext+1], &rc); + + /* Read the old.* and new.* records for the update change. */ + pInput->iNext += 2; + rc = sessionReadRecord(pInput, nCol, 0, &apVal[0]); + if( rc==SQLITE_OK ){ + rc = sessionReadRecord(pInput, nCol, 0, &apVal[nCol]); + } + + /* Write the new old.* record. Consists of the PK columns from the + ** original old.* record, and the other values from the original + ** new.* record. */ + for(iCol=0; iCol=SESSIONS_STRM_CHUNK_SIZE ){ + rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); + sOut.nBuf = 0; + if( rc!=SQLITE_OK ) goto finished_invert; + } + } + + assert( rc==SQLITE_OK ); + if( pnInverted ){ + *pnInverted = sOut.nBuf; + *ppInverted = sOut.aBuf; + sOut.aBuf = 0; + }else if( sOut.nBuf>0 ){ + rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); + } + + finished_invert: + sqlite3_free(sOut.aBuf); + sqlite3_free(apVal); + sqlite3_free(sPK.aBuf); + return rc; +} + + +/* +** Invert a changeset object. +*/ +int sqlite3changeset_invert( + int nChangeset, /* Number of bytes in input */ + const void *pChangeset, /* Input changeset */ + int *pnInverted, /* OUT: Number of bytes in output changeset */ + void **ppInverted /* OUT: Inverse of pChangeset */ +){ + SessionInput sInput; + + /* Set up the input stream */ + memset(&sInput, 0, sizeof(SessionInput)); + sInput.nData = nChangeset; + sInput.aData = (u8*)pChangeset; + + return sessionChangesetInvert(&sInput, 0, 0, pnInverted, ppInverted); +} + +/* +** Streaming version of sqlite3changeset_invert(). +*/ +int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + SessionInput sInput; + int rc; + + /* Set up the input stream */ + memset(&sInput, 0, sizeof(SessionInput)); + sInput.xInput = xInput; + sInput.pIn = pIn; + + rc = sessionChangesetInvert(&sInput, xOutput, pOut, 0, 0); + sqlite3_free(sInput.buf.aBuf); + return rc; +} + +typedef struct SessionApplyCtx SessionApplyCtx; +struct SessionApplyCtx { + sqlite3 *db; + sqlite3_stmt *pDelete; /* DELETE statement */ + sqlite3_stmt *pUpdate; /* UPDATE statement */ + sqlite3_stmt *pInsert; /* INSERT statement */ + sqlite3_stmt *pSelect; /* SELECT statement */ + int nCol; /* Size of azCol[] and abPK[] arrays */ + const char **azCol; /* Array of column names */ + u8 *abPK; /* Boolean array - true if column is in PK */ + + int bDeferConstraints; /* True to defer constraints */ + SessionBuffer constraints; /* Deferred constraints are stored here */ +}; + +/* +** Formulate a statement to DELETE a row from database db. Assuming a table +** structure like this: +** +** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); +** +** The DELETE statement looks like this: +** +** DELETE FROM x WHERE a = :1 AND c = :3 AND (:5 OR b IS :2 AND d IS :4) +** +** Variable :5 (nCol+1) is a boolean. It should be set to 0 if we require +** matching b and d values, or 1 otherwise. The second case comes up if the +** conflict handler is invoked with NOTFOUND and returns CHANGESET_REPLACE. +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pDelete is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionDeleteRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + int i; + const char *zSep = ""; + int rc = SQLITE_OK; + SessionBuffer buf = {0, 0, 0}; + int nPk = 0; + + sessionAppendStr(&buf, "DELETE FROM ", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " WHERE ", &rc); + + for(i=0; inCol; i++){ + if( p->abPK[i] ){ + nPk++; + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " = ?", &rc); + sessionAppendInteger(&buf, i+1, &rc); + zSep = " AND "; + } + } + + if( nPknCol ){ + sessionAppendStr(&buf, " AND (?", &rc); + sessionAppendInteger(&buf, p->nCol+1, &rc); + sessionAppendStr(&buf, " OR ", &rc); + + zSep = ""; + for(i=0; inCol; i++){ + if( !p->abPK[i] ){ + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " IS ?", &rc); + sessionAppendInteger(&buf, i+1, &rc); + zSep = "AND "; + } + } + sessionAppendStr(&buf, ")", &rc); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pDelete, 0); + } + sqlite3_free(buf.aBuf); + + return rc; +} + +/* +** Formulate and prepare a statement to UPDATE a row from database db. +** Assuming a table structure like this: +** +** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); +** +** The UPDATE statement looks like this: +** +** UPDATE x SET +** a = CASE WHEN ?2 THEN ?3 ELSE a END, +** b = CASE WHEN ?5 THEN ?6 ELSE b END, +** c = CASE WHEN ?8 THEN ?9 ELSE c END, +** d = CASE WHEN ?11 THEN ?12 ELSE d END +** WHERE a = ?1 AND c = ?7 AND (?13 OR +** (?5==0 OR b IS ?4) AND (?11==0 OR d IS ?10) AND +** ) +** +** For each column in the table, there are three variables to bind: +** +** ?(i*3+1) The old.* value of the column, if any. +** ?(i*3+2) A boolean flag indicating that the value is being modified. +** ?(i*3+3) The new.* value of the column, if any. +** +** Also, a boolean flag that, if set to true, causes the statement to update +** a row even if the non-PK values do not match. This is required if the +** conflict-handler is invoked with CHANGESET_DATA and returns +** CHANGESET_REPLACE. This is variable "?(nCol*3+1)". +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pUpdate is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionUpdateRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + int rc = SQLITE_OK; + int i; + const char *zSep = ""; + SessionBuffer buf = {0, 0, 0}; + + /* Append "UPDATE tbl SET " */ + sessionAppendStr(&buf, "UPDATE ", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " SET ", &rc); + + /* Append the assignments */ + for(i=0; inCol; i++){ + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " = CASE WHEN ?", &rc); + sessionAppendInteger(&buf, i*3+2, &rc); + sessionAppendStr(&buf, " THEN ?", &rc); + sessionAppendInteger(&buf, i*3+3, &rc); + sessionAppendStr(&buf, " ELSE ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " END", &rc); + zSep = ", "; + } + + /* Append the PK part of the WHERE clause */ + sessionAppendStr(&buf, " WHERE ", &rc); + for(i=0; inCol; i++){ + if( p->abPK[i] ){ + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " = ?", &rc); + sessionAppendInteger(&buf, i*3+1, &rc); + sessionAppendStr(&buf, " AND ", &rc); + } + } + + /* Append the non-PK part of the WHERE clause */ + sessionAppendStr(&buf, " (?", &rc); + sessionAppendInteger(&buf, p->nCol*3+1, &rc); + sessionAppendStr(&buf, " OR 1", &rc); + for(i=0; inCol; i++){ + if( !p->abPK[i] ){ + sessionAppendStr(&buf, " AND (?", &rc); + sessionAppendInteger(&buf, i*3+2, &rc); + sessionAppendStr(&buf, "=0 OR ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " IS ?", &rc); + sessionAppendInteger(&buf, i*3+1, &rc); + sessionAppendStr(&buf, ")", &rc); + } + } + sessionAppendStr(&buf, ")", &rc); + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pUpdate, 0); + } + sqlite3_free(buf.aBuf); + + return rc; +} + +/* +** Formulate and prepare an SQL statement to query table zTab by primary +** key. Assuming the following table structure: +** +** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); +** +** The SELECT statement looks like this: +** +** SELECT * FROM x WHERE a = ?1 AND c = ?3 +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pSelect is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionSelectRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + return sessionSelectStmt( + db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); +} + +/* +** Formulate and prepare an INSERT statement to add a record to table zTab. +** For example: +** +** INSERT INTO main."zTab" VALUES(?1, ?2, ?3 ...); +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pInsert is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionInsertRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + int rc = SQLITE_OK; + int i; + SessionBuffer buf = {0, 0, 0}; + + sessionAppendStr(&buf, "INSERT INTO main.", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " VALUES(?", &rc); + for(i=1; inCol; i++){ + sessionAppendStr(&buf, ", ?", &rc); + } + sessionAppendStr(&buf, ")", &rc); + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pInsert, 0); + } + sqlite3_free(buf.aBuf); + return rc; +} + +/* +** A wrapper around sqlite3_bind_value() that detects an extra problem. +** See comments in the body of this function for details. +*/ +static int sessionBindValue( + sqlite3_stmt *pStmt, /* Statement to bind value to */ + int i, /* Parameter number to bind to */ + sqlite3_value *pVal /* Value to bind */ +){ + int eType = sqlite3_value_type(pVal); + /* COVERAGE: The (pVal->z==0) branch is never true using current versions + ** of SQLite. If a malloc fails in an sqlite3_value_xxx() function, either + ** the (pVal->z) variable remains as it was or the type of the value is + ** set to SQLITE_NULL. */ + if( (eType==SQLITE_TEXT || eType==SQLITE_BLOB) && pVal->z==0 ){ + /* This condition occurs when an earlier OOM in a call to + ** sqlite3_value_text() or sqlite3_value_blob() (perhaps from within + ** a conflict-handler) has zeroed the pVal->z pointer. Return NOMEM. */ + return SQLITE_NOMEM; + } + return sqlite3_bind_value(pStmt, i, pVal); +} + +/* +** Iterator pIter must point to an SQLITE_INSERT entry. This function +** transfers new.* values from the current iterator entry to statement +** pStmt. The table being inserted into has nCol columns. +** +** New.* value $i from the iterator is bound to variable ($i+1) of +** statement pStmt. If parameter abPK is NULL, all values from 0 to (nCol-1) +** are transfered to the statement. Otherwise, if abPK is not NULL, it points +** to an array nCol elements in size. In this case only those values for +** which abPK[$i] is true are read from the iterator and bound to the +** statement. +** +** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. +*/ +static int sessionBindRow( + sqlite3_changeset_iter *pIter, /* Iterator to read values from */ + int(*xValue)(sqlite3_changeset_iter *, int, sqlite3_value **), + int nCol, /* Number of columns */ + u8 *abPK, /* If not NULL, bind only if true */ + sqlite3_stmt *pStmt /* Bind values to this statement */ +){ + int i; + int rc = SQLITE_OK; + + /* Neither sqlite3changeset_old or sqlite3changeset_new can fail if the + ** argument iterator points to a suitable entry. Make sure that xValue + ** is one of these to guarantee that it is safe to ignore the return + ** in the code below. */ + assert( xValue==sqlite3changeset_old || xValue==sqlite3changeset_new ); + + for(i=0; rc==SQLITE_OK && idb, pIter, p->abPK, p->pSelect); + }else{ + rc = SQLITE_OK; + } + + if( rc==SQLITE_ROW ){ + /* There exists another row with the new.* primary key. */ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + rc = sqlite3_reset(p->pSelect); + }else if( rc==SQLITE_OK ){ + if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ + /* Instead of invoking the conflict handler, append the change blob + ** to the SessionApplyCtx.constraints buffer. */ + u8 *aBlob = &pIter->in.aData[pIter->in.iCurrent]; + int nBlob = pIter->in.iNext - pIter->in.iCurrent; + sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc); + res = SQLITE_CHANGESET_OMIT; + }else{ + /* No other row with the new.* primary key. */ + res = xConflict(pCtx, eType+1, pIter); + if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE; + } + } + + if( rc==SQLITE_OK ){ + switch( res ){ + case SQLITE_CHANGESET_REPLACE: + assert( pbReplace ); + *pbReplace = 1; + break; + + case SQLITE_CHANGESET_OMIT: + break; + + case SQLITE_CHANGESET_ABORT: + rc = SQLITE_ABORT; + break; + + default: + rc = SQLITE_MISUSE; + break; + } + } + + return rc; +} + +/* +** Attempt to apply the change that the iterator passed as the first argument +** currently points to to the database. If a conflict is encountered, invoke +** the conflict handler callback. +** +** If argument pbRetry is NULL, then ignore any CHANGESET_DATA conflict. If +** one is encountered, update or delete the row with the matching primary key +** instead. Or, if pbRetry is not NULL and a CHANGESET_DATA conflict occurs, +** invoke the conflict handler. If it returns CHANGESET_REPLACE, set *pbRetry +** to true before returning. In this case the caller will invoke this function +** again, this time with pbRetry set to NULL. +** +** If argument pbReplace is NULL and a CHANGESET_CONFLICT conflict is +** encountered invoke the conflict handler with CHANGESET_CONSTRAINT instead. +** Or, if pbReplace is not NULL, invoke it with CHANGESET_CONFLICT. If such +** an invocation returns SQLITE_CHANGESET_REPLACE, set *pbReplace to true +** before retrying. In this case the caller attempts to remove the conflicting +** row before invoking this function again, this time with pbReplace set +** to NULL. +** +** If any conflict handler returns SQLITE_CHANGESET_ABORT, this function +** returns SQLITE_ABORT. Otherwise, if no error occurs, SQLITE_OK is +** returned. +*/ +static int sessionApplyOneOp( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + SessionApplyCtx *p, /* changeset_apply() context */ + int(*xConflict)(void *, int, sqlite3_changeset_iter *), + void *pCtx, /* First argument for the conflict handler */ + int *pbReplace, /* OUT: True to remove PK row and retry */ + int *pbRetry /* OUT: True to retry. */ +){ + const char *zDummy; + int op; + int nCol; + int rc = SQLITE_OK; + + assert( p->pDelete && p->pUpdate && p->pInsert && p->pSelect ); + assert( p->azCol && p->abPK ); + assert( !pbReplace || *pbReplace==0 ); + + sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); + + if( op==SQLITE_DELETE ){ + + /* Bind values to the DELETE statement. If conflict handling is required, + ** bind values for all columns and set bound variable (nCol+1) to true. + ** Or, if conflict handling is not required, bind just the PK column + ** values and, if it exists, set (nCol+1) to false. Conflict handling + ** is not required if: + ** + ** * this is a patchset, or + ** * (pbRetry==0), or + ** * all columns of the table are PK columns (in this case there is + ** no (nCol+1) variable to bind to). + */ + u8 *abPK = (pIter->bPatchset ? p->abPK : 0); + rc = sessionBindRow(pIter, sqlite3changeset_old, nCol, abPK, p->pDelete); + if( rc==SQLITE_OK && sqlite3_bind_parameter_count(p->pDelete)>nCol ){ + rc = sqlite3_bind_int(p->pDelete, nCol+1, (pbRetry==0 || abPK)); + } + if( rc!=SQLITE_OK ) return rc; + + sqlite3_step(p->pDelete); + rc = sqlite3_reset(p->pDelete); + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + rc = sessionConflictHandler( + SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry + ); + }else if( (rc&0xff)==SQLITE_CONSTRAINT ){ + rc = sessionConflictHandler( + SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, 0 + ); + } + + }else if( op==SQLITE_UPDATE ){ + int i; + + /* Bind values to the UPDATE statement. */ + for(i=0; rc==SQLITE_OK && ipUpdate, i*3+2, !!pNew); + if( pOld ){ + rc = sessionBindValue(p->pUpdate, i*3+1, pOld); + } + if( rc==SQLITE_OK && pNew ){ + rc = sessionBindValue(p->pUpdate, i*3+3, pNew); + } + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int(p->pUpdate, nCol*3+1, pbRetry==0 || pIter->bPatchset); + } + if( rc!=SQLITE_OK ) return rc; + + /* Attempt the UPDATE. In the case of a NOTFOUND or DATA conflict, + ** the result will be SQLITE_OK with 0 rows modified. */ + sqlite3_step(p->pUpdate); + rc = sqlite3_reset(p->pUpdate); + + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + /* A NOTFOUND or DATA error. Search the table to see if it contains + ** a row with a matching primary key. If so, this is a DATA conflict. + ** Otherwise, if there is no primary key match, it is a NOTFOUND. */ + + rc = sessionConflictHandler( + SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry + ); + + }else if( (rc&0xff)==SQLITE_CONSTRAINT ){ + /* This is always a CONSTRAINT conflict. */ + rc = sessionConflictHandler( + SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, 0 + ); + } + + }else{ + assert( op==SQLITE_INSERT ); + rc = sessionBindRow(pIter, sqlite3changeset_new, nCol, 0, p->pInsert); + if( rc!=SQLITE_OK ) return rc; + + sqlite3_step(p->pInsert); + rc = sqlite3_reset(p->pInsert); + if( (rc&0xff)==SQLITE_CONSTRAINT ){ + rc = sessionConflictHandler( + SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, pbReplace + ); + } + } + + return rc; +} + +/* +** Attempt to apply the change that the iterator passed as the first argument +** currently points to to the database. If a conflict is encountered, invoke +** the conflict handler callback. +** +** The difference between this function and sessionApplyOne() is that this +** function handles the case where the conflict-handler is invoked and +** returns SQLITE_CHANGESET_REPLACE - indicating that the change should be +** retried in some manner. +*/ +static int sessionApplyOneWithRetry( + sqlite3 *db, /* Apply change to "main" db of this handle */ + sqlite3_changeset_iter *pIter, /* Changeset iterator to read change from */ + SessionApplyCtx *pApply, /* Apply context */ + int(*xConflict)(void*, int, sqlite3_changeset_iter*), + void *pCtx /* First argument passed to xConflict */ +){ + int bReplace = 0; + int bRetry = 0; + int rc; + + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, &bReplace, &bRetry); + assert( rc==SQLITE_OK || (bRetry==0 && bReplace==0) ); + + /* If the bRetry flag is set, the change has not been applied due to an + ** SQLITE_CHANGESET_DATA problem (i.e. this is an UPDATE or DELETE and + ** a row with the correct PK is present in the db, but one or more other + ** fields do not contain the expected values) and the conflict handler + ** returned SQLITE_CHANGESET_REPLACE. In this case retry the operation, + ** but pass NULL as the final argument so that sessionApplyOneOp() ignores + ** the SQLITE_CHANGESET_DATA problem. */ + if( bRetry ){ + assert( pIter->op==SQLITE_UPDATE || pIter->op==SQLITE_DELETE ); + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + + /* If the bReplace flag is set, the change is an INSERT that has not + ** been performed because the database already contains a row with the + ** specified primary key and the conflict handler returned + ** SQLITE_CHANGESET_REPLACE. In this case remove the conflicting row + ** before reattempting the INSERT. */ + else if( bReplace ){ + assert( pIter->op==SQLITE_INSERT ); + rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sessionBindRow(pIter, + sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete); + sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pApply->pDelete); + rc = sqlite3_reset(pApply->pDelete); + } + if( rc==SQLITE_OK ){ + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + } + } + + return rc; +} + +/* +** Retry the changes accumulated in the pApply->constraints buffer. +*/ +static int sessionRetryConstraints( + sqlite3 *db, + int bPatchset, + const char *zTab, + SessionApplyCtx *pApply, + int(*xConflict)(void*, int, sqlite3_changeset_iter*), + void *pCtx /* First argument passed to xConflict */ +){ + int rc = SQLITE_OK; + + while( pApply->constraints.nBuf ){ + sqlite3_changeset_iter *pIter2 = 0; + SessionBuffer cons = pApply->constraints; + memset(&pApply->constraints, 0, sizeof(SessionBuffer)); + + rc = sessionChangesetStart(&pIter2, 0, 0, cons.nBuf, cons.aBuf); + if( rc==SQLITE_OK ){ + int nByte = 2*pApply->nCol*sizeof(sqlite3_value*); + int rc2; + pIter2->bPatchset = bPatchset; + pIter2->zTab = (char*)zTab; + pIter2->nCol = pApply->nCol; + pIter2->abPK = pApply->abPK; + sessionBufferGrow(&pIter2->tblhdr, nByte, &rc); + pIter2->apValue = (sqlite3_value**)pIter2->tblhdr.aBuf; + if( rc==SQLITE_OK ) memset(pIter2->apValue, 0, nByte); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter2) ){ + rc = sessionApplyOneWithRetry(db, pIter2, pApply, xConflict, pCtx); + } + + rc2 = sqlite3changeset_finalize(pIter2); + if( rc==SQLITE_OK ) rc = rc2; + } + assert( pApply->bDeferConstraints || pApply->constraints.nBuf==0 ); + + sqlite3_free(cons.aBuf); + if( rc!=SQLITE_OK ) break; + if( pApply->constraints.nBuf>=cons.nBuf ){ + /* No progress was made on the last round. */ + pApply->bDeferConstraints = 0; + } + } + + return rc; +} + +/* +** Argument pIter is a changeset iterator that has been initialized, but +** not yet passed to sqlite3changeset_next(). This function applies the +** changeset to the main database attached to handle "db". The supplied +** conflict handler callback is invoked to resolve any conflicts encountered +** while applying the change. +*/ +static int sessionChangesetApply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + sqlite3_changeset_iter *pIter, /* Changeset to apply */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of fifth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +){ + int schemaMismatch = 0; + int rc; /* Return code */ + const char *zTab = 0; /* Name of current table */ + int nTab = 0; /* Result of sqlite3Strlen30(zTab) */ + SessionApplyCtx sApply; /* changeset_apply() context object */ + int bPatchset; + + assert( xConflict!=0 ); + + pIter->in.bNoDiscard = 1; + memset(&sApply, 0, sizeof(sApply)); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "PRAGMA defer_foreign_keys = 1", 0, 0, 0); + } + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter) ){ + int nCol; + int op; + const char *zNew; + + sqlite3changeset_op(pIter, &zNew, &nCol, &op, 0); + + if( zTab==0 || sqlite3_strnicmp(zNew, zTab, nTab+1) ){ + u8 *abPK; + + rc = sessionRetryConstraints( + db, pIter->bPatchset, zTab, &sApply, xConflict, pCtx + ); + if( rc!=SQLITE_OK ) break; + + sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ + sqlite3_finalize(sApply.pDelete); + sqlite3_finalize(sApply.pUpdate); + sqlite3_finalize(sApply.pInsert); + sqlite3_finalize(sApply.pSelect); + memset(&sApply, 0, sizeof(sApply)); + sApply.db = db; + sApply.bDeferConstraints = 1; + + /* If an xFilter() callback was specified, invoke it now. If the + ** xFilter callback returns zero, skip this table. If it returns + ** non-zero, proceed. */ + schemaMismatch = (xFilter && (0==xFilter(pCtx, zNew))); + if( schemaMismatch ){ + zTab = sqlite3_mprintf("%s", zNew); + if( zTab==0 ){ + rc = SQLITE_NOMEM; + break; + } + nTab = (int)strlen(zTab); + sApply.azCol = (const char **)zTab; + }else{ + sqlite3changeset_pk(pIter, &abPK, 0); + rc = sessionTableInfo( + db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + ); + if( rc!=SQLITE_OK ) break; + + if( sApply.nCol==0 ){ + schemaMismatch = 1; + sqlite3_log(SQLITE_SCHEMA, + "sqlite3changeset_apply(): no such table: %s", zTab + ); + } + else if( sApply.nCol!=nCol ){ + schemaMismatch = 1; + sqlite3_log(SQLITE_SCHEMA, + "sqlite3changeset_apply(): table %s has %d columns, expected %d", + zTab, sApply.nCol, nCol + ); + } + else if( memcmp(sApply.abPK, abPK, nCol)!=0 ){ + schemaMismatch = 1; + sqlite3_log(SQLITE_SCHEMA, "sqlite3changeset_apply(): " + "primary key mismatch for table %s", zTab + ); + } + else if( + (rc = sessionSelectRow(db, zTab, &sApply)) + || (rc = sessionUpdateRow(db, zTab, &sApply)) + || (rc = sessionDeleteRow(db, zTab, &sApply)) + || (rc = sessionInsertRow(db, zTab, &sApply)) + ){ + break; + } + nTab = sqlite3Strlen30(zTab); + } + } + + /* If there is a schema mismatch on the current table, proceed to the + ** next change. A log message has already been issued. */ + if( schemaMismatch ) continue; + + rc = sessionApplyOneWithRetry(db, pIter, &sApply, xConflict, pCtx); + } + + bPatchset = pIter->bPatchset; + if( rc==SQLITE_OK ){ + rc = sqlite3changeset_finalize(pIter); + }else{ + sqlite3changeset_finalize(pIter); + } + + if( rc==SQLITE_OK ){ + rc = sessionRetryConstraints(db, bPatchset, zTab, &sApply, xConflict, pCtx); + } + + if( rc==SQLITE_OK ){ + int nFk, notUsed; + sqlite3_db_status(db, SQLITE_DBSTATUS_DEFERRED_FKS, &nFk, ¬Used, 0); + if( nFk!=0 ){ + int res = SQLITE_CHANGESET_ABORT; + sqlite3_changeset_iter sIter; + memset(&sIter, 0, sizeof(sIter)); + sIter.nCol = nFk; + res = xConflict(pCtx, SQLITE_CHANGESET_FOREIGN_KEY, &sIter); + if( res!=SQLITE_CHANGESET_OMIT ){ + rc = SQLITE_CONSTRAINT; + } + } + } + sqlite3_exec(db, "PRAGMA defer_foreign_keys = 0", 0, 0, 0); + + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0); + }else{ + sqlite3_exec(db, "ROLLBACK TO changeset_apply", 0, 0, 0); + sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0); + } + + sqlite3_finalize(sApply.pInsert); + sqlite3_finalize(sApply.pDelete); + sqlite3_finalize(sApply.pUpdate); + sqlite3_finalize(sApply.pSelect); + sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ + sqlite3_free((char*)sApply.constraints.aBuf); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; +} + +/* +** Apply the changeset passed via pChangeset/nChangeset to the main database +** attached to handle "db". Invoke the supplied conflict handler callback +** to resolve any conflicts encountered while applying the change. +*/ +int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of fifth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx); + } + return rc; +} + +/* +** Apply the changeset passed via xInput/pIn to the main database +** attached to handle "db". Invoke the supplied conflict handler callback +** to resolve any conflicts encountered while applying the change. +*/ +int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx); + } + return rc; +} + +/* +** sqlite3_changegroup handle. +*/ +struct sqlite3_changegroup { + int rc; /* Error code */ + int bPatch; /* True to accumulate patchsets */ + SessionTable *pList; /* List of tables in current patch */ +}; + +/* +** This function is called to merge two changes to the same row together as +** part of an sqlite3changeset_concat() operation. A new change object is +** allocated and a pointer to it stored in *ppNew. +*/ +static int sessionChangeMerge( + SessionTable *pTab, /* Table structure */ + int bPatchset, /* True for patchsets */ + SessionChange *pExist, /* Existing change */ + int op2, /* Second change operation */ + int bIndirect, /* True if second change is indirect */ + u8 *aRec, /* Second change record */ + int nRec, /* Number of bytes in aRec */ + SessionChange **ppNew /* OUT: Merged change */ +){ + SessionChange *pNew = 0; + + if( !pExist ){ + pNew = (SessionChange *)sqlite3_malloc(sizeof(SessionChange) + nRec); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SessionChange)); + pNew->op = op2; + pNew->bIndirect = bIndirect; + pNew->nRecord = nRec; + pNew->aRecord = (u8*)&pNew[1]; + memcpy(pNew->aRecord, aRec, nRec); + }else{ + int op1 = pExist->op; + + /* + ** op1=INSERT, op2=INSERT -> Unsupported. Discard op2. + ** op1=INSERT, op2=UPDATE -> INSERT. + ** op1=INSERT, op2=DELETE -> (none) + ** + ** op1=UPDATE, op2=INSERT -> Unsupported. Discard op2. + ** op1=UPDATE, op2=UPDATE -> UPDATE. + ** op1=UPDATE, op2=DELETE -> DELETE. + ** + ** op1=DELETE, op2=INSERT -> UPDATE. + ** op1=DELETE, op2=UPDATE -> Unsupported. Discard op2. + ** op1=DELETE, op2=DELETE -> Unsupported. Discard op2. + */ + if( (op1==SQLITE_INSERT && op2==SQLITE_INSERT) + || (op1==SQLITE_UPDATE && op2==SQLITE_INSERT) + || (op1==SQLITE_DELETE && op2==SQLITE_UPDATE) + || (op1==SQLITE_DELETE && op2==SQLITE_DELETE) + ){ + pNew = pExist; + }else if( op1==SQLITE_INSERT && op2==SQLITE_DELETE ){ + sqlite3_free(pExist); + assert( pNew==0 ); + }else{ + u8 *aExist = pExist->aRecord; + int nByte; + u8 *aCsr; + + /* Allocate a new SessionChange object. Ensure that the aRecord[] + ** buffer of the new object is large enough to hold any record that + ** may be generated by combining the input records. */ + nByte = sizeof(SessionChange) + pExist->nRecord + nRec; + pNew = (SessionChange *)sqlite3_malloc(nByte); + if( !pNew ){ + sqlite3_free(pExist); + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SessionChange)); + pNew->bIndirect = (bIndirect && pExist->bIndirect); + aCsr = pNew->aRecord = (u8 *)&pNew[1]; + + if( op1==SQLITE_INSERT ){ /* INSERT + UPDATE */ + u8 *a1 = aRec; + assert( op2==SQLITE_UPDATE ); + pNew->op = SQLITE_INSERT; + if( bPatchset==0 ) sessionSkipRecord(&a1, pTab->nCol); + sessionMergeRecord(&aCsr, pTab->nCol, aExist, a1); + }else if( op1==SQLITE_DELETE ){ /* DELETE + INSERT */ + assert( op2==SQLITE_INSERT ); + pNew->op = SQLITE_UPDATE; + if( bPatchset ){ + memcpy(aCsr, aRec, nRec); + aCsr += nRec; + }else{ + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist, 0,aRec,0) ){ + sqlite3_free(pNew); + pNew = 0; + } + } + }else if( op2==SQLITE_UPDATE ){ /* UPDATE + UPDATE */ + u8 *a1 = aExist; + u8 *a2 = aRec; + assert( op1==SQLITE_UPDATE ); + if( bPatchset==0 ){ + sessionSkipRecord(&a1, pTab->nCol); + sessionSkipRecord(&a2, pTab->nCol); + } + pNew->op = SQLITE_UPDATE; + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aRec, aExist,a1,a2) ){ + sqlite3_free(pNew); + pNew = 0; + } + }else{ /* UPDATE + DELETE */ + assert( op1==SQLITE_UPDATE && op2==SQLITE_DELETE ); + pNew->op = SQLITE_DELETE; + if( bPatchset ){ + memcpy(aCsr, aRec, nRec); + aCsr += nRec; + }else{ + sessionMergeRecord(&aCsr, pTab->nCol, aRec, aExist); + } + } + + if( pNew ){ + pNew->nRecord = (int)(aCsr - pNew->aRecord); + } + sqlite3_free(pExist); + } + } + + *ppNew = pNew; + return SQLITE_OK; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp /* Changegroup object to add changeset to */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + SessionTable *pTab = 0; + + + while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec) ){ + const char *zNew; + int nCol; + int op; + int iHash; + int bIndirect; + SessionChange *pChange; + SessionChange *pExist = 0; + SessionChange **pp; + + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + break; + } + + sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); + if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ + /* Search the list for a matching table */ + int nNew = (int)strlen(zNew); + u8 *abPK; + + sqlite3changeset_pk(pIter, &abPK, 0); + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; + } + if( !pTab ){ + SessionTable **ppTab; + + pTab = sqlite3_malloc(sizeof(SessionTable) + nCol + nNew+1); + if( !pTab ){ + rc = SQLITE_NOMEM; + break; + } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zNew, nNew+1); + + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); + *ppTab = pTab; + }else if( pTab->nCol!=nCol || memcmp(pTab->abPK, abPK, nCol) ){ + rc = SQLITE_SCHEMA; + break; + } + } + + if( sessionGrowHash(pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + break; + } + iHash = sessionChangeHash( + pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange + ); + + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. + */ + for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ + int bPkOnly1 = 0; + int bPkOnly2 = 0; + if( pIter->bPatchset ){ + bPkOnly1 = (*pp)->op==SQLITE_DELETE; + bPkOnly2 = op==SQLITE_DELETE; + } + if( sessionChangeEqual(pTab, bPkOnly1, (*pp)->aRecord, bPkOnly2, aRec) ){ + pExist = *pp; + *pp = (*pp)->pNext; + pTab->nEntry--; + break; + } + } + + rc = sessionChangeMerge(pTab, + pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange + ); + if( rc ) break; + if( pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Serialize a changeset (or patchset) based on all changesets (or patchsets) +** added to the changegroup object passed as the first argument. +** +** If xOutput is not NULL, then the changeset/patchset is returned to the +** user via one or more calls to xOutput, as with the other streaming +** interfaces. +** +** Or, if xOutput is NULL, then (*ppOut) is populated with a pointer to a +** buffer containing the output changeset before this function returns. In +** this case (*pnOut) is set to the size of the output buffer in bytes. It +** is the responsibility of the caller to free the output buffer using +** sqlite3_free() when it is no longer required. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an SQLite +** error code. If an error occurs and xOutput is NULL, (*ppOut) and (*pnOut) +** are both set to 0 before returning. +*/ +static int sessionChangegroupOutput( + sqlite3_changegroup *pGrp, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, + int *pnOut, + void **ppOut +){ + int rc = SQLITE_OK; + SessionBuffer buf = {0, 0, 0}; + SessionTable *pTab; + assert( xOutput==0 || (ppOut==0 && pnOut==0) ); + + /* Create the serialized output changeset based on the contents of the + ** hash tables attached to the SessionTable objects in list p->pList. + */ + for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ + int i; + if( pTab->nEntry==0 ) continue; + + sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc); + for(i=0; inChange; i++){ + SessionChange *p; + for(p=pTab->apChange[i]; p; p=p->pNext){ + sessionAppendByte(&buf, p->op, &rc); + sessionAppendByte(&buf, p->bIndirect, &rc); + sessionAppendBlob(&buf, p->aRecord, p->nRecord, &rc); + } + } + + if( rc==SQLITE_OK && xOutput && buf.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){ + rc = xOutput(pOut, buf.aBuf, buf.nBuf); + buf.nBuf = 0; + } + } + + if( rc==SQLITE_OK ){ + if( xOutput ){ + if( buf.nBuf>0 ) rc = xOutput(pOut, buf.aBuf, buf.nBuf); + }else{ + *ppOut = buf.aBuf; + *pnOut = buf.nBuf; + buf.aBuf = 0; + } + } + sqlite3_free(buf.aBuf); + + return rc; +} + +/* +** Allocate a new, empty, sqlite3_changegroup. +*/ +int sqlite3changegroup_new(sqlite3_changegroup **pp){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_changegroup *p; /* New object */ + p = (sqlite3_changegroup*)sqlite3_malloc(sizeof(sqlite3_changegroup)); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(p, 0, sizeof(sqlite3_changegroup)); + } + *pp = p; + return rc; +} + +/* +** Add the changeset currently stored in buffer pData, size nData bytes, +** to changeset-group p. +*/ +int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void *pData){ + sqlite3_changeset_iter *pIter; /* Iterator opened on pData/nData */ + int rc; /* Return code */ + + rc = sqlite3changeset_start(&pIter, nData, pData); + if( rc==SQLITE_OK ){ + rc = sessionChangesetToHash(pIter, pGrp); + } + sqlite3changeset_finalize(pIter); + return rc; +} + +/* +** Obtain a buffer containing a changeset representing the concatenation +** of all changesets added to the group so far. +*/ +int sqlite3changegroup_output( + sqlite3_changegroup *pGrp, + int *pnData, + void **ppData +){ + return sessionChangegroupOutput(pGrp, 0, 0, pnData, ppData); +} + +/* +** Streaming versions of changegroup_add(). +*/ +int sqlite3changegroup_add_strm( + sqlite3_changegroup *pGrp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +){ + sqlite3_changeset_iter *pIter; /* Iterator opened on pData/nData */ + int rc; /* Return code */ + + rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); + if( rc==SQLITE_OK ){ + rc = sessionChangesetToHash(pIter, pGrp); + } + sqlite3changeset_finalize(pIter); + return rc; +} + +/* +** Streaming versions of changegroup_output(). +*/ +int sqlite3changegroup_output_strm( + sqlite3_changegroup *pGrp, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + return sessionChangegroupOutput(pGrp, xOutput, pOut, 0, 0); +} + +/* +** Delete a changegroup object. +*/ +void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ + if( pGrp ){ + sessionDeleteTable(pGrp->pList); + sqlite3_free(pGrp); + } +} + +/* +** Combine two changesets together. +*/ +int sqlite3changeset_concat( + int nLeft, /* Number of bytes in lhs input */ + void *pLeft, /* Lhs input changeset */ + int nRight /* Number of bytes in rhs input */, + void *pRight, /* Rhs input changeset */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: changeset (left right) */ +){ + sqlite3_changegroup *pGrp; + int rc; + + rc = sqlite3changegroup_new(&pGrp); + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add(pGrp, nLeft, pLeft); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add(pGrp, nRight, pRight); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_output(pGrp, pnOut, ppOut); + } + sqlite3changegroup_delete(pGrp); + + return rc; +} + +/* +** Streaming version of sqlite3changeset_concat(). +*/ +int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + sqlite3_changegroup *pGrp; + int rc; + + rc = sqlite3changegroup_new(&pGrp); + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add_strm(pGrp, xInputA, pInA); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add_strm(pGrp, xInputB, pInB); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_output_strm(pGrp, xOutput, pOut); + } + sqlite3changegroup_delete(pGrp); + + return rc; +} + +#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h new file mode 100644 index 00000000..c995d4ec --- /dev/null +++ b/ext/session/sqlite3session.h @@ -0,0 +1,1281 @@ + +#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) +#define __SQLITESESSION_H_ 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +extern "C" { +#endif + +#include "sqlite3.h" + +/* +** CAPI3REF: Session Object Handle +*/ +typedef struct sqlite3_session sqlite3_session; + +/* +** CAPI3REF: Changeset Iterator Handle +*/ +typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; + +/* +** CAPI3REF: Create A New Session Object +** +** Create a new session object attached to database handle db. If successful, +** a pointer to the new object is written to *ppSession and SQLITE_OK is +** returned. If an error occurs, *ppSession is set to NULL and an SQLite +** error code (e.g. SQLITE_NOMEM) is returned. +** +** It is possible to create multiple session objects attached to a single +** database handle. +** +** Session objects created using this function should be deleted using the +** [sqlite3session_delete()] function before the database handle that they +** are attached to is itself closed. If the database handle is closed before +** the session object is deleted, then the results of calling any session +** module function, including [sqlite3session_delete()] on the session object +** are undefined. +** +** Because the session module uses the [sqlite3_preupdate_hook()] API, it +** is not possible for an application to register a pre-update hook on a +** database handle that has one or more session objects attached. Nor is +** it possible to create a session object attached to a database handle for +** which a pre-update hook is already defined. The results of attempting +** either of these things are undefined. +** +** The session object will be used to create changesets for tables in +** database zDb, where zDb is either "main", or "temp", or the name of an +** attached database. It is not an error if database zDb is not attached +** to the database when the session object is created. +*/ +int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +); + +/* +** CAPI3REF: Delete A Session Object +** +** Delete a session object previously allocated using +** [sqlite3session_create()]. Once a session object has been deleted, the +** results of attempting to use pSession with any other session module +** function are undefined. +** +** Session objects must be deleted before the database handle to which they +** are attached is closed. Refer to the documentation for +** [sqlite3session_create()] for details. +*/ +void sqlite3session_delete(sqlite3_session *pSession); + + +/* +** CAPI3REF: Enable Or Disable A Session Object +** +** Enable or disable the recording of changes by a session object. When +** enabled, a session object records changes made to the database. When +** disabled - it does not. A newly created session object is enabled. +** Refer to the documentation for [sqlite3session_changeset()] for further +** details regarding how enabling and disabling a session object affects +** the eventual changesets. +** +** Passing zero to this function disables the session. Passing a value +** greater than zero enables it. Passing a value less than zero is a +** no-op, and may be used to query the current state of the session. +** +** The return value indicates the final state of the session object: 0 if +** the session is disabled, or 1 if it is enabled. +*/ +int sqlite3session_enable(sqlite3_session *pSession, int bEnable); + +/* +** CAPI3REF: Set Or Clear the Indirect Change Flag +** +** Each change recorded by a session object is marked as either direct or +** indirect. A change is marked as indirect if either: +** +**
      +**
    • The session object "indirect" flag is set when the change is +** made, or +**
    • The change is made by an SQL trigger or foreign key action +** instead of directly as a result of a users SQL statement. +**
    +** +** If a single row is affected by more than one operation within a session, +** then the change is considered indirect if all operations meet the criteria +** for an indirect change above, or direct otherwise. +** +** This function is used to set, clear or query the session object indirect +** flag. If the second argument passed to this function is zero, then the +** indirect flag is cleared. If it is greater than zero, the indirect flag +** is set. Passing a value less than zero does not modify the current value +** of the indirect flag, and may be used to query the current state of the +** indirect flag for the specified session object. +** +** The return value indicates the final state of the indirect flag: 0 if +** it is clear, or 1 if it is set. +*/ +int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); + +/* +** CAPI3REF: Attach A Table To A Session Object +** +** If argument zTab is not NULL, then it is the name of a table to attach +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See +** documentation for [sqlite3session_changeset()] for further details. +** +** Or, if argument zTab is NULL, then changes are recorded for all tables +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for +** the new tables are also recorded. +** +** Changes can only be recorded for tables that have a PRIMARY KEY explicitly +** defined as part of their CREATE TABLE statement. It does not matter if the +** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY +** KEY may consist of a single column, or may be a composite key. +** +** It is not an error if the named table does not exist in the database. Nor +** is it an error if the named table does not have a PRIMARY KEY. However, +** no changes will be recorded in either of these scenarios. +** +** Changes are not recorded for individual rows that have NULL values stored +** in one or more of their PRIMARY KEY columns. +** +** SQLITE_OK is returned if the call completes without error. Or, if an error +** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +*/ +int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zTab /* Table name */ +); + +/* +** CAPI3REF: Set a table filter on a Session Object. +** +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes is not tracked. Note that once a table is +** attached, xFilter will not be called again. +*/ +void sqlite3session_table_filter( + sqlite3_session *pSession, /* Session object */ + int(*xFilter)( + void *pCtx, /* Copy of third arg to _filter_table() */ + const char *zTab /* Table name */ + ), + void *pCtx /* First argument passed to xFilter */ +); + +/* +** CAPI3REF: Generate A Changeset From A Session Object +** +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset +** and *pnChangeset to the size of the changeset in bytes before returning +** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to +** zero and return an SQLite error code. +** +** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, +** each representing a change to a single row of an attached table. An INSERT +** change contains the values of each field of a new database row. A DELETE +** contains the original values of each field of a deleted database row. An +** UPDATE change contains the original values of each field of an updated +** database row along with the updated values for each updated non-primary-key +** column. It is not possible for an UPDATE change to represent a change that +** modifies the values of primary key columns. If such a change is made, it +** is represented in a changeset as a DELETE followed by an INSERT. +** +** Changes are not recorded for rows that have NULL values stored in one or +** more of their PRIMARY KEY columns. If such a row is inserted or deleted, +** no corresponding change is present in the changesets returned by this +** function. If an existing row with one or more NULL values stored in +** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, +** only an INSERT is appears in the changeset. Similarly, if an existing row +** with non-NULL PRIMARY KEY values is updated so that one or more of its +** PRIMARY KEY columns are set to NULL, the resulting changeset contains a +** DELETE change only. +** +** The contents of a changeset may be traversed using an iterator created +** using the [sqlite3changeset_start()] API. A changeset may be applied to +** a database with a compatible schema using the [sqlite3changeset_apply()] +** API. +** +** Within a changeset generated by this function, all changes related to a +** single table are grouped together. In other words, when iterating through +** a changeset or when applying a changeset to a database, all changes related +** to a single table are processed before moving on to the next table. Tables +** are sorted in the same order in which they were attached (or auto-attached) +** to the sqlite3_session object. The order in which the changes related to +** a single table are stored is undefined. +** +** Following a successful call to this function, it is the responsibility of +** the caller to eventually free the buffer that *ppChangeset points to using +** [sqlite3_free()]. +** +**

    Changeset Generation

    +** +** Once a table has been attached to a session object, the session object +** records the primary key values of all new rows inserted into the table. +** It also records the original primary key and other column values of any +** deleted or updated rows. For each unique primary key value, data is only +** recorded once - the first time a row with said primary key is inserted, +** updated or deleted in the lifetime of the session. +** +** There is one exception to the previous paragraph: when a row is inserted, +** updated or deleted, if one or more of its primary key columns contain a +** NULL value, no record of the change is made. +** +** The session object therefore accumulates two types of records - those +** that consist of primary key values only (created when the user inserts +** a new record) and those that consist of the primary key values and the +** original values of other table columns (created when the users deletes +** or updates a record). +** +** When this function is called, the requested changeset is created using +** both the accumulated records and the current contents of the database +** file. Specifically: +** +**
      +**
    • For each record generated by an insert, the database is queried +** for a row with a matching primary key. If one is found, an INSERT +** change is added to the changeset. If no such row is found, no change +** is added to the changeset. +** +**
    • For each record generated by an update or delete, the database is +** queried for a row with a matching primary key. If such a row is +** found and one or more of the non-primary key fields have been +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE +** change is added to the changeset. If there is a row with a matching +** primary key in the database, but all fields contain their original +** values, no change is added to the changeset. +**
    +** +** This means, amongst other things, that if a row is inserted and then later +** deleted while a session object is active, neither the insert nor the delete +** will be present in the changeset. Or if a row is deleted and then later a +** row with the same primary key values inserted while a session object is +** active, the resulting changeset will contain an UPDATE change instead of +** a DELETE and an INSERT. +** +** When a session object is disabled (see the [sqlite3session_enable()] API), +** it does not accumulate records when rows are inserted, updated or deleted. +** This may appear to have some counter-intuitive effects if a single row +** is written to more than once during a session. For example, if a row +** is inserted while a session object is enabled, then later deleted while +** the same session object is disabled, no INSERT record will appear in the +** changeset, even though the delete took place while the session was disabled. +** Or, if one field of a row is updated while a session is disabled, and +** another field of the same row is updated while the session is enabled, the +** resulting changeset will contain an UPDATE change that updates both fields. +*/ +int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** +** If it is not already attached to the session object passed as the first +** argument, this function attaches table zTbl in the same manner as the +** [sqlite3session_attach()] function. If zTbl does not exist, or if it +** does not have a primary key, this function is a no-op (but does not return +** an error). +** +** Argument zFromDb must be the name of a database ("main", "temp" etc.) +** attached to the same database handle as the session object that contains +** a table compatible with the table attached to the session by this function. +** A table is considered compatible if it: +** +**
      +**
    • Has the same name, +**
    • Has the same set of columns declared in the same order, and +**
    • Has the same PRIMARY KEY definition. +**
    +** +** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables +** are compatible but do not have any PRIMARY KEY columns, it is not an error +** but no changes are added to the session object. As with other session +** APIs, tables without PRIMARY KEYs are simply ignored. +** +** This function adds a set of changes to the session object that could be +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session +** object (call this the "to-table"). Specifically: +** +**
      +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, an INSERT record is added to the session object. +** +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, a DELETE record is added to the session object. +** +**
    • For each row (primary key) that exists in both tables, but features +** different in each, an UPDATE record is added to the session. +**
    +** +** To clarify, if this function is called and then a changeset constructed +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be +** identical. +** +** It an error if database zFrom does not exist or does not contain the +** required compatible table. +** +** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite +** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to free this buffer using +** sqlite3_free(). +*/ +int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFromDb, + const char *zTbl, + char **pzErrMsg +); + + +/* +** CAPI3REF: Generate A Patchset From A Session Object +** +** The differences between a patchset and a changeset are that: +** +**
      +**
    • DELETE records consist of the primary key fields only. The +** original values of other fields are omitted. +**
    • The original values of any modified fields are omitted from +** UPDATE records. +**
    +** +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, +** attempting to use a patchset blob with old versions of the +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** +** Because the non-primary key "old.*" fields are omitted, no +** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset +** is passed to the sqlite3changeset_apply() API. Other conflict types work +** in the same way as for changesets. +** +** Changes within a patchset are ordered in the same way as for changesets +** generated by the sqlite3session_changeset() function (i.e. all changes for +** a single table are grouped together, tables appear in the order in which +** they were attached to the session object). +*/ +int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ + void **ppPatchset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Test if a changeset has recorded any changes. +** +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or +** more changes have been recorded, return zero. +** +** Even if this function returns zero, it is possible that calling +** [sqlite3session_changeset()] on the session handle may still return a +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values +** are restored. However, if this function returns non-zero, then it is +** guaranteed that a call to sqlite3session_changeset() will return a +** changeset containing zero changes. +*/ +int sqlite3session_isempty(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** +** Create an iterator used to iterate through the contents of a changeset. +** If successful, *pp is set to point to the iterator handle and SQLITE_OK +** is returned. Otherwise, if an error occurs, *pp is set to zero and an +** SQLite error code is returned. +** +** The following functions can be used to advance and query a changeset +** iterator created by this function: +** +**
      +**
    • [sqlite3changeset_next()] +**
    • [sqlite3changeset_op()] +**
    • [sqlite3changeset_new()] +**
    • [sqlite3changeset_old()] +**
    +** +** It is the responsibility of the caller to eventually destroy the iterator +** by passing it to [sqlite3changeset_finalize()]. The buffer containing the +** changeset (pChangeset) must remain valid until after the iterator is +** destroyed. +** +** Assuming the changeset blob was created by one of the +** [sqlite3session_changeset()], [sqlite3changeset_concat()] or +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit +** another change for table X. +*/ +int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset /* Pointer to blob containing changeset */ +); + + +/* +** CAPI3REF: Advance A Changeset Iterator +** +** This function may only be used with iterators created by function +** [sqlite3changeset_start()]. If it is called on an iterator passed to +** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE +** is returned and the call has no effect. +** +** Immediately after an iterator is created by sqlite3changeset_start(), it +** does not point to any change in the changeset. Assuming the changeset +** is not empty, the first call to this function advances the iterator to +** point to the first change in the changeset. Each subsequent call advances +** the iterator to point to the next change in the changeset (if any). If +** no error occurs and the iterator points to a valid change after a call +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** Otherwise, if all changes in the changeset have already been visited, +** SQLITE_DONE is returned. +** +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** SQLITE_NOMEM. +*/ +int sqlite3changeset_next(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this +** is not the case, this function returns [SQLITE_MISUSE]. +** +** If argument pzTab is not NULL, then *pzTab is set to point to a +** nul-terminated utf-8 encoded string containing the name of the table +** affected by the current change. The buffer remains valid until either +** sqlite3changeset_next() is called on the iterator or until the +** conflict-handler function returns. If pnCol is not NULL, then *pnCol is +** set to the number of columns in the table affected by the change. If +** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change +** is an indirect change, or false (0) otherwise. See the documentation for +** [sqlite3session_indirect()] for a description of direct and indirect +** changes. Finally, if pOp is not NULL, then *pOp is set to one of +** [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], depending on the +** type of change that the iterator currently points to. +** +** If no error occurs, SQLITE_OK is returned. If an error does occur, an +** SQLite error code is returned. The values of the output variables may not +** be trusted in this case. +*/ +int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator object */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True for an 'indirect' change */ +); + +/* +** CAPI3REF: Obtain The Primary Key Definition Of A Table +** +** For each modified table, a changeset includes the following: +** +**
      +**
    • The number of columns in the table, and +**
    • Which of those columns make up the tables PRIMARY KEY. +**
    +** +** This function is used to find which columns comprise the PRIMARY KEY of +** the table modified by the change that iterator pIter currently points to. +** If successful, *pabPK is set to point to an array of nCol entries, where +** nCol is the number of columns in the table. Elements of *pabPK are set to +** 0x01 if the corresponding column is part of the tables primary key, or +** 0x00 if it is not. +** +** If argument pnCol is not NULL, then *pnCol is set to the number of columns +** in the table. +** +** If this function is called when the iterator does not point to a valid +** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, +** SQLITE_OK is returned and the output variables populated as described +** above. +*/ +int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +); + +/* +** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** original row values stored as part of the UPDATE or DELETE change and +** returns SQLITE_OK. The name of the function comes from the fact that this +** is similar to the "old.*" columns available to update or delete triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** new row values stored as part of the UPDATE or INSERT change and +** returns SQLITE_OK. If the change is an UPDATE and does not include +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete +** triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** +** This function should only be used with iterator objects passed to a +** conflict-handler callback by [sqlite3changeset_apply()] with either +** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function +** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue +** is set to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the +** "conflicting row" associated with the current conflict-handler callback +** and returns SQLITE_OK. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +); + +/* +** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +); + + +/* +** CAPI3REF: Finalize A Changeset Iterator +** +** This function is used to finalize an iterator allocated with +** [sqlite3changeset_start()]. +** +** This function should only be called on iterators created using the +** [sqlite3changeset_start()] function. If an application calls this +** function with an iterator passed to a conflict-handler by +** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the +** call has no effect. +** +** If an error was encountered within a call to an sqlite3changeset_xxx() +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding +** to that error is returned by this function. Otherwise, SQLITE_OK is +** returned. This is to allow the following pattern (pseudo-code): +** +** sqlite3changeset_start(); +** while( SQLITE_ROW==sqlite3changeset_next() ){ +** // Do something with change. +** } +** rc = sqlite3changeset_finalize(); +** if( rc!=SQLITE_OK ){ +** // An error has occurred +** } +*/ +int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Invert A Changeset +** +** This function is used to "invert" a changeset object. Applying an inverted +** changeset to a database reverses the effects of applying the uninverted +** changeset. Specifically: +** +**
      +**
    • Each DELETE change is changed to an INSERT, and +**
    • Each INSERT change is changed to a DELETE, and +**
    • For each UPDATE change, the old.* and new.* values are exchanged. +**
    +** +** This function does not change the order in which changes appear within +** the changeset. It merely reverses the sense of each individual change. +** +** If successful, a pointer to a buffer containing the inverted changeset +** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and +** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are +** zeroed and an SQLite error code returned. +** +** It is the responsibility of the caller to eventually call sqlite3_free() +** on the *ppOut pointer to free the buffer allocation following a successful +** call to this function. +** +** WARNING/TODO: This function currently assumes that the input is a valid +** changeset. If it is not, the results are undefined. +*/ +int sqlite3changeset_invert( + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + +/* +** CAPI3REF: Concatenate Two Changeset Objects +** +** This function is used to concatenate two changesets, A and B, into a +** single changeset. The result is a changeset equivalent to applying +** changeset A followed by changeset B. +** +** This function combines the two input changesets using an +** sqlite3_changegroup object. Calling it produces similar results as the +** following code fragment: +** +** sqlite3_changegroup *pGrp; +** rc = sqlite3_changegroup_new(&pGrp); +** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA); +** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB); +** if( rc==SQLITE_OK ){ +** rc = sqlite3changegroup_output(pGrp, pnOut, ppOut); +** }else{ +** *ppOut = 0; +** *pnOut = 0; +** } +** +** Refer to the sqlite3_changegroup documentation below for details. +*/ +int sqlite3changeset_concat( + int nA, /* Number of bytes in buffer pA */ + void *pA, /* Pointer to buffer containing changeset A */ + int nB, /* Number of bytes in buffer pB */ + void *pB, /* Pointer to buffer containing changeset B */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Buffer containing output changeset */ +); + + +/* +** CAPI3REF: Changegroup Handle +*/ +typedef struct sqlite3_changegroup sqlite3_changegroup; + +/* +** CAPI3REF: Create A New Changegroup Object +** +** An sqlite3_changegroup object is used to combine two or more changesets +** (or patchsets) into a single changeset (or patchset). A single changegroup +** object may combine changesets or patchsets, but not both. The output is +** always in the same format as the input. +** +** If successful, this function returns SQLITE_OK and populates (*pp) with +** a pointer to a new sqlite3_changegroup object before returning. The caller +** should eventually free the returned object using a call to +** sqlite3changegroup_delete(). If an error occurs, an SQLite error code +** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. +** +** The usual usage pattern for an sqlite3_changegroup object is as follows: +** +**
      +**
    • It is created using a call to sqlite3changegroup_new(). +** +**
    • Zero or more changesets (or patchsets) are added to the object +** by calling sqlite3changegroup_add(). +** +**
    • The result of combining all input changesets together is obtained +** by the application via a call to sqlite3changegroup_output(). +** +**
    • The object is deleted using a call to sqlite3changegroup_delete(). +**
    +** +** Any number of calls to add() and output() may be made between the calls to +** new() and delete(), and in any order. +** +** As well as the regular sqlite3changegroup_add() and +** sqlite3changegroup_output() functions, also available are the streaming +** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). +*/ +int sqlite3changegroup_new(sqlite3_changegroup **pp); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** +** Add all changes within the changeset (or patchset) in buffer pData (size +** nData bytes) to the changegroup. +** +** If the buffer contains a patchset, then all prior calls to this function +** on the same changegroup object must also have specified patchsets. Or, if +** the buffer contains a changeset, so must have the earlier calls to this +** function. Otherwise, SQLITE_ERROR is returned and no changes are added +** to the changegroup. +** +** Rows within the changeset and changegroup are identified by the values in +** their PRIMARY KEY columns. A change in the changeset is considered to +** apply to the same row as a change already present in the changegroup if +** the two rows have the same primary key. +** +** Changes to rows that do not already appear in the changegroup are +** simply copied into it. Or, if both the new changeset and the changegroup +** contain changes that apply to a single row, the final contents of the +** changegroup depends on the type of each change, as follows: +** +** +** +** +**
    Existing Change New Change Output Change +**
    INSERT INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    INSERT UPDATE +** The INSERT change remains in the changegroup. The values in the +** INSERT change are modified as if the row was inserted by the +** existing change and then updated according to the new change. +**
    INSERT DELETE +** The existing INSERT is removed from the changegroup. The DELETE is +** not added. +**
    UPDATE INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    UPDATE UPDATE +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once +** by the existing change and then again by the new change. +**
    UPDATE DELETE +** The existing UPDATE is replaced by the new DELETE within the +** changegroup. +**
    DELETE INSERT +** If one or more of the column values in the row inserted by the +** new change differ from those in the row deleted by the existing +** change, the existing DELETE is replaced by an UPDATE within the +** changegroup. Otherwise, if the inserted row is exactly the same +** as the deleted row, the existing DELETE is simply discarded. +**
    DELETE UPDATE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    DELETE DELETE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    +** +** If the new changeset contains changes to a table that is already present +** in the changegroup, then the number of columns and the position of the +** primary key columns for the table must be consistent. If this is not the +** case, this function fails with SQLITE_SCHEMA. If the input changeset +** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is +** returned. Or, if an out-of-memory condition occurs during processing, this +** function returns SQLITE_NOMEM. In all cases, if an error occurs the +** final contents of the changegroup is undefined. +** +** If no error occurs, SQLITE_OK is returned. +*/ +int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** +** Obtain a buffer containing a changeset (or patchset) representing the +** current contents of the changegroup. If the inputs to the changegroup +** were themselves changesets, the output is a changeset. Or, if the +** inputs were patchsets, the output is also a patchset. +** +** As with the output of the sqlite3session_changeset() and +** sqlite3session_patchset() functions, all changes related to a single +** table are grouped together in the output of this function. Tables appear +** in the same order as for the very first changeset added to the changegroup. +** If the second or subsequent changesets added to the changegroup contain +** changes for tables that do not appear in the first changeset, they are +** appended onto the end of the output changeset, again in the order in +** which they are first encountered. +** +** If an error occurs, an SQLite error code is returned and the output +** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK +** is returned and the output variables are set to the size of and a +** pointer to the output buffer, respectively. In this case it is the +** responsibility of the caller to eventually free the buffer using a +** call to sqlite3_free(). +*/ +int sqlite3changegroup_output( + sqlite3_changegroup*, + int *pnData, /* OUT: Size of output buffer in bytes */ + void **ppData /* OUT: Pointer to output buffer */ +); + +/* +** CAPI3REF: Delete A Changegroup Object +*/ +void sqlite3changegroup_delete(sqlite3_changegroup*); + +/* +** CAPI3REF: Apply A Changeset To A Database +** +** Apply a changeset to a database. This function attempts to update the +** "main" database attached to handle db with the changes found in the +** changeset passed via the second and third arguments. +** +** The fourth argument (xFilter) passed to this function is the "filter +** callback". If it is not NULL, then for each table affected by at least one +** change in the changeset, the filter callback is invoked with +** the table name as the second argument, and a copy of the context pointer +** passed as the sixth argument to this function as the first. If the "filter +** callback" returns zero, then no attempt is made to apply any changes to +** the table. Otherwise, if the return value is non-zero or the xFilter +** argument to this function is NULL, all changes related to the table are +** attempted. +** +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is +** considered compatible if all of the following are true: +** +**
      +**
    • The table has the same name as the name recorded in the +** changeset, and +**
    • The table has the same number of columns as recorded in the +** changeset, and +**
    • The table has primary key columns in the same position as +** recorded in the changeset. +**
    +** +** If there is no compatible table, it is not an error, but none of the +** changes associated with the table are applied. A warning message is issued +** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most +** one such warning is issued for each table in the changeset. +** +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for +** each type of change is below. +** +** Unlike the xFilter argument, xConflict may not be passed NULL. The results +** of passing anything other than a valid function pointer as the xConflict +** argument are undefined. +** +** Each time the conflict handler function is invoked, it must return one +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned +** if the second argument passed to the conflict handler is either +** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler +** returns an illegal value, any changes already made are rolled back and +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** actions are taken by sqlite3changeset_apply() depending on the value +** returned by each invocation of the conflict-handler function. Refer to +** the documentation for the three +** [SQLITE_CHANGESET_OMIT|available return values] for details. +** +**
    +**
    DELETE Changes
    +** For each DELETE change, this function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is deleted from the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from the original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT +** (which can only happen if a foreign key constraint is violated), the +** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] +** passed as the second argument. This includes the case where the DELETE +** operation is attempted because an earlier call to the conflict handler +** function returned [SQLITE_CHANGESET_REPLACE]. +** +**
    INSERT Changes
    +** For each INSERT change, an attempt is made to insert the new row into +** the database. +** +** If the attempt to insert the row fails because the database already +** contains a row with the same primary key values, the conflict handler +** function is invoked with the second argument set to +** [SQLITE_CHANGESET_CONFLICT]. +** +** If the attempt to insert the row fails because of some other constraint +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +** +**
    UPDATE Changes
    +** For each UPDATE change, this function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is updated within the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from an original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** UPDATE changes only contain values for non-primary key fields that are +** to be modified, only those fields need to match the original values to +** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. +** This includes the case where the UPDATE operation is attempted after +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +**
    +** +** It is safe to execute SQL statements, including those that write to the +** table that the callback related to, from within the xConflict callback. +** This can be used to further customize the applications conflict +** resolution strategy. +** +** All changes made by this function are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. +*/ +int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); + +/* +** CAPI3REF: Constants Passed To The Conflict Handler +** +** Values that may be passed as the second argument to a conflict-handler. +** +**
    +**
    SQLITE_CHANGESET_DATA
    +** The conflict handler is invoked with CHANGESET_DATA as the second argument +** when processing a DELETE or UPDATE change if a row with the required +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the +** expected "before" values. +** +** The conflicting row, in this case, is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_NOTFOUND
    +** The conflict handler is invoked with CHANGESET_NOTFOUND as the second +** argument when processing a DELETE or UPDATE change if a row with the +** required PRIMARY KEY fields is not present in the database. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    SQLITE_CHANGESET_CONFLICT
    +** CHANGESET_CONFLICT is passed as the second argument to the conflict +** handler while processing an INSERT change if the operation would result +** in duplicate primary key values. +** +** The conflicting row in this case is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_FOREIGN_KEY
    +** If foreign key handling is enabled, and applying a changeset leaves the +** database in a state containing foreign key violations, the conflict +** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument +** exactly once before the changeset is committed. If the conflict handler +** returns CHANGESET_OMIT, the changes, including those that caused the +** foreign key constraint violation, are committed. Or, if it returns +** CHANGESET_ABORT, the changeset is rolled back. +** +** No current or conflicting row information is provided. The only function +** it is possible to call on the supplied sqlite3_changeset_iter handle +** is sqlite3changeset_fk_conflicts(). +** +**
    SQLITE_CHANGESET_CONSTRAINT
    +** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** invoked with CHANGESET_CONSTRAINT as the second argument. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    +*/ +#define SQLITE_CHANGESET_DATA 1 +#define SQLITE_CHANGESET_NOTFOUND 2 +#define SQLITE_CHANGESET_CONFLICT 3 +#define SQLITE_CHANGESET_CONSTRAINT 4 +#define SQLITE_CHANGESET_FOREIGN_KEY 5 + +/* +** CAPI3REF: Constants Returned By The Conflict Handler +** +** A conflict handler callback must return one of the following three values. +** +**
    +**
    SQLITE_CHANGESET_OMIT
    +** If a conflict handler returns this value no special action is taken. The +** change that caused the conflict is not applied. The session module +** continues to the next change in the changeset. +** +**
    SQLITE_CHANGESET_REPLACE
    +** This value may only be returned if the second argument to the conflict +** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this +** is not the case, any changes applied so far are rolled back and the +** call to sqlite3changeset_apply() returns SQLITE_MISUSE. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict +** handler, then the conflicting row is either updated or deleted, depending +** on the type of change. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict +** handler, then the conflicting row is removed from the database and a +** second attempt to apply the change is made. If this second attempt fails, +** the original row is restored to the database before continuing. +** +**
    SQLITE_CHANGESET_ABORT
    +** If this value is returned, any changes applied so far are rolled back +** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. +**
    +*/ +#define SQLITE_CHANGESET_OMIT 0 +#define SQLITE_CHANGESET_REPLACE 1 +#define SQLITE_CHANGESET_ABORT 2 + +/* +** CAPI3REF: Streaming Versions of API functions. +** +** The six streaming API xxx_strm() functions serve similar purposes to the +** corresponding non-streaming API functions: +** +** +** +**
    Streaming functionNon-streaming equivalent
    sqlite3changeset_apply_str[sqlite3changeset_apply] +**
    sqlite3changeset_concat_str[sqlite3changeset_concat] +**
    sqlite3changeset_invert_str[sqlite3changeset_invert] +**
    sqlite3changeset_start_str[sqlite3changeset_start] +**
    sqlite3session_changeset_str[sqlite3session_changeset] +**
    sqlite3session_patchset_str[sqlite3session_patchset] +**
    +** +** Non-streaming functions that accept changesets (or patchsets) as input +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a +** low-memory environment is required to handle very large changesets, the +** large contiguous memory allocations required can become onerous. +** +** In order to avoid this problem, instead of a single large buffer, input +** is passed to a streaming API functions by way of a callback function that +** the sessions module invokes to incrementally request input data as it is +** required. In all cases, a pair of API function parameters such as +** +** 
    +**        int nChangeset,
+**        void *pChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xInput)(void *pIn, void *pData, int *pnData),
+**        void *pIn,
+**
    +** +** Each time the xInput callback is invoked by the sessions module, the first +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** error code should be returned. In all cases, if an xInput callback returns +** an error, all processing is abandoned and the streaming API function +** returns a copy of the error code to the caller. +** +** In the case of sqlite3changeset_start_strm(), the xInput callback may be +** invoked by the sessions module at any point during the lifetime of the +** iterator. If such an xInput callback returns an error, the iterator enters +** an error state, whereby all subsequent calls to iterator functions +** immediately fail with the same error code as returned by xInput. +** +** Similarly, streaming API functions that return changesets (or patchsets) +** return them in chunks by way of a callback function instead of via a +** pointer to a single large buffer. In this case, a pair of parameters such +** as: +** +** 
    +**        int *pnChangeset,
+**        void **ppChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xOutput)(void *pOut, const void *pData, int nData),
+**        void *pOut
+**
    +** +** The xOutput callback is invoked zero or more times to return data to +** the application. The first parameter passed to each call is a copy of the +** pOut pointer supplied by the application. The second parameter, pData, +** points to a buffer nData bytes in size containing the chunk of output +** data being returned. If the xOutput callback successfully processes the +** supplied data, it should return SQLITE_OK to indicate success. Otherwise, +** it should return some other SQLite error code. In this case processing +** is immediately abandoned and the streaming API function returns a copy +** of the xOutput error code to the application. +** +** The sessions module never invokes an xOutput callback with the third +** parameter set to a value less than or equal to zero. Other than this, +** no guarantees are made as to the size of the chunks of data returned. +*/ +int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changegroup_add_strm(sqlite3_changegroup*, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +} +#endif + +#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ diff --git a/ext/session/test_session.c b/ext/session/test_session.c new file mode 100644 index 00000000..411354cc --- /dev/null +++ b/ext/session/test_session.c @@ -0,0 +1,1046 @@ + +#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_SESSION) \ + && defined(SQLITE_ENABLE_PREUPDATE_HOOK) + +#include "sqlite3session.h" +#include +#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif + +typedef struct TestSession TestSession; +struct TestSession { + sqlite3_session *pSession; + Tcl_Interp *interp; + Tcl_Obj *pFilterScript; +}; + +typedef struct TestStreamInput TestStreamInput; +struct TestStreamInput { + int nStream; /* Maximum chunk size */ + unsigned char *aData; /* Pointer to buffer containing data */ + int nData; /* Size of buffer aData in bytes */ + int iData; /* Bytes of data already read by sessions */ +}; + +/* +** Extract an sqlite3* db handle from the object passed as the second +** argument. If successful, set *pDb to point to the db handle and return +** TCL_OK. Otherwise, return TCL_ERROR. +*/ +static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ + Tcl_CmdInfo info; + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + return TCL_ERROR; + } + + *pDb = *(sqlite3 **)info.objClientData; + return TCL_OK; +} + +/************************************************************************* +** The following code is copied byte-for-byte from the sessions module +** documentation. It is used by some of the sessions modules tests to +** ensure that the example in the documentation does actually work. +*/ +/* +** Argument zSql points to a buffer containing an SQL script to execute +** against the database handle passed as the first argument. As well as +** executing the SQL script, this function collects a changeset recording +** all changes made to the "main" database file. Assuming no error occurs, +** output variables (*ppChangeset) and (*pnChangeset) are set to point +** to a buffer containing the changeset and the size of the changeset in +** bytes before returning SQLITE_OK. In this case it is the responsibility +** of the caller to eventually free the changeset blob by passing it to +** the sqlite3_free function. +** +** Or, if an error does occur, return an SQLite error code. The final +** value of (*pChangeset) and (*pnChangeset) are undefined in this case. +*/ +int sql_exec_changeset( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL script to execute */ + int *pnChangeset, /* OUT: Size of changeset blob in bytes */ + void **ppChangeset /* OUT: Pointer to changeset blob */ +){ + sqlite3_session *pSession = 0; + int rc; + + /* Create a new session object */ + rc = sqlite3session_create(db, "main", &pSession); + + /* Configure the session object to record changes to all tables */ + if( rc==SQLITE_OK ) rc = sqlite3session_attach(pSession, NULL); + + /* Execute the SQL script */ + if( rc==SQLITE_OK ) rc = sqlite3_exec(db, zSql, 0, 0, 0); + + /* Collect the changeset */ + if( rc==SQLITE_OK ){ + rc = sqlite3session_changeset(pSession, pnChangeset, ppChangeset); + } + + /* Delete the session object */ + sqlite3session_delete(pSession); + + return rc; +} +/************************************************************************/ + +/* +** Tclcmd: sql_exec_changeset DB SQL +*/ +static int SQLITE_TCLAPI test_sql_exec_changeset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zSql; + sqlite3 *db; + void *pChangeset; + int nChangeset; + int rc; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SQL"); + return TCL_ERROR; + } + if( dbHandleFromObj(interp, objv[1], &db) ) return TCL_ERROR; + zSql = (const char*)Tcl_GetString(objv[2]); + + rc = sql_exec_changeset(db, zSql, &nChangeset, &pChangeset); + if( rc!=SQLITE_OK ){ + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, "error in sql_exec_changeset()", 0); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pChangeset, nChangeset)); + sqlite3_free(pChangeset); + return TCL_OK; +} + + + +#define SESSION_STREAM_TCL_VAR "sqlite3session_streams" + +/* +** Attempt to find the global variable zVar within interpreter interp +** and extract an integer value from it. Return this value. +** +** If the named variable cannot be found, or if it cannot be interpreted +** as a integer, return 0. +*/ +static int test_tcl_integer(Tcl_Interp *interp, const char *zVar){ + Tcl_Obj *pObj; + int iVal = 0; + pObj = Tcl_ObjGetVar2(interp, Tcl_NewStringObj(zVar, -1), 0, TCL_GLOBAL_ONLY); + if( pObj ) Tcl_GetIntFromObj(0, pObj, &iVal); + return iVal; +} + +static int test_session_error(Tcl_Interp *interp, int rc, char *zErr){ + extern const char *sqlite3ErrName(int); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + if( zErr ){ + Tcl_AppendResult(interp, " - ", zErr, 0); + sqlite3_free(zErr); + } + return TCL_ERROR; +} + +static int test_table_filter(void *pCtx, const char *zTbl){ + TestSession *p = (TestSession*)pCtx; + Tcl_Obj *pEval; + int rc; + int bRes = 0; + + pEval = Tcl_DuplicateObj(p->pFilterScript); + Tcl_IncrRefCount(pEval); + rc = Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zTbl, -1)); + if( rc==TCL_OK ){ + rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); + } + if( rc==TCL_OK ){ + rc = Tcl_GetBooleanFromObj(p->interp, Tcl_GetObjResult(p->interp), &bRes); + } + if( rc!=TCL_OK ){ + /* printf("error: %s\n", Tcl_GetStringResult(p->interp)); */ + Tcl_BackgroundError(p->interp); + } + Tcl_DecrRefCount(pEval); + + return bRes; +} + +struct TestSessionsBlob { + void *p; + int n; +}; +typedef struct TestSessionsBlob TestSessionsBlob; + +static int testStreamOutput( + void *pCtx, + const void *pData, + int nData +){ + TestSessionsBlob *pBlob = (TestSessionsBlob*)pCtx; + char *pNew; + + assert( nData>0 ); + pNew = (char*)sqlite3_realloc(pBlob->p, pBlob->n + nData); + if( pNew==0 ){ + return SQLITE_NOMEM; + } + pBlob->p = (void*)pNew; + memcpy(&pNew[pBlob->n], pData, nData); + pBlob->n += nData; + return SQLITE_OK; +} + +/* +** Tclcmd: $session attach TABLE +** $session changeset +** $session delete +** $session enable BOOL +** $session indirect INTEGER +** $session patchset +** $session table_filter SCRIPT +*/ +static int SQLITE_TCLAPI test_session_cmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + TestSession *p = (TestSession*)clientData; + sqlite3_session *pSession = p->pSession; + struct SessionSubcmd { + const char *zSub; + int nArg; + const char *zMsg; + int iSub; + } aSub[] = { + { "attach", 1, "TABLE", }, /* 0 */ + { "changeset", 0, "", }, /* 1 */ + { "delete", 0, "", }, /* 2 */ + { "enable", 1, "BOOL", }, /* 3 */ + { "indirect", 1, "BOOL", }, /* 4 */ + { "isempty", 0, "", }, /* 5 */ + { "table_filter", 1, "SCRIPT", }, /* 6 */ + { "patchset", 0, "", }, /* 7 */ + { "diff", 2, "FROMDB TBL", }, /* 8 */ + { 0 } + }; + int iSub; + int rc; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( objc!=2+aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: { /* attach */ + char *zArg = Tcl_GetString(objv[2]); + if( zArg[0]=='*' && zArg[1]=='\0' ) zArg = 0; + rc = sqlite3session_attach(pSession, zArg); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + break; + } + + case 7: /* patchset */ + case 1: { /* changeset */ + TestSessionsBlob o = {0, 0}; + if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ + void *pCtx = (void*)&o; + if( iSub==7 ){ + rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx); + }else{ + rc = sqlite3session_changeset_strm(pSession, testStreamOutput, pCtx); + } + }else{ + if( iSub==7 ){ + rc = sqlite3session_patchset(pSession, &o.n, &o.p); + }else{ + rc = sqlite3session_changeset(pSession, &o.n, &o.p); + } + } + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(o.p, o.n)); + } + sqlite3_free(o.p); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + break; + } + + case 2: /* delete */ + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + break; + + case 3: { /* enable */ + int val; + if( Tcl_GetIntFromObj(interp, objv[2], &val) ) return TCL_ERROR; + val = sqlite3session_enable(pSession, val); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(val)); + break; + } + + case 4: { /* indirect */ + int val; + if( Tcl_GetIntFromObj(interp, objv[2], &val) ) return TCL_ERROR; + val = sqlite3session_indirect(pSession, val); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(val)); + break; + } + + case 5: { /* isempty */ + int val; + val = sqlite3session_isempty(pSession); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(val)); + break; + } + + case 6: { /* table_filter */ + if( p->pFilterScript ) Tcl_DecrRefCount(p->pFilterScript); + p->interp = interp; + p->pFilterScript = Tcl_DuplicateObj(objv[2]); + Tcl_IncrRefCount(p->pFilterScript); + sqlite3session_table_filter(pSession, test_table_filter, clientData); + break; + } + + case 8: { /* diff */ + char *zErr = 0; + rc = sqlite3session_diff(pSession, + Tcl_GetString(objv[2]), + Tcl_GetString(objv[3]), + &zErr + ); + assert( rc!=SQLITE_OK || zErr==0 ); + if( rc ){ + return test_session_error(interp, rc, zErr); + } + break; + } + } + + return TCL_OK; +} + +static void SQLITE_TCLAPI test_session_del(void *clientData){ + TestSession *p = (TestSession*)clientData; + if( p->pFilterScript ) Tcl_DecrRefCount(p->pFilterScript); + sqlite3session_delete(p->pSession); + ckfree((char*)p); +} + +/* +** Tclcmd: sqlite3session CMD DB-HANDLE DB-NAME +*/ +static int SQLITE_TCLAPI test_sqlite3session( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + Tcl_CmdInfo info; + int rc; /* sqlite3session_create() return code */ + TestSession *p; /* New wrapper object */ + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE DB-NAME"); + return TCL_ERROR; + } + + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + + p = (TestSession*)ckalloc(sizeof(TestSession)); + memset(p, 0, sizeof(TestSession)); + rc = sqlite3session_create(db, Tcl_GetString(objv[3]), &p->pSession); + if( rc!=SQLITE_OK ){ + ckfree((char*)p); + return test_session_error(interp, rc, 0); + } + + Tcl_CreateObjCommand( + interp, Tcl_GetString(objv[1]), test_session_cmd, (ClientData)p, + test_session_del + ); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + +static void test_append_value(Tcl_Obj *pList, sqlite3_value *pVal){ + if( pVal==0 ){ + Tcl_ListObjAppendElement(0, pList, Tcl_NewObj()); + Tcl_ListObjAppendElement(0, pList, Tcl_NewObj()); + }else{ + Tcl_Obj *pObj; + switch( sqlite3_value_type(pVal) ){ + case SQLITE_NULL: + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("n", 1)); + pObj = Tcl_NewObj(); + break; + case SQLITE_INTEGER: + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("i", 1)); + pObj = Tcl_NewWideIntObj(sqlite3_value_int64(pVal)); + break; + case SQLITE_FLOAT: + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("f", 1)); + pObj = Tcl_NewDoubleObj(sqlite3_value_double(pVal)); + break; + case SQLITE_TEXT: { + const char *z = (char*)sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("t", 1)); + pObj = Tcl_NewStringObj(z, n); + break; + } + default: + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("b", 1)); + pObj = Tcl_NewByteArrayObj( + sqlite3_value_blob(pVal), + sqlite3_value_bytes(pVal) + ); + break; + } + Tcl_ListObjAppendElement(0, pList, pObj); + } +} + +typedef struct TestConflictHandler TestConflictHandler; +struct TestConflictHandler { + Tcl_Interp *interp; + Tcl_Obj *pConflictScript; + Tcl_Obj *pFilterScript; +}; + +static int test_obj_eq_string(Tcl_Obj *p, const char *z){ + int n; + int nObj; + char *zObj; + + n = (int)strlen(z); + zObj = Tcl_GetStringFromObj(p, &nObj); + + return (nObj==n && (n==0 || 0==memcmp(zObj, z, n))); +} + +static int test_filter_handler( + void *pCtx, /* Pointer to TestConflictHandler structure */ + const char *zTab /* Table name */ +){ + TestConflictHandler *p = (TestConflictHandler *)pCtx; + int res = 1; + Tcl_Obj *pEval; + Tcl_Interp *interp = p->interp; + + pEval = Tcl_DuplicateObj(p->pFilterScript); + Tcl_IncrRefCount(pEval); + + if( TCL_OK!=Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj(zTab, -1)) + || TCL_OK!=Tcl_EvalObjEx(interp, pEval, TCL_EVAL_GLOBAL) + || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &res) + ){ + Tcl_BackgroundError(interp); + } + + Tcl_DecrRefCount(pEval); + return res; +} + +static int test_conflict_handler( + void *pCtx, /* Pointer to TestConflictHandler structure */ + int eConf, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *pIter /* Handle describing change and conflict */ +){ + TestConflictHandler *p = (TestConflictHandler *)pCtx; + Tcl_Obj *pEval; + Tcl_Interp *interp = p->interp; + int ret = 0; /* Return value */ + + int op; /* SQLITE_UPDATE, DELETE or INSERT */ + const char *zTab; /* Name of table conflict is on */ + int nCol; /* Number of columns in table zTab */ + + pEval = Tcl_DuplicateObj(p->pConflictScript); + Tcl_IncrRefCount(pEval); + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0); + + if( eConf==SQLITE_CHANGESET_FOREIGN_KEY ){ + int nFk; + sqlite3changeset_fk_conflicts(pIter, &nFk); + Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj("FOREIGN_KEY", -1)); + Tcl_ListObjAppendElement(0, pEval, Tcl_NewIntObj(nFk)); + }else{ + + /* Append the operation type. */ + Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj( + op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : + "DELETE", -1 + )); + + /* Append the table name. */ + Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj(zTab, -1)); + + /* Append the conflict type. */ + switch( eConf ){ + case SQLITE_CHANGESET_DATA: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("DATA",-1)); + break; + case SQLITE_CHANGESET_NOTFOUND: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("NOTFOUND",-1)); + break; + case SQLITE_CHANGESET_CONFLICT: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("CONFLICT",-1)); + break; + case SQLITE_CHANGESET_CONSTRAINT: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("CONSTRAINT",-1)); + break; + } + + /* If this is not an INSERT, append the old row */ + if( op!=SQLITE_INSERT ){ + int i; + Tcl_Obj *pOld = Tcl_NewObj(); + for(i=0; inData - p->iData; /* Bytes of data available */ + int nRet = p->nStream; /* Bytes actually returned */ + + /* Allocate and free some space. There is no point to this, other than + ** that it allows the regular OOM fault-injection tests to cause an error + ** in this function. */ + void *pAlloc = sqlite3_malloc(10); + if( pAlloc==0 ) return SQLITE_NOMEM; + sqlite3_free(pAlloc); + + if( nRet>nReq ) nRet = nReq; + if( nRet>nRem ) nRet = nRem; + + assert( nRet>=0 ); + if( nRet>0 ){ + memcpy(pData, &p->aData[p->iData], nRet); + p->iData += nRet; + } + + *pnData = nRet; + return SQLITE_OK; +} + + +/* +** sqlite3changeset_apply DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT? +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_apply( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; /* Database handle */ + Tcl_CmdInfo info; /* Database Tcl command (objv[1]) info */ + int rc; /* Return code from changeset_invert() */ + void *pChangeset; /* Buffer containing changeset */ + int nChangeset; /* Size of buffer aChangeset in bytes */ + TestConflictHandler ctx; + TestStreamInput sStr; + + memset(&sStr, 0, sizeof(sStr)); + sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + + if( objc!=4 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, + "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?" + ); + return TCL_ERROR; + } + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); + ctx.pConflictScript = objv[3]; + ctx.pFilterScript = objc==5 ? objv[4] : 0; + ctx.interp = interp; + + if( sStr.nStream==0 ){ + rc = sqlite3changeset_apply(db, nChangeset, pChangeset, + (objc==5) ? test_filter_handler : 0, test_conflict_handler, (void *)&ctx + ); + }else{ + sStr.aData = (unsigned char*)pChangeset; + sStr.nData = nChangeset; + rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr, + (objc==5) ? test_filter_handler : 0, test_conflict_handler, (void *)&ctx + ); + } + + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** sqlite3changeset_apply_replace_all DB CHANGESET +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_apply_replace_all( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; /* Database handle */ + Tcl_CmdInfo info; /* Database Tcl command (objv[1]) info */ + int rc; /* Return code from changeset_invert() */ + void *pChangeset; /* Buffer containing changeset */ + int nChangeset; /* Size of buffer aChangeset in bytes */ + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB CHANGESET"); + return TCL_ERROR; + } + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); + + rc = sqlite3changeset_apply(db, nChangeset, pChangeset, 0, replace_handler,0); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + Tcl_ResetResult(interp); + return TCL_OK; +} + + +/* +** sqlite3changeset_invert CHANGESET +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_invert( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; /* Return code from changeset_invert() */ + TestStreamInput sIn; /* Input stream */ + TestSessionsBlob sOut; /* Output blob */ + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHANGESET"); + return TCL_ERROR; + } + + memset(&sIn, 0, sizeof(sIn)); + memset(&sOut, 0, sizeof(sOut)); + sIn.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + sIn.aData = Tcl_GetByteArrayFromObj(objv[1], &sIn.nData); + + if( sIn.nStream ){ + rc = sqlite3changeset_invert_strm( + testStreamInput, (void*)&sIn, testStreamOutput, (void*)&sOut + ); + }else{ + rc = sqlite3changeset_invert(sIn.nData, sIn.aData, &sOut.n, &sOut.p); + } + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + }else{ + Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n)); + } + sqlite3_free(sOut.p); + return rc; +} + +/* +** sqlite3changeset_concat LEFT RIGHT +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_concat( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; /* Return code from changeset_invert() */ + + TestStreamInput sLeft; /* Input stream */ + TestStreamInput sRight; /* Input stream */ + TestSessionsBlob sOut = {0,0}; /* Output blob */ + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "LEFT RIGHT"); + return TCL_ERROR; + } + + memset(&sLeft, 0, sizeof(sLeft)); + memset(&sRight, 0, sizeof(sRight)); + sLeft.aData = Tcl_GetByteArrayFromObj(objv[1], &sLeft.nData); + sRight.aData = Tcl_GetByteArrayFromObj(objv[2], &sRight.nData); + sLeft.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + sRight.nStream = sLeft.nStream; + + if( sLeft.nStream>0 ){ + rc = sqlite3changeset_concat_strm( + testStreamInput, (void*)&sLeft, + testStreamInput, (void*)&sRight, + testStreamOutput, (void*)&sOut + ); + }else{ + rc = sqlite3changeset_concat( + sLeft.nData, sLeft.aData, sRight.nData, sRight.aData, &sOut.n, &sOut.p + ); + } + + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + }else{ + Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n)); + } + sqlite3_free(sOut.p); + return rc; +} + +/* +** sqlite3session_foreach VARNAME CHANGESET SCRIPT +*/ +static int SQLITE_TCLAPI test_sqlite3session_foreach( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void *pChangeset; + int nChangeset; + sqlite3_changeset_iter *pIter; + int rc; + Tcl_Obj *pVarname; + Tcl_Obj *pCS; + Tcl_Obj *pScript; + int isCheckNext = 0; + + TestStreamInput sStr; + memset(&sStr, 0, sizeof(sStr)); + + if( objc>1 ){ + char *zOpt = Tcl_GetString(objv[1]); + isCheckNext = (strcmp(zOpt, "-next")==0); + } + if( objc!=4+isCheckNext ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-next? VARNAME CHANGESET SCRIPT"); + return TCL_ERROR; + } + + pVarname = objv[1+isCheckNext]; + pCS = objv[2+isCheckNext]; + pScript = objv[3+isCheckNext]; + + pChangeset = (void *)Tcl_GetByteArrayFromObj(pCS, &nChangeset); + sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + if( sStr.nStream==0 ){ + rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); + }else{ + sStr.aData = (unsigned char*)pChangeset; + sStr.nData = nChangeset; + rc = sqlite3changeset_start_strm(&pIter, testStreamInput, (void*)&sStr); + } + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + + while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ + int nCol; /* Number of columns in table */ + int nCol2; /* Number of columns in table */ + int op; /* SQLITE_INSERT, UPDATE or DELETE */ + const char *zTab; /* Name of table change applies to */ + Tcl_Obj *pVar; /* Tcl value to set $VARNAME to */ + Tcl_Obj *pOld; /* Vector of old.* values */ + Tcl_Obj *pNew; /* Vector of new.* values */ + int bIndirect; + + char *zPK; + unsigned char *abPK; + int i; + + /* Test that _fk_conflicts() returns SQLITE_MISUSE if called on this + ** iterator. */ + int nDummy; + if( SQLITE_MISUSE!=sqlite3changeset_fk_conflicts(pIter, &nDummy) ){ + sqlite3changeset_finalize(pIter); + return TCL_ERROR; + } + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + pVar = Tcl_NewObj(); + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj( + op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : + "DELETE", -1 + )); + + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zTab, -1)); + Tcl_ListObjAppendElement(0, pVar, Tcl_NewBooleanObj(bIndirect)); + + zPK = ckalloc(nCol+1); + memset(zPK, 0, nCol+1); + sqlite3changeset_pk(pIter, &abPK, &nCol2); + assert( nCol==nCol2 ); + for(i=0; izCmd, p->xProc, 0, 0); + } + + return TCL_OK; +} + +#endif /* SQLITE_TEST && SQLITE_SESSION && SQLITE_PREUPDATE_HOOK */ diff --git a/main.mk b/main.mk index e9920180..a5a50d20 100644 --- a/main.mk +++ b/main.mk @@ -46,6 +46,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth +TCCX += -I$(TOP)/ext/session TCCX += -I$(TOP)/ext/fts5 THREADLIB += $(LIBS) @@ -61,7 +62,7 @@ LIBOBJ+= vdbe.o parse.o \ fts3_tokenize_vtab.o \ fts3_unicode.o fts3_unicode2.o \ fts3_write.o fts5.o func.o global.o hash.o \ - icu.o insert.o journal.o json1.o legacy.o loadext.o \ + icu.o insert.o json1.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ @@ -74,6 +75,7 @@ LIBOBJ+= vdbe.o parse.o \ vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \ utf.o vtab.o +LIBOBJ += sqlite3session.o # All of the source code files. # @@ -104,7 +106,6 @@ SRC = \ $(TOP)/src/hash.h \ $(TOP)/src/hwtime.h \ $(TOP)/src/insert.c \ - $(TOP)/src/journal.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ $(TOP)/src/main.c \ @@ -223,6 +224,9 @@ SRC += \ $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c +SRC += \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/userauth/userauth.c \ $(TOP)/ext/userauth/sqlite3userauth.h @@ -285,9 +289,11 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_bestindex.c \ $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ + $(TOP)/src/test_delete.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ @@ -319,7 +325,9 @@ TESTSRC = \ # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/csv.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ @@ -381,7 +389,9 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c + $(TOP)/ext/async/sqlite3async.c \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/test_session.c # Header files used by all library source files. # @@ -443,7 +453,8 @@ TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ - sqldiff$(EXE) + sqldiff$(EXE) \ + dbhash$(EXE) # Databases containing fuzzer test cases # @@ -460,6 +471,8 @@ TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 @@ -480,6 +493,13 @@ sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) +dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h + $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ + $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) + +scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o + $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) + srcck1$(EXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c @@ -534,6 +554,7 @@ target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . + cp $(TOP)/ext/session/sqlite3session.h . echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c @@ -700,10 +721,12 @@ fts5.c: $(FTS5_SRC) $(FTS5_HDR) userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c +sqlite3session.o: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c + sqlite3rbu.o: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c - # Rules for building test programs and for running tests # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a @@ -725,16 +748,20 @@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c # Rules to build the 'testfixture' application. # TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 -TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE +TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE +TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 +TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024 testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ -o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB) -amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c +amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/session/test_session.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ + $(TOP)/ext/session/test_session.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c @@ -830,6 +857,10 @@ showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) +changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ + $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) + fts3view$(EXE): $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o fts3view$(EXE) \ $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(THREADLIB) @@ -886,7 +917,6 @@ clean: rm -f mkkeywordhash mkkeywordhash.exe keywordhash.h rm -f $(PUBLISH) rm -f *.da *.bb *.bbg gmon.out - rm -rf quota2a quota2b quota2c rm -rf tsrc target_source rm -f testloadext.dll libtestloadext.so rm -f amalgamation-testfixture amalgamation-testfixture.exe @@ -900,6 +930,7 @@ clean: rm -f showjournal showjournal.exe rm -f showstat4 showstat4.exe rm -f showwal showwal.exe + rm -f changeset changeset.exe rm -f speedtest1 speedtest1.exe rm -f wordcount wordcount.exe rm -f rbu rbu.exe diff --git a/manifest b/manifest index 6129d48a..c45c5825 100644 --- a/manifest +++ b/manifest @@ -1,24 +1,24 @@ -C Version\s3.11.0 -D 2016-02-15T17:29:24.141 -F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142 +C Version\s3.15.2 +D 2016-11-28T19:13:37.853 +F Makefile.in 6fd48ffcf7c2deea7499062d1f3747f986c19678 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 30f075dc4f27a07abb76088946b2944178d85347 +F Makefile.msc e0217f2d35a0448abbe4b066132ae20136e8b408 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 -F VERSION 866588d1edf0ccb5b0d33896974338f97564f719 +F VERSION 1efaccf9d0764b8f1ca730ded1f5530961a11844 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am 29e2a6e8d0c5e32723a48b4faf6b168854dde5f4 -F autoconf/Makefile.msc b865d2c72cf43cbf39913336415556af8ff2e819 +F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 +F autoconf/Makefile.msc b6d27f735911fd09a589d3c966936c47a5850c17 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt 7c31da66232f7590bb987cfcd4e2381744b25d24 -F autoconf/configure.ac 72a5e42beb090b32bca580285dc0ab3c4670adb8 +F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 +F autoconf/configure.ac cacf2616abf6e4a569bde2ef365c143caeec40bc F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac 93d43c79e936fb16556e22498177d7e8571efa04 +F autoconf/tea/configure.ac 8aa16e3f0a5ca7959d4af198f46934ec187d395f F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in 3ef61715cf1c7bdcff56947ffadb26bc991ca39d @@ -28,12 +28,12 @@ F autoconf/tea/win/makefile.vc f89d0184d0eee5f7e356ea407964dcd139939928 F autoconf/tea/win/nmakehlp.c 247538ad8e8c508f33c03ec1fbd67d3a07ef6291 F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 -F config.h.in 42b71ad3fe21c9e88fa59e8458ca1a6bc72eb0c0 +F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 12d96e3798e612e0ffa53a7a8c4d7fb1090df80e x -F configure.ac a2224b1162f79848982d3618ac1deffcd94e88ec +F configure b2d6400d41ea39410a111b698a91dc78ecacd13d x +F configure.ac b5d3df43161374f8dffd2e5f4b88fbb51685b975 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad -F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 +F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 @@ -62,7 +62,7 @@ F ext/fts2/fts2_hash.c 011a1d32de45bb1b519a1fd0048e857d6a843558 F ext/fts2/fts2_hash.h 1824b99dfd8d0225facbdb26a2c87289b2e7dcf8 F ext/fts2/fts2_icu.c 51c5cd3c04954badd329fa738c95fcdb717b5188 F ext/fts2/fts2_porter.c 2cd4a507bf3c3085fe66f59b0f2a325f65aaacf5 -F ext/fts2/fts2_tokenizer.c 3dbe8058e97afb55fff3ea844120ce3208b114cc +F ext/fts2/fts2_tokenizer.c b529493d55e55497213c37e1f31680a77746be26 F ext/fts2/fts2_tokenizer.h 27a1a99ca2d615cf7e142839b8d79e8751b4529e F ext/fts2/fts2_tokenizer1.c 07e223eecb483d448313b5f1553a4f299a7fb7a1 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 @@ -81,48 +81,48 @@ F ext/fts3/fts3_icu.c deb46f7020d87ea7a14a433fb7a7f4bef42a9652 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 F ext/fts3/fts3_snippet.c 68ae118b0f834ea53d2b89e4087fc0f0b8c4ee4e F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7 -F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 +F ext/fts3/fts3_test.c 79f2a7fbb3f672fa032e5a432ca274ea3ee93c34 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 -F ext/fts3/fts3_tokenizer.c 4bd72f767f61c9ce5a7575c844e8d1ed2c3c561a +F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 -F ext/fts3/fts3_write.c f442223e4a1914dc1fc12b65af7e4f2c255fa47c +F ext/fts3/fts3_write.c c3863f23b6b4623c8b9d5cf31c12ce4469f78ca9 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 5d78b668f4e9598af9147f8999632599fb0d9dd5 +F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h fa7c17e5c3ec9c8690387ff962f9dc6aee75e114 -F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458 +F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 +F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f +F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd -F ext/fts5/fts5_config.c 35c5173cae4eb17e82164a7f5aeef56a48903079 -F ext/fts5/fts5_expr.c 8e8e4635f655133eb39018072fc0f0942a2c4337 -F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337 -F ext/fts5/fts5_index.c c6cd5190c7e5dd94151cb17acd939c82e5c7be2d -F ext/fts5/fts5_main.c 0e01ead4e817483e378e7e38e6d902f50b68d29e -F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e -F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 -F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070 +F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 +F ext/fts5/fts5_expr.c e12421946774482f4d49e334c974571177adbc70 +F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 +F ext/fts5/fts5_index.c fdbe46da11e81c1817cd4cb2de25ffdb4dd83711 +F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 +F ext/fts5/fts5_storage.c de0ed8a06738bde433afe11e92295ceaffbc4e58 +F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a +F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 -F ext/fts5/fts5_vocab.c dba72ca393d71c2588548b51380387f6b44c77a8 -F ext/fts5/fts5parse.y 59432ea369f1aa65511bad465f55d31a22f9f223 +F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7 +F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9 -F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 +F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 +F ext/fts5/test/fts5aa.test bd2d88182b9f7f30d300044048ad14683306b745 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 -F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c +F ext/fts5/test/fts5af.test c92825778ed2adc80014832762c056bbe968ef88 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df @@ -131,124 +131,140 @@ F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9 F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 F ext/fts5/test/fts5auto.test 401c20e89f1114d733b94809be1e6f893e16c09e -F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f +F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb -F ext/fts5/test/fts5bigtok.test 981b2790f6fa02773c889bd35d42c6b97f80f0f4 +F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a +F ext/fts5/test/fts5colset.test 1cdf56e079316005aabda790059aee86f2222ee4 F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 -F ext/fts5/test/fts5config.test 8b2bc6dcc0eb06fa2b7dd65b2ce2db09e829e873 +F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 -F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c -F ext/fts5/test/fts5corrupt3.test b9558d5b0ca44a8b6247fbb5d4a47592a8976892 +F ext/fts5/test/fts5corrupt2.test 128eb6e2d26b09f4da339e581f424b3321e0fdaa +F ext/fts5/test/fts5corrupt3.test f77f65e386231daf62902466b40ff998b2c8ce4f F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69 -F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d +F ext/fts5/test/fts5determin.test 10648edb75ef1e196b10978fd21a9be0c31e09c3 +F ext/fts5/test/fts5dlidx.test 007e9390c94638760797dbec2990c97c3fa08dfe F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 -F ext/fts5/test/fts5eb.test 021aa80b7ac09b964249aa32ced9ee908703e4aa +F ext/fts5/test/fts5eb.test 14597b5ecc9b2b7039ac5e0b0fca0311a7d22005 F ext/fts5/test/fts5fault1.test e09040d3e17b8c0837101e8c79c8a874c4376fb7 F ext/fts5/test/fts5fault2.test d8c6c7f916ccbdfc10b2c69530e9dd3bc8313232 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 -F ext/fts5/test/fts5fault4.test 532b6dacb963016cbf7003196bd87fb366540277 +F ext/fts5/test/fts5fault4.test dcbe3043c5611edd350191ea03a8daa190f0de5a F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080 -F ext/fts5/test/fts5fault8.test 430837fe6dd0511fd3aea52bd602ac02441bcb58 +F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea +F ext/fts5/test/fts5faultB.test 92ae906284062bf081b6c854afa54dcb1aa9ef88 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 +F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9 F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d -F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1 -F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 -F ext/fts5/test/fts5merge2.test c0cb66eb38a41c26cc5848fb9e50093e0f59ac93 +F ext/fts5/test/fts5matchinfo.test f7dde99697bcb310ea8faa8eb2714d9f4dfc0e1b +F ext/fts5/test/fts5merge.test 9f65f090d214ff865c56bef4f864aaa1182af6e3 +F ext/fts5/test/fts5merge2.test a6da3c16d694235938d1939f503cfa53f0943d75 +F ext/fts5/test/fts5multiclient.test 5bd54a6bc8ff818be233a50a8dec93da99098f2a F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c -F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 +F ext/fts5/test/fts5optimize.test 9d3ac53bb9cae58cb070d795db86bcb2f9fec105 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f F ext/fts5/test/fts5plan.test 6a55ecbac9890765b0e16f8c421c7e0888cfe436 F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487 F ext/fts5/test/fts5prefix.test efd42e00bb8e8a36383f25c838185508681c093f F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40 -F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab +F ext/fts5/test/fts5rank.test 2bdc0c5f22ccc1f9dbe9f4d0b82a491dce6f8a32 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 -F ext/fts5/test/fts5simple.test e6fe2fb10a2b9193648b32bbc2caecabdf8c333d +F ext/fts5/test/fts5simple.test 5da9b15ed534eb0be9f279d8a2bb2e24d30e4e38 F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46 -F ext/fts5/test/fts5simple3.test 8e71733b3d1b0e695011d02c68ebc5ca40b6124e +F ext/fts5/test/fts5simple3.test 25faa8cb8ad470c6f01f670bcc1317c19a89f091 F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 -F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 +F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e -F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 +F ext/fts5/test/fts5vocab.test 026799f8d24befc452cbddda79822f5ae8d5f4eb F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 -F ext/fts5/tool/fts5txt2db.tcl 1343745b89ca2a1e975c23f836d0cee410052975 +F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 -F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413 +F ext/icu/icu.c 03ff6f90f3004a7e5a86205b581b2b7035ebf6e1 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 -F ext/misc/amatch.c a1a8f66c29d40bd71b075546ddeddb477b17a2bb +F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234 +F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 +F ext/misc/csv.c 816a3715356e4210dae2d242057745e937050896 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f -F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 +F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c b9c88d5c3b6ecd8c731ffdd7f5b3d902857f8c96 +F ext/misc/json1.c f2ff73bb3f54cc643409766162174ffec3d7ea64 +F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 -F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 -F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc +F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e +F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/series.c b8fb7befd85b3a9b4a10e701b30b2b79ca92b6d4 +F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb +F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 525190484b7a9dbc6be646c4842274fff4f27d53 +F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 +F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/rbu/rbu.c ba3983dceffa0938532e79142f391737513de023 -F ext/rbu/rbu1.test 57601977588603e82700a43c279bd55282ffa482 +F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4 +F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 F ext/rbu/rbu12.test bde22ed0004dd5d1888c72a84ae407e574aeae16 F ext/rbu/rbu13.test 462ff799c4afedc3ef8a47ff818c0ffbf14ae4f2 F ext/rbu/rbu14.test 01f5dcba904aecadbaea69d4ccdc2ea43dd30560 F ext/rbu/rbu3.test 8bd4c6b87367c358981b6a47dc3d654fa60bff90 -F ext/rbu/rbu5.test 2e24fee3e615aecd99bbdd46967935a641e866f7 +F ext/rbu/rbu5.test cf1fa84d6107efd73e292a7be8e1fb31953ca45c F ext/rbu/rbu6.test 32e8ed60631b6facdb6366bd2b5f5f25245e7edb F ext/rbu/rbu7.test fd025d5ba440fcfe151fbb0e3835e1e7fe964fa1 F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0 F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 -F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a +F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 +F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 -F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d +F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 +F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 +F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda +F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c bea954197524631f2691ec272e8a42df8cad01cc -F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde -F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573 +F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 +F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc +F ext/rbu/sqlite3rbu.c e074c38798b90591f7f0cf0032d62f152ce5a95e +F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba +F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 0f9b595bd0debcbedf1d7a63d0e0678d619e6c9c +F ext/rtree/rtree.c 078fc27417b48aaebe988621bf96ef0e1645b201 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test 96a80c08440c932cd72aac50660e7af2612d2cda +F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba -F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc +F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482 F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 @@ -257,22 +273,45 @@ F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test b5eb13849545dfd271a54ff16784cb00d8792aea F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e -F ext/rtree/rtreeC.test 90aaaffe2fd4f0dcd12289cad5515f6d41f45ffd +F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtreeE.test 45a147a64a76306172819562309681d8e90f94bb F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4 +F ext/rtree/rtreeG.test 3b185719630795f38594f64cd7d1de86a33f91f1 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 +F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a +F ext/session/session1.test e5125b216d1e8c91e0984b361b0b68529e7c5dfb +F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 +F ext/session/session3.test a7a9ce59b8d1e49e2cc23d81421ac485be0eea01 +F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 +F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 +F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 +F ext/session/session8.test 8e194b3f655d861ca36de5d4de53f702751bab3b +F ext/session/session9.test 5409d90d8141881d08285ed1c2c0d8d10fb92069 +F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f +F ext/session/sessionB.test 886252dcb7e692e62ef7e357456200912e367823 +F ext/session/sessionC.test 97556f5164ac29f2344b24bd7de6a3a35a95c390 +F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2 +F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 +F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce +F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 +F ext/session/session_common.tcl 9b696a341cf1d3744823715ed92bb19749b6c3d4 +F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 +F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 +F ext/session/sqlite3session.c 37485891b4add26cf61495df193c419f36556a32 +F ext/session/sqlite3session.h 9345166bd8f80562145586cf817f707de5ecada2 +F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 791597b87204f6bd56cdf7b2ab2feeecd608c60d +F main.mk 06dc0b1a9c9e2d05c9275937dd5b894bfe7d17d8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -284,153 +323,155 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c 1fbb01c26c64528088f1df8015992fefda387889 -F src/analyze.c fbf0e80d83cc893734e872f932f249a056b86e11 -F src/attach.c c16c2648a577fa3def2adfa48c28901376389bc5 -F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 -F src/backup.c 2869a76c03eb393ee795416e2387005553df72bc -F src/bitvec.c 1a78d450a17c5016710eec900bedfc5729bf9bdf +F src/alter.c 299117695b1f21ac62dfc5b608588810ba22ed0d +F src/analyze.c 8b62b2cf4da85451534ac0af82cafc418d837f68 +F src/attach.c f6725410c184a80d8141b294fdf98a854c8a52b5 +F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 +F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b +F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63 F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73 -F src/btree.c 4c8caaeed7878aafdb607c3d2bcbc365bb0d19a1 -F src/btree.h 368ceeb4bd9312dc8df2ffd64b4b7dbcf4db5f8e +F src/btree.c 72965b023102daacc042443c707f4d415e88cbd7 +F src/btree.h d05b2fcc290991a8a3d9ea1816ddd55a4359dcde F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5 -F src/build.c 54866fbafa09d494269bdefc79995eb7207003a6 -F src/callback.c ed6c2a4a712eb7287ff64e20e3c23265dfb8a7ce -F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f -F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 -F src/date.c ca17321bc17cca8f40e0843edea4fafff974998e -F src/dbstat.c b2ec6793eef97aebb4d171d490a4ffdfa9f2475c -F src/delete.c 48802aa3ee6339f576d074336d3ae1b5f40e240f -F src/expr.c fbf0706199aea23c54efe36b6932d8307c4eb872 +F src/build.c 59dcfdc1ee55439d069af301ef7f2e84421b5102 +F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 +F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e +F src/ctime.c e77f3dc297b4b65c96da78b4ae4272fdfae863d7 +F src/date.c 95c9a8d00767e7221a8e9a31f4e913fc8029bf6b +F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d +F src/delete.c cb3f6300df24c26c609778b2731f82644b5532ec +F src/expr.c bf9c58cf9b6d9be4d2deb2b64aa96f25660a1e7f F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 08edad1fce30f761f14b3997e89bad58f9f7f4e0 -F src/func.c 86e55fee35b9577e485f47d9dd5c1d34cd513288 -F src/global.c bd5a0af3f30b0c01be6db756c626cd3c33a3d260 -F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 -F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 -F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 9ca97272e9f74ed0efddf3b4350ee12740cebbef -F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c b1b0880fc474abfab89e737b0ecfde0bd7a60902 -F src/loadext.c 84996d7d70a605597d79c1f1d7b2012a5fd34f2b -F src/main.c b67a45397b93b7ba8fbd6bfcb03423d245baed05 -F src/malloc.c 337e9808b5231855fe28857950f4f60ae42c417f +F src/fkey.c b9ca262f6ad4d030a3cab737ebf9b0b3c8b4ac80 +F src/func.c 7057bc2c105b82faa668d8e2ec85fad4540e5c51 +F src/global.c 9da4ca5d74b90715f0ec4957f3d17a4749009f34 +F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd +F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 +F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da +F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 +F src/insert.c 2d5e197f2f60351937b201196965fd14cd88489c +F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e +F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d +F src/main.c c9e49e376820bccca55fa9bf2ca321a51725db91 +F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 -F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 +F src/memjournal.c 95752936c11dc6995672d1dd783cd633eea0cc95 +F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c 205fa2bad945a0dc7cad48f9f95ea3e8dc5408ff -F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf +F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 +F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 -F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 821ed110197175165cf2f50b0930c7ff9a24504c -F src/os_win.c ccf29ddded3e41e506b6bd98c1171aa0963b23f2 -F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 67cd2fbab58d0e35fed5f81432856f4f0af9fc6d -F src/pager.h f3eb324a3ff2408b28bab7e81c1c55c13720f865 -F src/parse.y d7bff41d460f2df96fb890f36700e85cb0fc5634 -F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23 -F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 -F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051 -F src/pragma.c 80ee77226d0008d9188356a6cbbe6010866e1bee +F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 +F src/os_unix.c be9ca0f901a2b6c1bc93dc338f4863675180c189 +F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 +F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a +F src/pager.c b1ee3ec0e58a0f8ca090aec67b2d0c4d256d6bea +F src/pager.h ec9140e5349491e561bb76aeac7a756f191191a1 +F src/parse.y 0338f906b61e311c2b7e11a3f89b0092c780b664 +F src/pcache.c 5ff2a08f76a9c1b22f43eb063b7068fb085465ac +F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 +F src/pcache1.c e3967219b2a92b9edcb9324a4ba75009090d3953 +F src/pragma.c d932ba278654617cdd281f88a790a3185fca7c44 F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c -F src/prepare.c c12b786713df3e8270c0f85f988c5359d8b4d87c -F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 +F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a +F src/printf.c a5f0ca08ddede803c241266abb46356ec748ded1 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c -F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e -F src/select.c ff80004a9a6ece891a8d9327a88e7b6e2588ee6d -F src/shell.c 0367440658104bf2ce8d8a9a5a713a4b11c9acbe -F src/sqlite.h.in f80c6ebd85588fc514bfedf3ecb00cec269cb410 +F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb +F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac +F src/select.c ea3af83e2d0f245fef81ea4cf04cb730ce67f722 +F src/shell.c b80396d2fadce4681397707e30078bf416e1dec2 +F src/sqlite.h.in 1011de924a6a7340c74e5442cb76f7b49c134512 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h 04ca5b3cdb3bcf87ba6300d5d36b51498f65f28c -F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 -F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba -F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e -F src/tclsqlite.c 13debcc6a5ca1217486f8903768c01114fbe8b58 -F src/test1.c 4f1b42699068b7806af3111786f5ad760c2c1ff7 -F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b -F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f -F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e -F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 -F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 -F src/test7.c 9c89a4f1ed6bb13af0ed805b8d782bd83fcd57e3 -F src/test8.c fa262391d3edea6490a71bfaa8fed477ccbbac75 -F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 -F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 -F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 -F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 -F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07 -F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 7985332c806d1cece793475c75a6abcccde9d331 -F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 -F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc -F src/test_fs.c a61f54247fdb843761d709879c3bcd1989b2050c -F src/test_func.c 37453d346cfcf118774efd5bf6187f7e6a7e3254 -F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd -F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 -F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 -F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 -F src/test_journal.c 5360fbe1d1e4416ca36290562fd5a2e3f70f32aa +F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae +F src/sqliteInt.h 9fbddc799b0ea6ca61fce4a0ef8ed2a11358d515 +F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 +F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 +F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 +F src/tclsqlite.c aef87dcd8cb66564d560ab48d43d19ac812a1eab +F src/test1.c 1f1a399956c8730d5f7b09c729e98764f2082162 +F src/test2.c b7174313e993754303a8b33c43df7c44b46857ab +F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee +F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 +F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d +F src/test6.c 55aa2775c154415dcf4ed7cd1e19a193122b3a02 +F src/test7.c 5612e9aecf934d6df7bba6ce861fdf5ba5456010 +F src/test8.c 4f4904721167b32f7a4fa8c7b32a07a673d6cc86 +F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 +F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a +F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 +F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 +F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96 +F src/test_blob.c a0f7ad49a0c9d4b72f693fe2a71c58d7e507174d +F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 +F src/test_config.c 4d3d4a886416f369771d69a6dba926866deda788 +F src/test_delete.c 32690ea215abf29dda589a580cd1b790389759b6 +F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e +F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 +F src/test_fs.c e16cbe68d3b107e00a907c20a9a02629870eb69b +F src/test_func.c a2b4993da973b0ea60dd5d51a7066238ede8c329 +F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d +F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 +F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64 +F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c +F src/test_journal.c d3b83f2bcb7792c709e57abddc456a2b1818643a F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd -F src/test_malloc.c 96df9381a1ff1f6d3805ff7231b9baf1386aaabf -F src/test_multiplex.c 6a088d8d9d4aad4bec45dd8878af11b15900702d -F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 -F src/test_mutex.c dbdfaff8580071f2212a0deae3325a93a737819c -F src/test_onefile.c 38f7cbe79d5bafe95bde683cc3a53b8ca16daf10 -F src/test_osinst.c 5423dc1d355f594371f27dd292ca54bd320b8196 +F src/test_malloc.c c05f6c40bd6c8bfe5f1718212f81fd5687f91766 +F src/test_multiplex.c 7c5b218d855cd0505a43185d55283b3fc257c8da +F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635 +F src/test_mutex.c 7f4337ba23ee6b1d2ec81c189653608cb069926a +F src/test_onefile.c 416f87a28d6d673352d33fc4b1c7d39db878e50f +F src/test_osinst.c 98ef31ff03d55497829ca0f6c74a9f4e1aa48690 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 180813f43683be5725458fc1ff13ac455d8e722d +F src/test_quota.c 6cb9297115b551f433a9ad1741817a9831abed99 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d -F src/test_rtree.c 43fff4c5a01576d6d213f27472598801a247890c -F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091 +F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b +F src/test_schema.c f575932cb6274d12147a77e13ea4b49d52408513 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 0d138a8180a312bf996b37fa66da5c5799d4d57b -F src/test_superlock.c 06797157176eb7085027d9dd278c0d7a105e3ec9 -F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b -F src/test_tclvar.c d86412527da65468ee6fa1b8607c65d0af736bc4 -F src/test_thread.c af391ec03d23486dffbcc250b7e58e073f172af9 -F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283 +F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e +F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 +F src/test_tclvar.c df9fe1213c2634687a9ca0b0bec0d2119d359ae3 +F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 +F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 -F src/test_windirent.c 8f5fada630348558d5745b334702f301da1ffc61 -F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5 +F src/test_windirent.c 600398db0198ca1c77ca183831bf456746b6f5c4 +F src/test_windirent.h 7edc57e2faa727026dbd5d010dd0e2e665d5aa01 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c bbfb74450643cb5372a43ad4f6cffd7e9dfcecb0 -F src/tokenize.c 813934be70597edfbb685ae08fc4c8b549cf5a1e -F src/treeview.c dc39ccf04e9331237388b9cb73289c9d87ea050b -F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 -F src/update.c a7eeeaffad59c6506f01303a071dac11de8269ca -F src/utf.c 10cc2519e82e3369344d0969ad4b1a333dc86d18 -F src/util.c 49ce0a65306c1c51d61cb5bc214c71cb62452de6 -F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c c193299e595a13eba247738e22fce25c49346a6c -F src/vdbe.h c743791f723049db94f009e3e30958952bc2d512 -F src/vdbeInt.h 4b69d5451bcadd473e745af53ef1e8abfdce0a79 -F src/vdbeapi.c 9324f6baee1a1b2284c6543e98f916888a81e459 -F src/vdbeaux.c deae5d3bd45da0e57c7d9e1d7436333d142dc3bb -F src/vdbeblob.c 3b570b730109e8f653d9d2081649f6e7015113db -F src/vdbemem.c 68fcfac37dc6601d98c32cc5adee4d39f2c1b7b4 -F src/vdbesort.c ef3c6d1f1a7d44cf67bb2bee59ea3d1fe5bad174 -F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484 -F src/vtab.c bef51b4f693d82b4b0184457faa8625654534091 +F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c +F src/tokenize.c 78c8085bc7af1922aa687f0f4bbd716821330de5 +F src/treeview.c f51b75a28b377adde9f79bc3deb6c7770bcf97c0 +F src/trigger.c 3419bb9862983d84d70735fb4c94b21b934cd0c5 +F src/update.c 8179e699dbd45b92934fd02d3d8e3732e8da8802 +F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c +F src/util.c 3e2da6101888d073e79ecc6af5e0a2f70fa1e498 +F src/vacuum.c 33c174b28886b2faf26e503b5a49a1c01a9b1c16 +F src/vdbe.c f43aa96f2efe9bc8a06d17115661af527a3318fa +F src/vdbe.h c044be7050ac6bf596eecc6ab159f5dbc020a3b7 +F src/vdbeInt.h d8a56a491b752dbb5f671963b8c861ec72ea875e +F src/vdbeapi.c 97129bec6b1553da50d8e73f523c278bda66d9f6 +F src/vdbeaux.c 52d9b840859f260ce8418be9333dd77c902e1c74 +F src/vdbeblob.c 3e82a797b60c3b9fed7b8de8c539ca7607874937 +F src/vdbemem.c fcdd73a2001a673f48a244a77b1038928ae729c9 +F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c +F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 +F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 1dd3f1aeed5765be62f27d7d2e44d46a68d891ab -F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c -F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c 984084584c10c41e46c89ac027a5cca991bc37e6 -F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a -F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d -F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03 +F src/wal.c 02eeecc265f6ffd0597378f5d8ae9070b62a406a +F src/wal.h 6dd221ed384afdc204bc61e25c23ef7fd5a511f2 +F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 +F src/where.c 125af074723cf390d0e71ed1ca385240e6d4ea7c +F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d +F src/wherecode.c e895f34fe8cd06c386524f84da63eea5040b19b1 +F src/whereexpr.c c19a84ac530835d37217db2181e4fe75901b7b97 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -439,8 +480,8 @@ F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 2facdddf08d0d48e75dc6cc312cd2b030f4835dd F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 -F test/alter3.test b3568d11c38c4599c92f24242eda34144d78dc10 -F test/alter4.test c461150723ac957f3b2214aa0b11552cd72023ec +F test/alter3.test 4d79934d812eaeacc6f22781a080f8cfe012fdc3 +F test/alter4.test 0c33c542247ba5aee4f4a0133ac44bcf8f97e5e0 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 3eb35a4af972f98422e5dc0586501b17d103d321 @@ -450,14 +491,14 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82 -F test/analyze9.test 88c1f2aa20b614236f03e1cc38c3619e7e8a38b4 +F test/analyze9.test b817b8e798315fc65b820a5463f73ad5f48ed8dd F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test a4c1c3048f6d9e090eb76e83eecb18bcf6d31a70 F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 -F test/analyzeD.test f3d77cd0fefe2849d784897d52df13beee19271d +F test/analyzeD.test 42af58de25a6436502e43006e9e59e2d71bcb0cf F test/analyzeE.test 8684e8ac5722fb97c251887ad97e5d496a98af1d F test/analyzeF.test 5d1fe1024ba2dfea3c18bede8c1ccef8aba1ab34 -F test/analyzer1.test 498e2ff4b62740c2751c3a2f8b744fe26689fae9 +F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -469,28 +510,31 @@ F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360 -F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa -F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 -F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 +F test/auth.test 872a122b3977c1d1bb9fd637dc20016e5c01880f +F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 +F test/auth3.test b810826b193831929951c0d50783a200e5ef6b72 +F test/autoinc.test c3a55c3667f429f4f4c7bed05f67b4d93e89e79d F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea -F test/autoindex2.test af7e595c6864cc6ef5fc38d5db579a3e34940cb8 +F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf F test/autoindex5.test 96f084a5e6024ea07cace5888df3223f3ea86990 -F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 +F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 -F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 -F test/backup.test b79299a536a4c6d919094786595b95be56d02014 +F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d +F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf -F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 +F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 -F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 +F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f -F test/bc_common.tcl 3eda41ef9cda7d5f6c205462c96228b301da4191 +F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c +F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee +F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 +F test/bestindex3.test dd5fa4f483cd0356549203c4522f8c9e21cb1fc0 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -515,15 +559,15 @@ F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test bf6f0308bbbba1e770dac13aa08e5c2ac61c7324 +F test/capi3.test f0c66919e43d42e1572a69be039e4527a931b00f F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test 06f6261f9e9b4ef6f76afcd9900f3665408af1c8 +F test/capi3c.test 0b9edb0c2156a964b9271cd5ea7ae56736cc2fcb F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 -F test/cffault.test aadc1f61f8811cb600e3e069acbf8796f472a096 +F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 -F test/close.test 340bd24cc58b16c6bc01967402755027c37eb815 +F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test 08c18e7512a5a32c97938854263fa15362eeb846 @@ -536,34 +580,35 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 +F test/collateB.test 8ec2accd2d7166c1eff0d2a39bc90262c6f89632 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b -F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 -F test/conflict2.test a82dd3b9b41fceb5dd6ff0707c5c7ffba208d538 +F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db +F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 F test/corrupt.test 141c39ea650c1365e85a49e402fa05cb9617fb97 -F test/corrupt2.test cb787825d761b0f869764d6990531382840de872 -F test/corrupt3.test 4b548d0bbe2933bc81d3f54099a05fc4d28aff18 -F test/corrupt4.test b99652079d542b21f4965f6248703b983e40fe80 +F test/corrupt2.test e4964cee73dda57a90958e0087a6b388b1d9cb58 +F test/corrupt3.test e676f478fe602915d721472811f6f410b75ddc7e +F test/corrupt4.test 8d1d86b850fcc43e417450454f2044e52d55778a F test/corrupt5.test 8ead52af76006f3286e9396cb41898018ccea107 -F test/corrupt6.test 269548d19427ac554c830763b1c5ea54a0252f80 -F test/corrupt7.test e4fa6d6584276679cc1d20c4e58beb9559a4eb85 +F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568 +F test/corrupt7.test b036f94bda4b0b23a2919bf717046ce9ecca4543 F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 53e56dafd180addcdadb402244b8cb9771d2ba26 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec F test/corruptC.test 0c46574f8d4f27ecc799b1b5c4cbf9b1817bce9a F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 -F test/corruptE.test be8e5088c369fc7979c662cd644efdaafc0f7f6d +F test/corruptE.test 82ccf4f8f543fdbedd4aa42c709cb077f7374c62 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 -F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 -F test/corruptH.test 99ad81a4bda7cc078c589ef7542ecbc64e453c80 -F test/corruptI.test 347babbf970e7947e3f91dccf7a1bec28a1bab04 -F test/corruptJ.test 9e29e7a81ee3b6ac50f77ea7a9e2f3fa03f32d91 +F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 +F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 +F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 +F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c -F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 +F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 F test/crash3.test 8f5de9d32ab9ab95475a9efe7f47a940aa889418 @@ -571,21 +616,24 @@ F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc F test/crash5.test 05dd3aa9dbb751a22d5cdaf22a9c49b6667aa219 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df -F test/crash8.test 61442a9964ab6b124fc5254e4258b45747842e6f +F test/crash8.test a63907617d8e74fb54b4bff23eca8a4435625245 F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c -F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 +F test/csv01.test e0ba3caaa57e4c667a0b45977689fb8082f14348 +F test/ctime.test 0b995accd44a52914bd4744d5c1b6e1a56c7897c F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 +F test/cursorhint2.test fa41f0d997e67db921d08c31e73111b32811201a F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65 -F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 +F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d -F test/delete.test e1bcdf8926234e27aac24b346ad83d3329ec8b6f +F test/delete.test acc38fca8ee4851467705b1c2cfea64cd26667e5 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab -F test/delete4.test 3ac0b8d23689ba764c2e8b78c1b56b8f1b942fa2 +F test/delete4.test 21d2113217eeaacac2d99defe14fe6611615ae86 +F test/delete_db.test c70a43629dd4d3e1dd03fdaf7a22153af6a69d92 F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 @@ -601,8 +649,8 @@ F test/e_createtable.test d4c6059d44dcd4b636de9aae322766062b471844 F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 03a84a6fa9bd3472112d6bd4599f5269f5f74803 -F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 +F test/e_expr.test 1ffa8866d38e7becc76893a8829e9432050e5716 +F test/e_fkey.test 54cc0046d2d952d6c42b0dd94414e7a8f75f79f4 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 3de217e95094d3d165992a6de1164bbc4bd92dc7 F test/e_reindex.test 2bebf7b393e519198b7c654407221cf171a439b8 @@ -611,35 +659,35 @@ F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 -F test/e_uri.test eed3eb41b22d051a1164110dacdc778899126e14 -F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 +F test/e_uri.test 25385396082b67fd02ae0038b95a3b3575fe0519 +F test/e_vacuum.test 1b8b4772d05374aa1b8958669138bbb4213ee26a F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 -F test/e_walauto.test 280714ddf14e1a47dcbc59d515cd0b026dfd5567 +F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 -F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 +F test/e_walhook.test 4c0613a0c76e7a9d5c4c211e1b4cbcc1143914df F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 6807f7a7740a00361ca8d0ccd66bc60c8dc5f2b6 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 85873fa5816c48915c82c4e74cb5c35a5b48160f +F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c -F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 +F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 -F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 -F test/expr.test 79c3e7502d9e571553b85f0ecc8ff2ac7d0e4931 +F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac +F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 3e979af17dfa7e5e9dda5eba1a696c04fa9d47f7 -F test/filectrl.test 7c13f96457435238da99aff7343ad6a3a4885787 -F test/filefmt.test cb34663f126cbc2d358af552dcaf5c72769b0146 +F test/filectrl.test c6547e6c3c4c3ef9707a0b3dc147de718135cacc +F test/filefmt.test e4edbdc637ca9576ccf4337a3cce627d9df7a56c F test/fkey1.test 13e3d48236a2b9f5c5ebd232eef9b3ab682a8a2c F test/fkey2.test f3d27ecba480a348c328965d154214719bb158a9 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 5a373303f201ac03c22ba1ef17a733d3f56e611a -F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 +F test/fkey6.test d078a1e323a740062bed38df32b8a736fd320dc0 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 F test/fkey8.test 7bd1dd0174a0e29a90c62c517b9e2a410a0b345d F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 @@ -697,14 +745,14 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2 -F test/fts3atoken.test 76262be798f23a390717d14266f0df551e52a7ee +F test/fts3atoken.test 4b4c16fdcfc972f2cdbba212375a060a86ccf5f1 F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9 -F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221 +F test/fts3aux1.test ca32c138aa777b476271df2da2c5538b7a43ada1 F test/fts3aux2.test 7ae2b2c13aefdf4169279a27a5f51780ce57f6ba F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c -F test/fts3conf.test 1c8b8adb0ab28a41b68d1514df44380bd7353402 +F test/fts3conf.test 60317efd562080e198b5bdc9fcd222ce32cf01d7 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 @@ -714,7 +762,7 @@ F test/fts3defer2.test c540f5f5c2840f70c68fd9b597df817ec7170468 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 -F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162 +F test/fts3expr.test 9466627007804d855bf9df2a0cfb3dac23686fdc F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3 @@ -741,16 +789,17 @@ F test/fts4aa.test 10aac8e9d62c7357590acfabe3fad01e9a9ce1cb F test/fts4check.test c3056eab9524232e4c9bdcd119912947e07bcc1c F test/fts4content.test 05716af19a899cd70d5cd916c580043c03f30db4 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 -F test/fts4growth.test 60d6bb3f78e25b34f533797dd9f2f9402310a13a +F test/fts4growth.test e5390da74619cacc389711bac9349640b32c4f9a F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d -F test/fts4langid.test 8bd8759e0d4b04d71771544b861193a6841fee84 -F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee +F test/fts4langid.test 65a7332c9bc257919e259a304aa8a38c41655b9d +F test/fts4merge.test d2b39f6b1bd4a9738a13540e2d044cba11c43d47 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 -F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 +F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309 F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a +F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96 F test/fts4unicode.test 27378af76394542cf490cf001d8d1505fe55f6a9 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -758,13 +807,13 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 -F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 +F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test b47377143f0c80f91ed29d722861077ff34415d5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzcheck.c 3309d793165ca61a9996271cb799694839348f9a +F test/fuzzcheck.c f2e8102f7829f7b946ebdb6f1a16c6f942d9de66 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -775,24 +824,24 @@ F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test d7b0a5f41123df1e43985b91b8b2e70f95282d21 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f -F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 +F test/hook.test 3b7b99d0eece6d279812c2aef6fa08bdfabc633e +F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test 61a24ae38d4b64ec69f06ccdf022992f68a98176 +F test/in.test 20c5529986998949908f889c8208b2cd894b2cc9 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554 -F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 +F test/in5.test 6c006e0bcd7351b69350ef566e65f244023489e9 +F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 F test/incrblob4.test f26502a5697893e5acea268c910f16478c2f0fab -F test/incrblob_err.test af1f12ba60d220c9752073ff2bda2ad59e88960d +F test/incrblob_err.test 69f9247fed50278d48ea710d1a8f9cdb09e4c0b8 F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 -F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac +F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 F test/index.test fe3c7a1aad82af92623747e9c3f3aa94ccd51238 @@ -800,10 +849,11 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 81bc47890b8abfb181bc35f8d10b56c069803386 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a -F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c +F test/index6.test b4fc812290067a578b98bb2667b676db89e202a7 +F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb +F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 -F test/indexexpr1.test cb71b6586177b840e28110dd952178bb2bdfedc2 +F test/indexexpr1.test 7d243fac508b4a99fb900ffe34eb488312cfce84 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -812,10 +862,11 @@ F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test a20432f1c0fbbcff8f11d0e6ab4acb8c9db58023 F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 F test/instr.test 737bbf80685232033f3abedc6ae92f75860b5dd2 +F test/instrfault.test aa90b7c3486a069151b28384ae525644a1f79d51 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc -F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 -F test/io.test 3a7abcef18727cc0f2399e04b0e8903eccae50f8 +F test/intpkey.test ac71107a49a06492b69b82aafaf225400598d3c8 +F test/io.test f95bca1783b01ea7761671560d023360d2dfa4cc F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd @@ -829,12 +880,12 @@ F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 8a5c0be6f0c260a5c7177c3b8f07c7856141038a F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 -F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 +F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test ef42283f0b60d8bacbc2243448e7c84988578e52 +F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff @@ -844,9 +895,10 @@ F test/like.test 81632c437a947bf1f7130b19537da6a1a844806a F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e -F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b +F test/limit2.test e35f57bd3a62d7c5dcb5ac4306e675c75f974809 +F test/loadext.test d077450695ddb5c1ea3ad7d48e5f5850fe732ad9 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 +F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 @@ -855,7 +907,7 @@ F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035 F test/lookaside.test 90052e87282de256d613fcf8c9cbb845e4001d2f -F test/main.test 16131264ea0c2b93b95201f0c92958e85f2ba11a +F test/main.test bb75e406c9b64931f3dc7e7f04626633365bb22f F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a @@ -874,7 +926,7 @@ F test/mallocE.test db1ed69d7eded1b080952e2a7c37f364ad241b08 F test/mallocF.test 2d5c590ebc2fc7f0dcebdf5aa8498b9aed69107e F test/mallocG.test 0ff91b65c50bdaba680fb75d87fe4ad35bb7934f F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb -F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 +F test/mallocI.test 6c23a71df077fa5d387be90e7e669c5b368ca38a F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 27cb5566a6e5f2d76f9d4aa2eca45524401fd61e F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc @@ -883,34 +935,34 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test d2b2d6ca37890b26703a2258df8fd66f9869da02 +F test/memsubsys1.test 6d268d0ae90f8d61a2356a1838665654d83de518 F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 -F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd +F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f -F test/misc1.test 48ebfb5b22a6a058f7b7e1df211226dd1d21409c +F test/misc1.test 6430dabfb4b4fa480633590118964201f94d3ccc F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3 -F test/misc5.test f96428ea95b3820aafc6f1c50cf48a09e4597ee1 +F test/misc5.test fff0f75e834bc09a39f6079320dd8c37de956f4f F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test fc2754d38892f7dac30c22db3616c2764f117d66 +F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 -F test/mmap1.test 44a5ff1c1bcc7dcf2de50227d1f997e75a8ef1ae +F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 -F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e +F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test e0a44072d98189003deae4b091106f085d94bea8 +F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 -F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a -F test/nolock.test 0540dd96f39b8876e3ffdd8814fad0ea425efeee +F test/nan.test dacc57f80859c06a433d30839336fe227d2038b3 +F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -931,23 +983,23 @@ F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f -F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa +F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test ae9ee0db5a30aecda9db8290ce3dd12e5f7bbaa1 +F test/pagerfault.test 42ff797b1e6426c141cc7ee8b7417c9f27427950 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 -F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 +F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 +F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 382a43c49f49bafe6fddffe904ea33d6bb3ff33e -F test/pragma.test 507ac7ef2ea5682241ea0ef041799ca70bb5e0bf +F test/permutations.test cd0b7bc04bf5e50d3a993d24c834d591f7d4f5fe +F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f -F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c +F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb @@ -955,7 +1007,7 @@ F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 -F test/quota.test 36cd78b178c4eb0401d4f25754ef410fbd9df2a7 +F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 @@ -963,8 +1015,9 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 +F test/regexp2.test aa7ffcc21350007a78361b82bcf3b74d12227144 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 975449bf742b8bb9025208292208af816a1fcb58 +F test/releasetest.tcl cb06c4df0de4872e65b178316f8a87ccf7624d59 x F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5 @@ -972,6 +1025,16 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d +F test/rowvalue.test f4dc97fe52d4e206e25b3395588b9497a8eed3ff +F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b +F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 +F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea +F test/rowvalue5.test c81c7d8cf36711ab37675ad7376084ae2a359cb6 +F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 +F test/rowvalue7.test 5d06ff19d9e6969e574a2e662a531dd0c67801a8 +F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 +F test/rowvalue9.test e24f9eb02baffc6a67b6eed9e40d4c612c98079d +F test/rowvaluefault.test 7b16485e3f2b371f3e3d05455b8ded6d0c090244 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d @@ -979,7 +1042,7 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 +F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 @@ -992,13 +1055,13 @@ F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 -F test/select4.test 453631158540e5f685b81cac5b7e8bd8c6b4c5fc +F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328 F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0 -F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61 +F test/select7.test f659f231489349e8c5734e610803d7654207318f F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test e452bdb975f488ea46d091382a9185b5853ed2c7 +F test/selectA.test 101e722370ac6e84978c2958b8931c78b10a1709 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 @@ -1006,6 +1069,7 @@ F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test e8600e379589e85e9fefd2fe4d44a4cdd63f6982 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 +F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 F test/shared3.test ab693f9b6e156b8bfb2a0ad94f29fe69602a5d38 @@ -1018,67 +1082,71 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test ce5e744870387164703bf2dee2cc9753e4a71513 -F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862 -F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 -F test/shell4.test ddf0a99044e2245a87fc17423e3aaa1445b3243b -F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9 +F test/shell1.test 65b10cd8a90cda9b5af9100a45689a57dcc01a31 +F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b +F test/shell3.test da513d522ef6f01cee8475dcf8332bff8982b3dd +F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d +F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test d37a75b4be4eb9dedeb69b4f38b1d0a74b5021d7 +F test/skipscan1.test 8ab5d2c7c5cd3fe7f172d366e6e74e887cb33cb4 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b -F test/snapshot.test 5ec4651d16c3d1eb6c010d102febe32b3614bf56 -F test/snapshot_fault.test 25973aeb1b86a280800e0bcf1eb5ce70e9ef57ab +F test/snapshot.test a19465046168b4420b5afeed37c3214e42a49f4a +F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c -F test/sort5.test d3041ce3c475aa04142a959ae56ef6593f98a99f +F test/sort5.test 30cc17768e0c06ecb048e08efec59c11811fd186 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded -F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 +F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c f8bf04214e7b5f745feea99f7bde68b1c4870666 +F test/speedtest1.c 43b25ae8e303bcada98e00be036e710de17d346a F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 -F test/spellfix3.test f7bf7b3482971473d32b6b00f6944c5c066cff97 -F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5 +F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 +F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a -F test/stat.test acc91e80517fff447ae8adcfd953cfdaa5efc0af +F test/stat.test f8f1279ffffabe6df825723af18cc6e0ae70a893 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 -F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 +F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f -F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 +F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 -F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 +F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 -F test/sync.test 2f607e1821aa3af3c5c53b58835c05e511c95899 +F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test cc33684f9480fcf1fd5ce287ac28d22971cad1cc +F test/tabfunc01.test 50a9fb379f9747fd0d40ea6d8fa3a101361bb537 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 -F test/tclsqlite.test 7fb866443c7deceed22b63948ccd6f76b52ad054 -F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c +F test/tclsqlite.test 1d73b9203b1ca8798d7d7310742b8d3febc0d56e +F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 +F test/tempdb2.test 4fc92055f2a3f7626c0f2eabbb637dc021b311d5 +F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 +F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e +F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl 462376b478c1429030911b4cb7c8c517ef1fbd9b +F test/tester.tcl 4ce5afd5e192db4cae178e1a983b060e0f08c5d6 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1091,13 +1159,14 @@ F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8 F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 +F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 -F test/tkt-2d1a5c67d.test d371279946622698ab393ff88cad9f5f6d82960b +F test/tkt-2d1a5c67d.test be1326f3061caec85085f4c9ee4490561ca037c0 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 -F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac +F test/tkt-313723c356.test 4b306ad45c736cedf2f5221f6155b92143244b6d F test/tkt-385a5b56b9.test c0a06ada41d7f06b1686da0e718553f853771d1e F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 @@ -1108,7 +1177,7 @@ F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 F test/tkt-4ef7e3cfca.test 3965ae11cc9cf6e334f9d7d3c1e20bf8d56254b1 F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 -F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa +F test/tkt-5d863f876e.test 726e76d725f6fe0eb2fc8a522b721b79807380ee F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 @@ -1124,7 +1193,7 @@ F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 -F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 +F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 @@ -1230,11 +1299,12 @@ F test/tkt3929.test cdf67acf5aa936ec4ffead81db87f8a71fe40e59 F test/tkt3935.test e15261fedb9e30a4305a311da614a5d8e693c767 F test/tkt3992.test f3e7d548ac26f763b47bc0f750da3d03c81071da F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd -F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 +F test/tkt4018.test 18dbc6617f7a4b90e938d1bd6d26ad18daafaf08 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 04adbf8d8300fa60a222f28d901abd76e7be6dd4 F test/trace.test 6f676313e3ebd2a50585036d2f212a3319dd5836 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983 +F test/trace3.test 56ab944fddacf628b118cc298503fc45c2e50ab0 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 @@ -1259,11 +1329,11 @@ F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff -F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 +F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 -F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 +F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 6630ecbdea2aac10df3c89dbae2243f4c2c353e4 @@ -1273,15 +1343,17 @@ F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 +F test/vacuum5.test c87234e8ca4107f349da4edbeda3e4ea5adc93f3 +F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test f6c3a39e0c819891265e1d0754e99960d81ef6c9 +F test/view.test 765802c7a66d37fabd5ac8e2f2dbe572b43eb9ab F test/vtab1.test 7c4b81abd88361ada9cbe414c459efca26be6bda F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 8e73ed268f3d596bc3590f45fc948fb40f28e9c3 F test/vtab5.test 889f444970393c73f1e077e2bdc5d845e157a391 -F test/vtab6.test d2986cf418dc51e7fb81d12366bea2caa8b812df +F test/vtab6.test 8e789f526e6594cf7ae933d1adee0caa87dc9f78 F test/vtab7.test ae560ebea870ed04e9aa4177cc302f910faaabb5 F test/vtab8.test e19fa4a538fcd1bb66c22825fa8f71618fb13583 F test/vtab9.test ea58d2b95d61955f87226381716b2d0b1d4e4f9b @@ -1290,44 +1362,45 @@ F test/vtabB.test 04df5dc531b9f44d9ca65b9c1b79f12b5922a796 F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe -F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e -F test/vtabH.test 5f5157a1501d9889ec35c1a1832f69612dd31444 +F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b +F test/vtabH.test 97f61b0253260831af6232163f7852e2653baed6 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad -F test/wal.test 0148c8b3421a25fdb4d9c160e84a681d0646371b -F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada -F test/wal3.test 5dd734147f1f8f958c5261a1f2775d346d7013ce +F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 +F test/wal2.test 56b0bc95b8693a0be294f8d210c49025dd094bd7 +F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 88b5d9a6a3d1532497ee9f4296f010d66f07e33c +F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 F test/wal6.test a9d6aa635b9d63607dabdc11406f5f96ca986635 F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd -F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 +F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe -F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 +F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af +F test/walcrash4.test e7b6e7639a950a0cca8e210e248c8dad4d63bf20 F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 -F test/walprotocol.test 059cb75484a1ecf6357a2c1b3324b8156749221e -F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 +F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 +F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 9902a3d84e9bc80357a2c54ed0e76c0d6d04a867 -F test/where2.test af78c55589cbc82d793449493adba0dc3d659f23 -F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e +F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757 +F test/where2.test 478d2170637b9211f593120648858593bf2445a1 +F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1337,7 +1410,7 @@ F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2 F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 -F test/whereD.test 51366b07cb6f546cd30cc803f7e754f063b940de +F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501 @@ -1351,6 +1424,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d +F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc F test/with1.test cef099a491eac9874f2c28bd2dc86394fb3e47b3 F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87 F test/with3.test 511bacdbe41c49cf34f9fd1bd3245fe1575bca98 @@ -1361,77 +1435,103 @@ F test/without_rowid3.test aad4f9d383e199349b6c7e508a778f7dff5dff79 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e -F test/wordcount.c 2a0a6c0d0e8e8bbbac1f06d72a6791828c37c0cf +F test/wordcount.c 97856eec21fd00d77da720007b1888c383f63dcf F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa -F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac +F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 -F tool/GetTclKit.bat 629d87562e0487c386db630033931d12d62e6372 -F tool/addopcodes.tcl 4ca9c3ef196f08da30add5d07ce0c9458dc8c633 -F tool/build-all-msvc.bat 31866578036cd1d962628059b0760d407c3ce4d8 x +F tool/GetTclKit.bat f94784e3bdc2f50c539266f5467cbf1f27612cb3 +F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 +F tool/addopcodes.tcl 10c889c4a65ec6c5604e4a47306fa77ff57ae189 +F tool/build-all-msvc.bat 018c1b273458a90c8ba633c6f0c5654cfcb138bf x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 +F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 -F tool/fuzzershell.c 94019b185caceffc9f7c7b678a6489e42bc2aefa +F tool/fuzzershell.c f294ca67a10e87db76af130d75b2c94be36359c6 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 799e73e19a33b8dd7767a7fa34618ed2a9c2397d -F tool/lempar.c 3ec1463a034b37d87d782be5f6b8b10a3b1ecbe7 +F tool/lemon.c e4fb7d888873ac88f20a41c84a7d1e61f5209a6d +F tool/lempar.c 7cf047778e74433126cb02896e3f898d7b1e20aa +F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 -F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 -F tool/mkautoconfamal.sh c78caa3214f25dc28ea157b5a82abb311f209906 +F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca +F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 F tool/mkkeywordhash.c f7f3b342211ac6a14258b9726d5b97cf4f548f22 -F tool/mkmsvcmin.tcl d57e6efc9428605f5418d0b235721ddf7b5d9c0b +F tool/mkmsvcmin.tcl 2f12f7fa8858bbe61cf81820a2da96c79ed1ca8d F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c -F tool/mkopcodeh.tcl 385c62d78c38b2d92146dcb5abd319dbbc33506d +F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl f0d5bb266d1d388cf86fce5ba01a891e95d72d41 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 87240b09c20042999b41d5fabe091b7111287835 -F tool/mksqlite3c.tcl b66b4170f693602cd6985aed15d9509fe2f18c84 -F tool/mksqlite3h.tcl 1d41ab59bffb025121f75b76e183125ce41b3ec8 +F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb +F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e +F tool/mksqlite3h.tcl c006c4e5da57c649b24b689511dcd270dd7b0249 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b -F tool/mkvsix.tcl fbeb0af7cffdf64e0fba6d65e2e5120dc14595f4 +F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 +F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b -F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0 +F tool/replace.tcl 60f91e8dd06ab81f74d213ecbd9c9945f32ac048 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a -F tool/showdb.c 82dca79a999b2701c62417636345e9974151fdad +F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 +F tool/showdb.c c695a5d5c8110640e0d9fadf5e254da90c79c36e F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 F tool/showstat4.c bda40d6e395df7edb6e9ea630784d3d762c35b4b F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl 93c1fdc9733c525b17a2024c7df193daa002e037 +F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a +F tool/speed-check.sh da6ce45957c509ba6343fe3fef167e7e2b306262 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 5a26205111e6fa856d9b1535b1637744dcdb930b -F tool/srcck1.c 4c39bdfa9a92edd20233ee720df84dbeb2417602 +F tool/sqldiff.c 3fb48a6c6669d2d2c59a7f072a410dd2583579b4 +F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d -F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f +F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 -F tool/warnings.sh ef6ebc6fd8d2dc35db3b622015c16a023d4fef4f +F tool/warnings.sh d9ded5274b3cb947e98e70d587f71d6263f803b5 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f694e60a79024967a3f4574e3928f0b28589d381 -R f2e9debe8cf42a0179962a66a44645cf +F vsixtest/App.xaml b76d3b48860e7454775c47ea38ffea9c4abe3e85 +F vsixtest/App.xaml.cpp c465147f50871165c60ca16955219f6c5812d6d8 +F vsixtest/App.xaml.h 4a9768e2983d05600ad1e1c2f1b00a132967da9f +F vsixtest/Assets/LockScreenLogo.scale-200.png e820c9a3deb909197081b0bf3216c06e13905f0a +F vsixtest/Assets/SplashScreen.scale-200.png cab70988ca71bebec7bfeb3b6dbafe17b9ab0b4a +F vsixtest/Assets/Square150x150Logo.scale-200.png e17b40817db7a239fc239d83efcc951fb824e3ff +F vsixtest/Assets/Square44x44Logo.scale-200.png 2f166237094dea94d952d10b9eeae81806844f1c +F vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png 5f6a6d391b95a3061ccca6e6fdd6955ede63b4ed +F vsixtest/Assets/StoreLogo.png 0828b7257db74a4ecd5eeb6b7b4971f0fdc4d9d1 +F vsixtest/Assets/Wide310x150Logo.scale-200.png 04ddefe5bc5f43ae12a7433f6f236ddab101ac42 +F vsixtest/MainPage.xaml 34f49897e3ca533a7e74506ba0759b66eebce151 +F vsixtest/MainPage.xaml.cpp 7f31fc6de751b64676c0924c97a5485d950a91d7 +F vsixtest/MainPage.xaml.h cc05cca10d50a003f6c6e4448b701cdd07f52f29 +F vsixtest/Package.appxmanifest 6b6db1eb7df3a315c5d681059754d5f0e0c47a93 +F vsixtest/pch.cpp cb823cfac36f1a39a7eb0acbd7e9a0b0de8f23af +F vsixtest/pch.h 9cab7980f2ac4baa40807d8b5e52af32a21cf78c +F vsixtest/vsixtest.sln 77cadbe4e96c1fe1bf51cd77de9e9b0a12ada547 +F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 +F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc +F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e +F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 +P e31d646a319e03587a913d8e23e8c9f767a3481e +R 1d14b6b58ab1fcbf27ff915cd0c67140 T +bgcolor * #d0c0ff T +sym-release * -T +sym-version-3.11.0 * +T +sym-version-3.15.2 * U drh -Z e719a8e2c047e401006dde85de2084bf +Z 790edc877406989babf0a6b5cdc9cbe9 # Remove this line to create a well-formed manifest. diff --git a/manifest.uuid b/manifest.uuid index 5e743126..4b7d526d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d862f207e3adc00f78066799ac5a8c282430a5f +bbd85d235f7037c6a033a9690534391ffeacecc8 diff --git a/src/alter.c b/src/alter.c index 34221777..2c86a8ba 100644 --- a/src/alter.c +++ b/src/alter.c @@ -229,7 +229,7 @@ static void renameTriggerFunc( ** Register built-in functions used to help implement ALTER TABLE */ void sqlite3AlterFunctions(void){ - static SQLITE_WSD FuncDef aAlterTableFuncs[] = { + static FuncDef aAlterTableFuncs[] = { FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), #ifndef SQLITE_OMIT_TRIGGER FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), @@ -238,13 +238,7 @@ void sqlite3AlterFunctions(void){ FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), #endif }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAlterTableFuncs); - - for(i=0; ia[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; db->flags |= SQLITE_PreferBuiltin; /* Get a NULL terminated version of the new table name. */ @@ -607,6 +601,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + int r1; /* Temporary registers */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; @@ -616,7 +611,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ assert( sqlite3BtreeHoldsAllMutexes(db) ); iDb = sqlite3SchemaToIndex(db, pNew->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ pCol = &pNew->aCol[pNew->nCol-1]; pDflt = pCol->pDflt; @@ -634,7 +629,8 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** literal NULL, then set pDflt to 0. This simplifies checking ** for an SQL NULL default below. */ - if( pDflt && pDflt->op==TK_NULL ){ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ pDflt = 0; } @@ -700,16 +696,18 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ db->flags = savedDbFlags; } - /* If the default value of the new column is NULL, then the file - ** format to 2. If the default value of the new column is not NULL, - ** the file format be 3. Back when this feature was first added - ** in 2006, we went to the trouble to upgrade the file format to the - ** minimum support values. But 10-years on, we can assume that all - ** extent versions of SQLite support file-format 4, so we always and - ** unconditionally upgrade to 4. + /* Make sure the schema version is at least 3. But do not upgrade + ** from less than 3 to 4, as that will corrupt any preexisting DESC + ** index. */ - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, - SQLITE_MAX_FILE_FORMAT); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); /* Reload the schema of the modified table. */ reloadTableSchema(pParse, pTab, pTab->zName); @@ -791,9 +789,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Column *pCol = &pNew->aCol[i]; pCol->zName = sqlite3DbStrDup(db, pCol->zName); pCol->zColl = 0; - pCol->zType = 0; pCol->pDflt = 0; - pCol->zDflt = 0; } pNew->pSchema = db->aDb[iDb].pSchema; pNew->addColOffset = pTab->addColOffset; diff --git a/src/analyze.c b/src/analyze.c index 1e026a75..e3955f30 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -210,14 +210,14 @@ static void openStatTable( for(i=0; izName))==0 ){ + if( (pStat = sqlite3FindTable(db, zTab, pDb->zDbSName))==0 ){ if( aTable[i].zCols ){ /* The sqlite_statN table does not exist. Create it. Note that a ** side-effect of the CREATE TABLE statement is to leave the rootpage ** of the new table in register pParse->regRoot. This is important ** because the OpenWrite opcode below will be needing it. */ sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); aRoot[i] = pParse->regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; @@ -232,7 +232,7 @@ static void openStatTable( if( zWhere ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE %s=%Q", - pDb->zName, zTab, zWhereType, zWhere + pDb->zDbSName, zTab, zWhereType, zWhere ); }else{ /* The sqlite_stat[134] table already exists. Delete all rows. */ @@ -481,8 +481,7 @@ static const FuncDef statInitFuncdef = { statInit, /* xSFunc */ 0, /* xFinalize */ "stat_init", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; #ifdef SQLITE_ENABLE_STAT4 @@ -781,8 +780,7 @@ static const FuncDef statPushFuncdef = { statPush, /* xSFunc */ 0, /* xFinalize */ "stat_push", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; #define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ @@ -927,8 +925,7 @@ static const FuncDef statGetFuncdef = { statGet, /* xSFunc */ 0, /* xFinalize */ "stat_get", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ @@ -997,7 +994,7 @@ static void analyzeOneTable( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -1387,7 +1384,7 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ /* Form 3: Analyze the fully qualified table name */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -1673,7 +1670,7 @@ static int loadStatTbl( assert( db->lookaside.bDisable ); zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -1713,7 +1710,7 @@ static int loadStatTbl( pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; pIdx->aAvgEq = pSpace; pSpace += nIdxCol; @@ -1729,7 +1726,7 @@ static int loadStatTbl( zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); sqlite3DbFree(db, zSql); @@ -1767,7 +1764,7 @@ static int loadStatTbl( pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); if( pSample->p==0 ){ sqlite3_finalize(pStmt); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); pIdx->nSample++; @@ -1829,7 +1826,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ analysisInfo sInfo; HashElem *i; char *zSql; - int rc; + int rc = SQLITE_OK; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); @@ -1838,30 +1835,33 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - sqlite3DefaultRowEst(pIdx); + pIdx->aiRowLogEst[0] = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif } - /* Check to make sure the sqlite_stat1 table exists */ - sInfo.db = db; - sInfo.zDatabase = db->aDb[iDb].zName; - if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ - return SQLITE_ERROR; - } - /* Load new statistics out of the sqlite_stat1 table */ - zSql = sqlite3MPrintf(db, - "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); - sqlite3DbFree(db, zSql); + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zDbSName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)!=0 ){ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); + } } + /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + } /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 diff --git a/src/attach.c b/src/attach.c index 2288ac9b..4fe97bac 100644 --- a/src/attach.c +++ b/src/attach.c @@ -97,7 +97,7 @@ static void attachFunc( goto attach_error; } for(i=0; inDb; i++){ - char *z = db->aDb[i].zName; + char *z = db->aDb[i].zDbSName; assert( z && zName ); if( sqlite3StrICmp(z, zName)==0 ){ zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); @@ -144,7 +144,7 @@ static void attachFunc( Pager *pPager; aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); if( !aNew->pSchema ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ zErrDyn = sqlite3MPrintf(db, "attached databases must use the same text encoding as main database"); @@ -161,10 +161,10 @@ static void attachFunc( #endif sqlite3BtreeLeave(aNew->pBt); } - aNew->safety_level = 3; - aNew->zName = sqlite3DbStrDup(db, zName); - if( rc==SQLITE_OK && aNew->zName==0 ){ - rc = SQLITE_NOMEM; + aNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + aNew->zDbSName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zDbSName==0 ){ + rc = SQLITE_NOMEM_BKPT; } @@ -192,7 +192,7 @@ static void attachFunc( case SQLITE_NULL: /* No key specified. Use the key from the main database */ sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey>0 || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ + if( nKey || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); } break; @@ -275,7 +275,7 @@ static void detachFunc( for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; + if( sqlite3StrICmp(pDb->zDbSName, zName)==0 ) break; } if( i>=db->nDb ){ @@ -325,6 +325,7 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; + if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; @@ -392,8 +393,7 @@ void sqlite3Detach(Parse *pParse, Expr *pDbname){ detachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } @@ -412,8 +412,7 @@ void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ attachFunc, /* xSFunc */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + {0} }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } @@ -435,7 +434,7 @@ void sqlite3FixInit( db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; - pFix->zDb = db->aDb[iDb].zName; + pFix->zDb = db->aDb[iDb].zDbSName; pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; @@ -532,7 +531,7 @@ int sqlite3FixExpr( return 1; } } - if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; + if( ExprHasProperty(pExpr, EP_TokenOnly|EP_Leaf) ) break; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; }else{ diff --git a/src/auth.c b/src/auth.c index 9768fc2f..77a95d4a 100644 --- a/src/auth.c +++ b/src/auth.c @@ -107,10 +107,11 @@ int sqlite3AuthReadCol( const char *zCol, /* Column name */ int iDb /* Index of containing database. */ ){ - sqlite3 *db = pParse->db; /* Database handle */ - char *zDb = db->aDb[iDb].zName; /* Name of attached database */ - int rc; /* Auth callback return code */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ + int rc; /* Auth callback return code */ + if( db->init.busy ) return SQLITE_OK; rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext #ifdef SQLITE_USER_AUTHENTICATION ,db->auth.zAuthUser diff --git a/src/backup.c b/src/backup.c index 1c282242..165144d9 100644 --- a/src/backup.c +++ b/src/backup.c @@ -83,22 +83,16 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int i = sqlite3FindDbName(pDb, zDb); if( i==1 ){ - Parse *pParse; + Parse sParse; int rc = 0; - pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); - if( pParse==0 ){ - sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); - rc = SQLITE_NOMEM; - }else{ - pParse->db = pDb; - if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); - rc = SQLITE_ERROR; - } - sqlite3DbFree(pErrorDb, pParse->zErrMsg); - sqlite3ParserReset(pParse); - sqlite3StackFree(pErrorDb, pParse); + memset(&sParse, 0, sizeof(sParse)); + sParse.db = pDb; + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + rc = SQLITE_ERROR; } + sqlite3DbFree(pErrorDb, sParse.zErrMsg); + sqlite3ParserReset(&sParse); if( rc ){ return 0; } @@ -182,7 +176,7 @@ sqlite3_backup *sqlite3_backup_init( ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM); + sqlite3Error(pDestDb, SQLITE_NOMEM_BKPT); } } @@ -196,7 +190,6 @@ sqlite3_backup *sqlite3_backup_init( p->isAttached = 0; if( 0==p->pSrc || 0==p->pDest - || setDestPgsz(p)==SQLITE_NOMEM || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK ){ /* One (or both) of the named databases did not exist or an OOM @@ -384,14 +377,6 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = SQLITE_OK; } - /* Lock the destination database, if it is not locked already. */ - if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) - ){ - p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); - } - /* If there is no open read-transaction on the source database, open ** one now. If a transaction is opened here, then it will be closed ** before this function exits. @@ -401,6 +386,24 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ bCloseTrans = 1; } + /* If the destination database has not yet been locked (i.e. if this + ** is the first call to backup_step() for the current backup operation), + ** try to set its page size to the same as the source database. This + ** is especially important on ZipVFS systems, as in that case it is + ** not possible to create a database file that uses one page size by + ** writing to it with another. */ + if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ + rc = SQLITE_NOMEM; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + } + /* Do not allow backup if the destination database is in WAL mode ** and the page sizes are different between source and destination */ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); @@ -581,7 +584,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ } if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } p->rc = rc; } @@ -777,10 +780,10 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ ** sqlite3_backup_step(), we can guarantee that the copy finishes ** within a single call (unless an error occurs). The assert() statement ** checks this assumption - (p->rc) should be set to either SQLITE_DONE - ** or an error code. - */ + ** or an error code. */ sqlite3_backup_step(&b, 0x7FFFFFFF); assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; diff --git a/src/bitvec.c b/src/bitvec.c index f7f544cf..9d13ba91 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -177,7 +177,7 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ i = i%p->iDivisor; if( p->u.apSub[bin]==0 ){ p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); - if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM; + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM_BKPT; } p = p->u.apSub[bin]; } @@ -212,7 +212,7 @@ bitvec_set_rehash: int rc; u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); if( aiValues==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; }else{ memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); diff --git a/src/btree.c b/src/btree.c index c6f9c34f..ebffe645 100644 --- a/src/btree.c +++ b/src/btree.c @@ -350,7 +350,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ if( !pLock ){ pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); if( !pLock ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pLock->iTable = iTable; pLock->pBtree = p; @@ -450,6 +450,15 @@ static void releasePage(MemPage *pPage); /* Forward reference */ static int cursorHoldsMutex(BtCursor *p){ return sqlite3_mutex_held(p->pBt->mutex); } + +/* Verify that the cursor and the BtShared agree about what is the current +** database connetion. This is important in shared-cache mode. If the database +** connection pointers get out-of-sync, it is possible for routines like +** btreeInitPage() to reference an stale connection pointer that references a +** a connection that has already closed. This routine is used inside assert() +** statements only and for the purpose of double-checking that the btree code +** does keep the database connection pointers up-to-date. +*/ static int cursorOwnsBtShared(BtCursor *p){ assert( cursorHoldsMutex(p) ); return (p->pBtree->db==p->pBt->db); @@ -553,7 +562,7 @@ static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ assert( pgno<=pBt->nPage ); pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); if( !pBt->pHasContent ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ @@ -609,21 +618,19 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ ** the key. */ static int saveCursorKey(BtCursor *pCur){ - int rc; + int rc = SQLITE_OK; assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. */ - if( 0==pCur->curIntKey ){ - void *pKey = sqlite3Malloc( pCur->nKey ); + if( pCur->curIntKey ){ + /* Only the rowid is required for a table btree */ + pCur->nKey = sqlite3BtreeIntegerKey(pCur); + }else{ + /* For an index btree, save the complete key content */ + void *pKey; + pCur->nKey = sqlite3BtreePayloadSize(pCur); + pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ @@ -632,7 +639,7 @@ static int saveCursorKey(BtCursor *pCur){ sqlite3_free(pKey); } }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } assert( !pCur->curIntKey || !pCur->pKey ); @@ -756,7 +763,7 @@ static int btreeMoveto( ){ int rc; /* Status code */ UnpackedRecord *pIdxKey; /* Unpacked index key */ - char aSpace[200]; /* Temp space for pIdxKey - to avoid a malloc */ + char aSpace[384]; /* Temp space for pIdxKey - to avoid a malloc */ char *pFree = 0; if( pKey ){ @@ -764,7 +771,7 @@ static int btreeMoveto( pIdxKey = sqlite3VdbeAllocUnpackedRecord( pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree ); - if( pIdxKey==0 ) return SQLITE_NOMEM; + if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 ){ sqlite3DbFree(pCur->pKeyInfo->db, pFree); @@ -1598,8 +1605,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( data[iPtr+1]==0 && data[iPtr]==0 ){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ - while( (iFreeBlk = get2byte(&data[iPtr]))>0 && iFreeBlkiLast ) return SQLITE_CORRUPT_BKPT; @@ -1676,11 +1686,11 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior - ** table b-tree page. */ + /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an + ** interior table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf - ** table b-tree page. */ + /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a + ** leaf table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; if( pPage->leaf ){ @@ -1694,11 +1704,11 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior - ** index b-tree page. */ + /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an + ** interior index b-tree page. */ assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf - ** index b-tree page. */ + /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a + ** leaf index b-tree page. */ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; @@ -2176,7 +2186,7 @@ int sqlite3BtreeOpen( } p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } p->inTrans = TRANS_NONE; p->db = db; @@ -2200,7 +2210,7 @@ int sqlite3BtreeOpen( p->sharable = 1; if( !zFullPathname ){ sqlite3_free(p); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( isMemdb ){ memcpy(zFullPathname, zFilename, nFilename); @@ -2268,7 +2278,7 @@ int sqlite3BtreeOpen( pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, @@ -2330,14 +2340,14 @@ int sqlite3BtreeOpen( #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* Add the new BtShared object to the linked list sharable BtShareds. */ + pBt->nRef = 1; if( p->sharable ){ MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) - pBt->nRef = 1; MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } } @@ -2360,12 +2370,12 @@ int sqlite3BtreeOpen( for(i=0; inDb; i++){ if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ while( pSib->pPrev ){ pSib = pSib->pPrev; } - if( p->pBtpBt ){ + if( (uptr)p->pBt<(uptr)pSib->pBt ){ p->pNext = pSib; p->pPrev = 0; pSib->pPrev = p; }else{ - while( pSib->pNext && pSib->pNext->pBtpBt ){ + while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ pSib = pSib->pNext; } p->pNext = pSib->pNext; @@ -2403,6 +2413,7 @@ btree_open_out: assert( sqlite3_mutex_held(mutexOpen) ); sqlite3_mutex_leave(mutexOpen); } + assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); return rc; } @@ -2619,21 +2630,6 @@ int sqlite3BtreeSetPagerFlags( } #endif -/* -** Return TRUE if the given btree is set to safety level 1. In other -** words, return TRUE if no sync() occurs on the disk files. -*/ -int sqlite3BtreeSyncDisabled(Btree *p){ - BtShared *pBt = p->pBt; - int rc; - assert( sqlite3_mutex_held(p->db->mutex) ); - sqlite3BtreeEnter(p); - assert( pBt && pBt->pPager ); - rc = sqlite3PagerNosync(pBt->pPager); - sqlite3BtreeLeave(p); - return rc; -} - /* ** Change the default pages size and the number of reserved bytes per page. ** Or, if the page size has already been fixed, return SQLITE_READONLY @@ -2879,9 +2875,25 @@ static int lockBtree(BtShared *pBt){ rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ goto page1_init_failed; - }else if( isOpen==0 ){ - releasePage(pPage1); - return SQLITE_OK; + }else{ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 + ){ + pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +#endif + if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; + } } rc = SQLITE_NOTADB; } @@ -4114,7 +4126,7 @@ static int btreeCursor( if( wrFlag ){ allocateTempSpace(pBt); - if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); @@ -4261,46 +4273,33 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){ #endif /* NDEBUG */ /* -** Set *pSize to the size of the buffer needed to hold the value of -** the key for the current entry. If the cursor is not pointing -** to a valid entry, *pSize is set to 0. -** -** For a table with the INTKEY flag set, this routine returns the key -** itself, not the number of bytes in the key. -** -** The caller must position the cursor prior to invoking this routine. -** -** This routine cannot fail. It always returns SQLITE_OK. +** Return the value of the integer key or "rowid" for a table btree. +** This routine is only valid for a cursor that is pointing into a +** ordinary table btree. If the cursor points to an index btree or +** is invalid, the result of this routine is undefined. */ -int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ +i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); + assert( pCur->curIntKey ); getCellInfo(pCur); - *pSize = pCur->info.nKey; - return SQLITE_OK; + return pCur->info.nKey; } /* -** Set *pSize to the number of bytes of data in the entry the -** cursor currently points to. +** Return the number of bytes of payload for the entry that pCur is +** currently pointing to. For table btrees, this will be the amount +** of data. For index btrees, this will be the size of the key. ** ** The caller must guarantee that the cursor is pointing to a non-NULL ** valid entry. In other words, the calling procedure must guarantee ** that the cursor has Cursor.eState==CURSOR_VALID. -** -** Failure is not possible. This function always returns SQLITE_OK. -** It might just as well be a procedure (returning void) but we continue -** to return an integer result code for historical reasons. */ -int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ - assert( cursorOwnsBtShared(pCur) ); +u32 sqlite3BtreePayloadSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 ); - assert( pCur->iPageapPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); - *pSize = pCur->info.nPayload; - return SQLITE_OK; + return pCur->info.nPayload; } /* @@ -4471,8 +4470,13 @@ static int accessPayload( #endif assert( offset+amt <= pCur->info.nPayload ); - if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ - /* Trying to read or write past the end of the data is an error */ + assert( aPayload > pPage->aData ); + if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ + /* Trying to read or write past the end of the data is an error. The + ** conditional above is really: + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ return SQLITE_CORRUPT_BKPT; } @@ -4512,7 +4516,7 @@ static int accessPayload( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; @@ -4597,7 +4601,7 @@ static int accessPayload( && (bEnd || a==ovflSize) /* (6) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && pBt->pPage1->aData[19]==0x01 /* (5) */ + && 0==sqlite3PagerUseWal(pBt->pPager) /* (5) */ && &pBuf[-4]>=pBufStart /* (7) */ ){ u8 aSave[4]; @@ -4737,10 +4741,7 @@ static const void *fetchPayload( ** These routines is used to get quick access to key and data ** in the common case where no overflow pages are used. */ -const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){ - return fetchPayload(pCur, pAmt); -} -const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ +const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ return fetchPayload(pCur, pAmt); } @@ -5073,11 +5074,12 @@ int sqlite3BtreeMovetoUnpacked( assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( pRes ); assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) ); /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 - && pCur->curIntKey + if( pIdxKey==0 + && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 ){ if( pCur->info.nKey==intKey ){ *pRes = 0; @@ -5217,7 +5219,7 @@ int sqlite3BtreeMovetoUnpacked( } pCellKey = sqlite3Malloc( nCell+18 ); if( pCellKey==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; @@ -6066,9 +6068,7 @@ static int clearCell( static int fillInCell( MemPage *pPage, /* The page that contains the cell */ unsigned char *pCell, /* Complete text of the cell */ - const void *pKey, i64 nKey, /* The key */ - const void *pData,int nData, /* The data */ - int nZero, /* Extra zero bytes to append to pData */ + const BtreePayload *pX, /* Payload with which to construct the cell */ int *pnSize /* Write cell size here */ ){ int nPayload; @@ -6092,26 +6092,21 @@ static int fillInCell( /* Fill in the header. */ nHeader = pPage->childPtrSize; - nPayload = nData + nZero; - if( pPage->intKeyLeaf ){ - nHeader += putVarint32(&pCell[nHeader], nPayload); - }else{ - assert( nData==0 ); - assert( nZero==0 ); - } - nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - - /* Fill in the payload size */ if( pPage->intKey ){ - pSrc = pData; - nSrc = nData; - nData = 0; - }else{ - assert( nKey<=0x7fffffff && pKey!=0 ); - nPayload = (int)nKey; - pSrc = pKey; - nSrc = (int)nKey; + nPayload = pX->nData + pX->nZero; + pSrc = pX->pData; + nSrc = pX->nData; + assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nHeader += putVarint32(&pCell[nHeader], nPayload); + nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); + }else{ + assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); + nSrc = nPayload = (int)pX->nKey; + pSrc = pX->pKey; + nHeader += putVarint32(&pCell[nHeader], nPayload); } + + /* Fill in the payload */ if( nPayload<=pPage->maxLocal ){ n = nHeader + nPayload; testcase( n==3 ); @@ -6149,7 +6144,7 @@ static int fillInCell( CellInfo info; pPage->xParseCell(pPage, pCell, &info); assert( nHeader==(int)(info.pPayload - pCell) ); - assert( info.nKey==nKey ); + assert( info.nKey==pX->nKey ); assert( *pnSize == info.nSize ); assert( spaceLeft == info.nLocal ); } @@ -6234,10 +6229,6 @@ static int fillInCell( pSrc += n; nSrc -= n; spaceLeft -= n; - if( nSrc==0 ){ - nSrc = nData; - pSrc = pData; - } } releasePage(pToRelease); return SQLITE_OK; @@ -6304,6 +6295,8 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. +** +** *pRC must be SQLITE_OK when this routine is called. */ static void insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -6319,8 +6312,7 @@ static void insertCell( u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - if( *pRC ) return; - + assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -6394,7 +6386,7 @@ static void insertCell( /* ** A CellArray object contains a cache of pointers and sizes for a -** consecutive sequence of cells that might be held multiple pages. +** consecutive sequence of cells that might be held on multiple pages. */ typedef struct CellArray CellArray; struct CellArray { @@ -6539,8 +6531,8 @@ static int pageInsertArray( u8 *pSlot; sz = cachedCellSize(pCArray, i); if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ + if( (pData - pBegin)apCell[i] will never overlap on a well-formed @@ -6702,7 +6694,7 @@ static int editPage( for(i=0; iapCell[i+iNew]; int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); - if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + if( SQLITE_WITHIN(pCell, aData, &aData[pPg->pBt->usableSize]) ){ pCell = &pTmp[pCell - aData]; } assert( 0==memcmp(pCell, &aData[iOff], @@ -6826,8 +6818,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ while( ((*(pOut++) = *(pCell++))&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + if( rc==SQLITE_OK ){ + insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + } /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); @@ -7036,7 +7030,7 @@ static int balance_nonroot( assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); if( !aOvflSpace ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* Find the sibling pages to balance. Also locate the cells in pParent @@ -7136,7 +7130,7 @@ static int balance_nonroot( assert( szScratch<=6*(int)pBt->pageSize ); b.apCell = sqlite3ScratchMalloc( szScratch ); if( b.apCell==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto balance_cleanup; } b.szCell = (u16*)&b.apCell[nMaxCells]; @@ -7347,7 +7341,7 @@ static int balance_nonroot( assert( r szLeft-(b.szCell[r]+2)) ){ + && (bBulk || szRight+b.szCell[d]+2 > szLeft-(b.szCell[r]+(i==k-1?0:2)))){ break; } szRight += b.szCell[d] + 2; @@ -7571,9 +7565,9 @@ static int balance_nonroot( ** any cell). But it is important to pass the correct size to ** insertCell(), so reparse the cell now. ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. + ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" + ** and WITHOUT ROWID tables with exactly one column which is the + ** primary key. */ if( b.szCell[j]==4 ){ assert(leafCorrection==4); @@ -7919,13 +7913,19 @@ static int balance(BtCursor *pCur){ /* -** Insert a new record into the BTree. The key is given by (pKey,nKey) -** and the data is given by (pData,nData). The cursor is used only to -** define what table the record should be inserted into. The cursor -** is left pointing at a random location. +** Insert a new record into the BTree. The content of the new record +** is described by the pX object. The pCur cursor is used only to +** define what table the record should be inserted into, and is left +** pointing at a random location. ** -** For an INTKEY table, only the nKey value of the key is used. pKey is -** ignored. For a ZERODATA table, the pData and nData are both ignored. +** For a table btree (used for rowid tables), only the pX.nKey value of +** the key is used. The pX.pKey value must be NULL. The pX.nKey is the +** rowid or INTEGER PRIMARY KEY of the row. The pX.nData,pData,nZero fields +** hold the content of the row. +** +** For an index btree (used for indexes and WITHOUT ROWID tables), the +** key is an arbitrary byte sequence stored in pX.pKey,nKey. The +** pX.pData,nData,nZero fields must be zero. ** ** If the seekResult parameter is non-zero, then a successful call to ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already @@ -7942,9 +7942,7 @@ static int balance(BtCursor *pCur){ */ int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ - const void *pKey, i64 nKey, /* The key of the new record */ - const void *pData, int nData, /* The data of the new record */ - int nZero, /* Number of extra 0 bytes to append to data */ + const BtreePayload *pX, /* Content of the row to be inserted */ int appendBias, /* True if this is likely an append */ int seekResult /* Result of prior MovetoUnpacked() call */ ){ @@ -7974,7 +7972,7 @@ int sqlite3BtreeInsert( ** keys with no associated data. If the cursor was opened expecting an ** intkey table, the caller should be inserting integer keys with a ** blob of associated data. */ - assert( (pKey==0)==(pCur->pKeyInfo==0) ); + assert( (pX->pKey==0)==(pCur->pKeyInfo==0) ); /* Save the positions of any other cursors open on this table. ** @@ -7993,38 +7991,38 @@ int sqlite3BtreeInsert( } if( pCur->pKeyInfo==0 ){ - assert( pKey==0 ); + assert( pX->pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ - invalidateIncrblobCursors(p, nKey, 0); + invalidateIncrblobCursors(p, pX->nKey, 0); /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary ** btreeMoveto() call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 - && pCur->info.nKey==nKey-1 ){ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0 + && pCur->info.nKey==pX->nKey-1 ){ loc = -1; }else if( loc==0 ){ - rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc); + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, appendBias, &loc); if( rc ) return rc; } }else if( loc==0 ){ - rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, appendBias, &loc); if( rc ) return rc; } assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); pPage = pCur->apPage[pCur->iPage]; - assert( pPage->intKey || nKey>=0 ); + assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", - pCur->pgnoRoot, nKey, nData, pPage->pgno, + pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); newCell = pBt->pTmpSpace; assert( newCell!=0 ); - rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); + rc = fillInCell(pPage, newCell, pX, &szNew); if( rc ) goto end_insert; assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); @@ -8050,6 +8048,7 @@ int sqlite3BtreeInsert( assert( pPage->leaf ); } insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); /* If no error has occurred and pPage has an overflow cell, call balance() @@ -8073,7 +8072,8 @@ int sqlite3BtreeInsert( ** row without seeking the cursor. This can be a big performance boost. */ pCur->info.nSize = 0; - if( rc==SQLITE_OK && pPage->nOverflow ){ + if( pPage->nOverflow ){ + assert( rc==SQLITE_OK ); pCur->curFlags &= ~(BTCF_ValidNKey); rc = balance(pCur); @@ -8134,6 +8134,28 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ pPage = pCur->apPage[iCellDepth]; pCell = findCell(pPage, iCellIdx); + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + /* If the page containing the entry to delete is not a leaf page, move ** the cursor to the largest entry in the tree that is smaller than ** the entry being deleted. This cell will replace the cell being deleted @@ -8160,28 +8182,6 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } - /* If the bPreserve flag is set to true, then the cursor position must - ** be preserved following this delete operation. If the current delete - ** will cause a b-tree rebalance, then this is done by saving the cursor - ** key and leaving the cursor in CURSOR_REQUIRESEEK state before - ** returning. - ** - ** Or, if the current delete will not cause a rebalance, then the cursor - ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately - ** before or after the deleted entry. In this case set bSkipnext to true. */ - if( bPreserve ){ - if( !pPage->leaf - || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) - ){ - /* A b-tree rebalance will be required after deleting this entry. - ** Save the cursor key. */ - rc = saveCursorKey(pCur); - if( rc ) return rc; - }else{ - bSkipnext = 1; - } - } - /* Make the page containing the entry to be deleted writable. Then free any ** overflow pages associated with the entry and finally remove the cell ** itself from within the page. */ @@ -8209,7 +8209,9 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + if( rc==SQLITE_OK ){ + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; } @@ -9698,4 +9700,14 @@ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } int sqlite3BtreeSharable(Btree *p){ return p->sharable; } + +/* +** Return the number of connections to the BtShared object accessed by +** the Btree handle passed as the only argument. For private caches +** this is always 1. For shared caches it may be 1 or greater. +*/ +int sqlite3BtreeConnectionCount(Btree *p){ + testcase( p->sharable ); + return p->pBt->nRef; +} #endif diff --git a/src/btree.h b/src/btree.h index 30522e99..0df98a3a 100644 --- a/src/btree.h +++ b/src/btree.h @@ -13,8 +13,8 @@ ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. */ -#ifndef _BTREE_H_ -#define _BTREE_H_ +#ifndef SQLITE_BTREE_H +#define SQLITE_BTREE_H /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. @@ -39,6 +39,7 @@ typedef struct Btree Btree; typedef struct BtCursor BtCursor; typedef struct BtShared BtShared; +typedef struct BtreePayload BtreePayload; int sqlite3BtreeOpen( @@ -68,7 +69,6 @@ int sqlite3BtreeSetSpillSize(Btree*,int); int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); #endif int sqlite3BtreeSetPagerFlags(Btree*,unsigned); -int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); int sqlite3BtreeGetPageSize(Btree*); int sqlite3BtreeMaxPageCount(Btree*,int); @@ -90,7 +90,9 @@ int sqlite3BtreeIsInReadTrans(Btree*); int sqlite3BtreeIsInBackup(Btree*); void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); int sqlite3BtreeSchemaLocked(Btree *pBtree); +#ifndef SQLITE_OMIT_SHARED_CACHE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +#endif int sqlite3BtreeSavepoint(Btree *, int, int); const char *sqlite3BtreeGetFilename(Btree *); @@ -251,26 +253,52 @@ int sqlite3BtreeDelete(BtCursor*, u8 flags); #define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ #define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ -int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, - const void *pData, int nData, - int nZero, int bias, int seekResult); +/* An instance of the BtreePayload object describes the content of a single +** entry in either an index or table btree. +** +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain +** an arbitrary key and no data. These btrees have pKey,nKey set to their +** key and pData,nData,nZero set to zero. +** +** Table btrees (used for rowid tables) contain an integer rowid used as +** the key and passed in the nKey field. The pKey field is zero. +** pData,nData hold the content of the new entry. nZero extra zero bytes +** are appended to the end of the content when constructing the entry. +** +** This object is used to pass information into sqlite3BtreeInsert(). The +** same information used to be passed as five separate parameters. But placing +** the information into this object helps to keep the interface more +** organized and understandable, and it also helps the resulting code to +** run a little faster by using fewer registers for parameter passing. +*/ +struct BtreePayload { + const void *pKey; /* Key content for indexes. NULL for tables */ + sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ + const void *pData; /* Data for tables. NULL for indexes */ + int nData; /* Size of pData. 0 if none. */ + int nZero; /* Extra zero data appended after pData,nData */ +}; + +int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, + int bias, int seekResult); int sqlite3BtreeFirst(BtCursor*, int *pRes); int sqlite3BtreeLast(BtCursor*, int *pRes); int sqlite3BtreeNext(BtCursor*, int *pRes); int sqlite3BtreeEof(BtCursor*); int sqlite3BtreePrevious(BtCursor*, int *pRes); -int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); +i64 sqlite3BtreeIntegerKey(BtCursor*); int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); -const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); -const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); -int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); +const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); +u32 sqlite3BtreePayloadSize(BtCursor*); int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); +#ifndef SQLITE_OMIT_INCRBLOB int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeIncrblobCursor(BtCursor *); +#endif void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); @@ -304,11 +332,13 @@ void sqlite3BtreeCursorList(Btree*); void sqlite3BtreeEnterAll(sqlite3*); int sqlite3BtreeSharable(Btree*); void sqlite3BtreeEnterCursor(BtCursor*); + int sqlite3BtreeConnectionCount(Btree*); #else # define sqlite3BtreeEnter(X) # define sqlite3BtreeEnterAll(X) # define sqlite3BtreeSharable(X) 0 # define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeConnectionCount(X) 1 #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE @@ -333,4 +363,4 @@ void sqlite3BtreeCursorList(Btree*); #endif -#endif /* _BTREE_H_ */ +#endif /* SQLITE_BTREE_H */ diff --git a/src/build.c b/src/build.c index 250dc20d..508e747e 100644 --- a/src/build.c +++ b/src/build.c @@ -146,15 +146,14 @@ void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ - while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); #if SQLITE_USER_AUTHENTICATION if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelrc = SQLITE_AUTH_USER; sqlite3ErrorMsg(pParse, "user not authenticated"); + pParse->rc = SQLITE_AUTH_USER; return; } } @@ -173,14 +172,16 @@ void sqlite3FinishCoding(Parse *pParse){ assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); sqlite3VdbeJumpHere(v, 0); for(iDb=0; iDbnDb; iDb++){ + Schema *pSchema; if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; sqlite3VdbeAddOp4Int(v, OP_Transaction, /* Opcode */ iDb, /* P1 */ DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pParse->cookieValue[iDb], /* P3 */ - db->aDb[iDb].pSchema->iGeneration /* P4 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); VdbeComment((v, @@ -231,16 +232,6 @@ void sqlite3FinishCoding(Parse *pParse){ }else{ pParse->rc = SQLITE_ERROR; } - - /* We are done with this Parse object. There is no need to de-initialize it */ -#if 0 - pParse->colNamesSet = 0; - pParse->nTab = 0; - pParse->nMem = 0; - pParse->nSet = 0; - pParse->nVar = 0; - DbMaskZero(pParse->cookieMask); -#endif } /* @@ -260,8 +251,7 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ char *zSql; char *zErrMsg = 0; sqlite3 *db = pParse->db; -# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) - char saveBuf[SAVE_SZ]; + char saveBuf[PARSE_TAIL_SZ]; if( pParse->nErr ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ @@ -272,12 +262,12 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ return; /* A malloc must have failed */ } pParse->nested++; - memcpy(saveBuf, &pParse->nVar, SAVE_SZ); - memset(&pParse->nVar, 0, SAVE_SZ); + memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); sqlite3RunParser(pParse, zSql, &zErrMsg); sqlite3DbFree(db, zErrMsg); sqlite3DbFree(db, zSql); - memcpy(&pParse->nVar, saveBuf, SAVE_SZ); + memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); pParse->nested--; } @@ -318,10 +308,11 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ #endif for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; - assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); - if( p ) break; + if( zDatabase==0 || sqlite3StrICmp(zDatabase, db->aDb[j].zDbSName)==0 ){ + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); + if( p ) break; + } } return p; } @@ -338,7 +329,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ */ Table *sqlite3LocateTable( Parse *pParse, /* context in which to report errors */ - int isView, /* True if looking for a VIEW rather than a TABLE */ + u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ const char *zName, /* Name of the table we are looking for */ const char *zDbase /* Name of the database. Might be NULL */ ){ @@ -352,7 +343,7 @@ Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ - const char *zMsg = isView ? "no such view" : "no such table"; + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; #ifndef SQLITE_OMIT_VIRTUALTABLE if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ /* If zName is the not the name of a table in the schema created using @@ -364,12 +355,14 @@ Table *sqlite3LocateTable( } } #endif - if( zDbase ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); - }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + if( (flags & LOCATE_NOERR)==0 ){ + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + pParse->checkSchema = 1; } - pParse->checkSchema = 1; } return p; @@ -386,18 +379,18 @@ Table *sqlite3LocateTable( */ Table *sqlite3LocateTableItem( Parse *pParse, - int isView, + u32 flags, struct SrcList_item *p ){ const char *zDb; assert( p->pSchema==0 || p->zDatabase==0 ); if( p->pSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); - zDb = pParse->db->aDb[iDb].zName; + zDb = pParse->db->aDb[iDb].zDbSName; }else{ zDb = p->zDatabase; } - return sqlite3LocateTable(pParse, isView, p->zName, zDb); + return sqlite3LocateTable(pParse, flags, p->zName, zDb); } /* @@ -421,7 +414,7 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; + if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zDbSName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; @@ -490,8 +483,8 @@ void sqlite3CollapseDatabaseArray(sqlite3 *db){ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ - sqlite3DbFree(db, pDb->zName); - pDb->zName = 0; + sqlite3DbFree(db, pDb->zDbSName); + pDb->zDbSName = 0; continue; } if( jnCol; i++, pCol++){ sqlite3DbFree(db, pCol->zName); sqlite3ExprDelete(db, pCol->pDflt); - sqlite3DbFree(db, pCol->zDflt); - sqlite3DbFree(db, pCol->zType); sqlite3DbFree(db, pCol->zColl); } sqlite3DbFree(db, pTable->aCol); @@ -594,16 +585,10 @@ void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ ** db parameter can be used with db->pnBytesFreed to measure the memory ** used by the Table object. */ -void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ +static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ - assert( !pTable || pTable->nRef>0 ); - - /* Do not delete the table until the reference count reaches zero. */ - if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - /* Record the number of outstanding lookaside allocations in schema Tables ** prior to doing any free() operations. Since schema Tables do not use ** lookaside, this number should not change. */ @@ -613,8 +598,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; - assert( pIndex->pSchema==pTable->pSchema ); - if( !db || db->pnBytesFreed==0 ){ + assert( pIndex->pSchema==pTable->pSchema + || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); + if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -643,6 +629,13 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Verify that no lookaside memory was used by schema tables */ assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); } +void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + deleteTable(db, pTable); +} + /* ** Unlink the given table from the hash tables and the delete the @@ -710,12 +703,8 @@ int sqlite3FindDbName(sqlite3 *db, const char *zName){ int i = -1; /* Database number */ if( zName ){ Db *pDb; - int n = sqlite3Strlen30(zName); for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( (!OMIT_TEMPDB || i!=1 ) && n==sqlite3Strlen30(pDb->zName) && - 0==sqlite3StrICmp(pDb->zName, zName) ){ - break; - } + if( 0==sqlite3StrICmp(pDb->zDbSName, zName) ) break; } } return i; @@ -774,7 +763,7 @@ int sqlite3TwoPartName( return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy ); + assert( db->init.iDb==0 || db->init.busy || (db->flags & SQLITE_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; } @@ -885,7 +874,7 @@ void sqlite3StartTable( SQLITE_CREATE_VIEW, SQLITE_CREATE_TEMP_VIEW }; - char *zDb = db->aDb[iDb].zName; + char *zDb = db->aDb[iDb].zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } @@ -904,7 +893,7 @@ void sqlite3StartTable( ** collisions. */ if( !IN_DECLARE_VTAB ){ - char *zDb = db->aDb[iDb].zName; + char *zDb = db->aDb[iDb].zDbSName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } @@ -927,7 +916,7 @@ void sqlite3StartTable( pTable = sqlite3DbMallocZero(db, sizeof(Table)); if( pTable==0 ){ assert( db->mallocFailed ); - pParse->rc = SQLITE_NOMEM; + pParse->rc = SQLITE_NOMEM_BKPT; pParse->nErr++; goto begin_table_error; } @@ -1043,10 +1032,11 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ ** first to get things going. Then this routine is called for each ** column. */ -void sqlite3AddColumn(Parse *pParse, Token *pName){ +void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ Table *p; int i; char *z; + char *zType; Column *pCol; sqlite3 *db = pParse->db; if( (p = pParse->pNewTable)==0 ) return; @@ -1056,8 +1046,11 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){ return; } #endif - z = sqlite3NameFromToken(db, pName); + z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; + memcpy(z, pName->z, pName->n); + z[pName->n] = 0; + sqlite3Dequote(z); for(i=0; inCol; i++){ if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); @@ -1079,13 +1072,21 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){ pCol->zName = z; sqlite3ColumnPropertiesFromName(p, pCol); - /* If there is no type specified, columns have the default affinity - ** 'BLOB'. If there is a type specified, then sqlite3AddColumnType() will - ** be called next to set pCol->affinity correctly. - */ - pCol->affinity = SQLITE_AFF_BLOB; - pCol->szEst = 1; + if( pType->n==0 ){ + /* If there is no type specified, columns have the default affinity + ** 'BLOB'. */ + pCol->affinity = SQLITE_AFF_BLOB; + pCol->szEst = 1; + }else{ + zType = z + sqlite3Strlen30(z) + 1; + memcpy(zType, pType->z, pType->n); + zType[pType->n] = 0; + sqlite3Dequote(zType); + pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->colFlags |= COLFLAG_HASTYPE; + } p->nCol++; + pParse->constraintName.n = 0; } /* @@ -1131,7 +1132,7 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){ char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; - if( zIn==0 ) return aff; + assert( zIn!=0 ); while( zIn[0] ){ h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; zIn++; @@ -1188,28 +1189,6 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){ return aff; } -/* -** This routine is called by the parser while in the middle of -** parsing a CREATE TABLE statement. The pFirst token is the first -** token in the sequence of tokens that describe the type of the -** column currently under construction. pLast is the last token -** in the sequence. Use this information to construct a string -** that contains the typename of the column and store that string -** in zType. -*/ -void sqlite3AddColumnType(Parse *pParse, Token *pType){ - Table *p; - Column *pCol; - - p = pParse->pNewTable; - if( p==0 || NEVER(p->nCol<1) ) return; - pCol = &p->aCol[p->nCol-1]; - assert( pCol->zType==0 || CORRUPT_DB ); - sqlite3DbFree(pParse->db, pCol->zType); - pCol->zType = sqlite3NameFromToken(pParse->db, pType); - pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst); -} - /* ** The expression is the default value for the most recently added column ** of the table currently under construction. @@ -1235,11 +1214,16 @@ void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ ** tokens that point to volatile memory. The 'span' of the expression ** is required by pragma table_info. */ + Expr x; sqlite3ExprDelete(db, pCol->pDflt); - pCol->pDflt = sqlite3ExprDup(db, pSpan->pExpr, EXPRDUP_REDUCE); - sqlite3DbFree(db, pCol->zDflt); - pCol->zDflt = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); + memset(&x, 0, sizeof(x)); + x.op = TK_SPAN; + x.u.zToken = sqlite3DbStrNDup(db, (char*)pSpan->zStart, + (int)(pSpan->zEnd - pSpan->zStart)); + x.pLeft = pSpan->pExpr; + x.flags = EP_Skip; + pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); + sqlite3DbFree(db, x.u.zToken); } } sqlite3ExprDelete(db, pSpan->pExpr); @@ -1295,10 +1279,10 @@ void sqlite3AddPrimaryKey( int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ ){ Table *pTab = pParse->pNewTable; - char *zType = 0; + Column *pCol = 0; int iCol = -1, i; int nTerm; - if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; + if( pTab==0 ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); @@ -1307,8 +1291,8 @@ void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; nTerm = 1; }else{ nTerm = pList->nExpr; @@ -1320,8 +1304,8 @@ void sqlite3AddPrimaryKey( const char *zCName = pCExpr->u.zToken; for(iCol=0; iColnCol; iCol++){ if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zType = pTab->aCol[iCol].zType; + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; break; } } @@ -1329,7 +1313,8 @@ void sqlite3AddPrimaryKey( } } if( nTerm==1 - && zType && sqlite3StrICmp(zType, "INTEGER")==0 + && pCol + && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; @@ -1343,12 +1328,8 @@ void sqlite3AddPrimaryKey( "INTEGER PRIMARY KEY"); #endif }else{ - Index *p; - p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, - 0, sortOrder, 0); - if( p ){ - p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; - } + sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); pList = 0; } @@ -1467,6 +1448,9 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ ** set back to prior value. But schema changes are infrequent ** and the probability of hitting the same cookie value is only ** 1 chance in 2^32. So we're safe enough. +** +** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments +** the schema-version whenever the schema changes. */ void sqlite3ChangeCookie(Parse *pParse, int iDb){ sqlite3 *db = pParse->db; @@ -1608,7 +1592,7 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ assert( pIdx->isResized==0 ); nByte = (sizeof(char*) + sizeof(i16) + 1)*N; zExtra = sqlite3DbMallocZero(db, nByte); - if( zExtra==0 ) return SQLITE_NOMEM; + if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); pIdx->azColl = (const char**)zExtra; zExtra += sizeof(char*)*N; @@ -1665,21 +1649,23 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** are appropriate for a WITHOUT ROWID table instead of a rowid table. ** Changes include: ** -** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is +** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (2) Convert the OP_CreateTable into an OP_CreateIndex. There is ** no rowid btree for a WITHOUT ROWID. Instead, the canonical ** data storage is a covering index btree. -** (2) Bypass the creation of the sqlite_master table entry +** (3) Bypass the creation of the sqlite_master table entry ** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. -** (3) Set the Index.tnum of the PRIMARY KEY Index object in the +** (4) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. -** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. ** (5) Add all table columns to the PRIMARY KEY Index object ** so that the PRIMARY KEY is a covering index. The surplus ** columns are part of KeyInfo.nXField and are not used for ** sorting or lookup or uniqueness checks. ** (6) Replace the rowid tail on all automatically generated UNIQUE ** indices with the PRIMARY KEY columns. +** +** For virtual tables, only (1) is performed. */ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; @@ -1689,6 +1675,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; + /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) + */ + if( !db->init.imposterTable ){ + for(i=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){ + pTab->aCol[i].notNull = OE_Abort; + } + } + } + + /* The remaining transformations only apply to b-tree tables, not to + ** virtual tables */ + if( IN_DECLARE_VTAB ) return; + /* Convert the OP_CreateTable opcode that would normally create the ** root-page for the table into an OP_CreateIndex opcode. The index ** created will become the PRIMARY KEY index. @@ -1710,9 +1710,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( pList==0 ) return; pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); - pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); - if( pPk==0 ) return; - pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, + SQLITE_IDXTYPE_PRIMARYKEY); + if( db->mallocFailed ) return; + pPk = sqlite3PrimaryKeyIndex(pTab); pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); @@ -1740,19 +1741,11 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } pPk->nKeyCol = j; } - pPk->isCovering = 1; assert( pPk!=0 ); + pPk->isCovering = 1; + if( !db->init.imposterTable ) pPk->uniqNotNull = 1; nPk = pPk->nKeyCol; - /* Make sure every column of the PRIMARY KEY is NOT NULL. (Except, - ** do not enforce this for imposter tables.) */ - if( !db->init.imposterTable ){ - for(i=0; iaCol[pPk->aiColumn[i]].notNull = OE_Abort; - } - pPk->uniqNotNull = 1; - } - /* The root page of the PRIMARY KEY is the table root page */ pPk->tnum = pTab->tnum; @@ -1996,7 +1989,7 @@ void sqlite3EndTable( "UPDATE %Q.%s " "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), zType, p->zName, p->zName, @@ -2011,13 +2004,13 @@ void sqlite3EndTable( /* Check to see if we need to create an sqlite_sequence table for ** keeping track of autoincrement keys. */ - if( p->tabFlags & TF_Autoincrement ){ + if( (p->tabFlags & TF_Autoincrement)!=0 ){ Db *pDb = &db->aDb[iDb]; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->pSeqTab==0 ){ sqlite3NestedParse(pParse, "CREATE TABLE %Q.sqlite_sequence(name,seq)", - pDb->zName + pDb->zDbSName ); } } @@ -2141,7 +2134,9 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ +#ifndef SQLITE_OMIT_AUTHORIZATION sqlite3_xauth xAuth; /* Saved xAuth pointer */ +#endif assert( pTable ); @@ -2187,44 +2182,55 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - if( pTable->pCheck ){ + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; db->lookaside.bDisable++; - sqlite3ColumnsFromExprList(pParse, pTable->pCheck, - &pTable->nCol, &pTable->aCol); - db->lookaside.bDisable--; - }else{ - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); - if( pSel ){ - n = pParse->nTab; - sqlite3SrcListAssignCursors(pParse, pSel->pSrc); - pTable->nCol = -1; - db->lookaside.bDisable++; #ifndef SQLITE_OMIT_AUTHORIZATION - xAuth = db->xAuth; - db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); - db->xAuth = xAuth; + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bDisable--; - pParse->nTab = n; - if( pSelTab ){ - assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; - pTable->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - }else{ - pTable->nCol = 0; - nErr++; + pParse->nTab = n; + if( pTable->pCheck ){ + /* CREATE VIEW name(arglist) AS ... + ** The names of the columns in the table are taken from + ** arglist which is stored in pTable->pCheck. The pCheck field + ** normally holds CHECK constraints on an ordinary table, but for + ** a VIEW it holds the list of column names. + */ + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + if( db->mallocFailed==0 + && pParse->nErr==0 + && pTable->nCol==pSel->pEList->nExpr + ){ + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); } - sqlite3SelectDelete(db, pSel); - } else { + }else if( pSelTab ){ + /* CREATE VIEW name AS... without an argument list. Construct + ** the column names from the SELECT statement that defines the view. + */ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + }else{ + pTable->nCol = 0; nErr++; } + sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDelete(db, pSel); + db->lookaside.bDisable--; + } else { + nErr++; } pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ @@ -2320,7 +2326,7 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ */ sqlite3NestedParse(pParse, "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", - pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); + pParse->db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), iTable, r1, r1); #endif sqlite3ReleaseTempReg(pParse, r1); } @@ -2396,7 +2402,7 @@ static void sqlite3ClearStatTables( const char *zName /* Name of index or table */ ){ int i; - const char *zDbName = pParse->db->aDb[iDb].zName; + const char *zDbName = pParse->db->aDb[iDb].zDbSName; for(i=1; i<=4; i++){ char zTab[24]; sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); @@ -2449,7 +2455,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ if( pTab->tabFlags & TF_Autoincrement ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName + pDb->zDbSName, pTab->zName ); } #endif @@ -2463,7 +2469,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ */ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + pDb->zDbSName, SCHEMA_TABLE(iDb), pTab->zName); if( !isView && !IsVirtual(pTab) ){ destroyTable(pParse, pTab); } @@ -2496,6 +2502,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ assert( pName->nSrc==1 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; + assert( isView==0 || isView==LOCATE_VIEW ); pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; @@ -2516,7 +2523,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ { int code; const char *zTab = SCHEMA_TABLE(iDb); - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zArg2 = 0; if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ goto exit_drop_table; @@ -2757,7 +2764,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -2773,6 +2780,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ tnum = pIndex->tnum; } pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; @@ -2796,8 +2804,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); - assert( pKey!=0 || db->mallocFailed || pParse->nErr ); - if( IsUniqueIndex(pIndex) && pKey!=0 ){ + if( IsUniqueIndex(pIndex) ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; sqlite3VdbeGoto(v, j2); addr2 = sqlite3VdbeCurrentAddr(v); @@ -2866,12 +2873,8 @@ Index *sqlite3AllocateIndexObject( ** pList is a list of columns to be indexed. pList will be NULL if this ** is a primary key or unique-constraint on the most recent column added ** to the table currently under construction. -** -** If the index is created successfully, return a pointer to the new Index -** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) */ -Index *sqlite3CreateIndex( +void sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ Token *pName1, /* First part of index name. May be NULL */ Token *pName2, /* Second part of index name. May be NULL */ @@ -2881,9 +2884,9 @@ Index *sqlite3CreateIndex( Token *pStart, /* The CREATE token that begins this statement */ Expr *pPIWhere, /* WHERE clause for partial indices */ int sortOrder, /* Sort order of primary key when pList==NULL */ - int ifNotExist /* Omit error if index already exists */ + int ifNotExist, /* Omit error if index already exists */ + u8 idxType /* The index type */ ){ - Index *pRet = 0; /* Pointer to return */ Table *pTab = 0; /* Table to be indexed */ Index *pIndex = 0; /* The index to be created */ char *zName = 0; /* Name of the index */ @@ -2901,7 +2904,10 @@ Index *sqlite3CreateIndex( char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){ + if( db->mallocFailed || pParse->nErr>0 ){ + goto exit_create_index; + } + if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ goto exit_create_index; } if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ @@ -3010,7 +3016,7 @@ Index *sqlite3CreateIndex( goto exit_create_index; } } - if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ if( !ifNotExist ){ sqlite3ErrorMsg(pParse, "index %s already exists", zName); }else{ @@ -3027,13 +3033,20 @@ Index *sqlite3CreateIndex( if( zName==0 ){ goto exit_create_index; } + + /* Automatic index names generated from within sqlite3_declare_vtab() + ** must have names that are distinct from normal automatic index names. + ** The following statement converts "sqlite3_autoindex..." into + ** "sqlite3_butoindex..." in order to make the names distinct. + ** The "vtab_err.test" test demonstrates the need of this statement. */ + if( IN_DECLARE_VTAB ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION { - const char *zDb = pDb->zName; + const char *zDb = pDb->zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; } @@ -3090,7 +3103,7 @@ Index *sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->onError = (u8)onError; pIndex->uniqNotNull = onError!=OE_None; - pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; + pIndex->idxType = idxType; pIndex->pSchema = db->aDb[iDb].pSchema; pIndex->nKeyCol = pList->nExpr; if( pPIWhere ){ @@ -3200,6 +3213,20 @@ Index *sqlite3CreateIndex( sqlite3DefaultRowEst(pIndex); if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); + /* If this index contains every column of its table, then mark + ** it as a covering index */ + assert( HasRowid(pTab) + || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ + pIndex->isCovering = 1; + for(j=0; jnCol; j++){ + if( j==pTab->iPKey ) continue; + if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; + pIndex->isCovering = 0; + break; + } + } + if( pTab==pParse->pNewTable ){ /* This routine has been called to create an automatic index as a ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or @@ -3237,7 +3264,7 @@ Index *sqlite3CreateIndex( if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; - if( z1!=z2 && sqlite3StrICmp(z1, z2) ) break; + if( sqlite3StrICmp(z1, z2) ) break; } if( k==pIdx->nKeyCol ){ if( pIdx->onError!=pIndex->onError ){ @@ -3256,7 +3283,7 @@ Index *sqlite3CreateIndex( pIdx->onError = pIndex->onError; } } - pRet = pIdx; + if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; goto exit_create_index; } } @@ -3268,6 +3295,7 @@ Index *sqlite3CreateIndex( assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; + assert( !IN_DECLARE_VTAB ); assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, pIndex->zName, pIndex); @@ -3333,7 +3361,7 @@ Index *sqlite3CreateIndex( */ sqlite3NestedParse(pParse, "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pIndex->zName, pTab->zName, iMem, @@ -3349,7 +3377,7 @@ Index *sqlite3CreateIndex( sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp1(v, OP_Expire, 0); + sqlite3VdbeAddOp0(v, OP_Expire); } sqlite3VdbeJumpHere(v, pIndex->tnum); @@ -3374,7 +3402,6 @@ Index *sqlite3CreateIndex( pIndex->pNext = pOther->pNext; pOther->pNext = pIndex; } - pRet = pIndex; pIndex = 0; } @@ -3385,7 +3412,6 @@ exit_create_index: sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); - return pRet; } /* @@ -3414,10 +3440,11 @@ void sqlite3DefaultRowEst(Index *pIdx){ int i; /* Set the first entry (number of rows in the index) to the estimated - ** number of rows in the table. Or 10, if the estimated number of rows - ** in the table is less than that. */ + ** number of rows in the table, or half the number of rows in the table + ** for a partial index. But do not let the estimate drop below 10. */ a[0] = pIdx->pTable->nRowLogEst; - if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10; assert( 10==sqlite3LogEst(2) ); + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is ** 6 and each subsequent value (if any) is 5. */ @@ -3468,7 +3495,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ { int code = SQLITE_DROP_INDEX; Table *pTab = pIndex->pTable; - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ goto exit_drop_index; @@ -3486,7 +3513,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName + db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pIndex->zName ); sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); sqlite3ChangeCookie(pParse, iDb); @@ -4007,15 +4034,13 @@ int sqlite3OpenTempDatabase(Parse *pParse){ */ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3 *db = pToplevel->db; - assert( iDb>=0 && iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDb>=0 && iDbdb->nDb ); + assert( pParse->db->aDb[iDb].pBt!=0 || iDb==1 ); assert( iDbdb, iDb, 0) ); if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ DbMaskSet(pToplevel->cookieMask, iDb); - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); } @@ -4031,7 +4056,7 @@ void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ sqlite3CodeVerifySchema(pParse, i); } } @@ -4278,7 +4303,7 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -4299,10 +4324,6 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ /* ** Return a KeyInfo structure that is appropriate for the given Index. ** -** The KeyInfo structure for an index is cached in the Index object. -** So there might be multiple references to the returned pointer. The -** caller should not try to modify the KeyInfo object. -** ** The caller should invoke sqlite3KeyInfoUnref() on the returned object ** when it has finished using it. */ diff --git a/src/callback.c b/src/callback.c index 2b955fdc..23511788 100644 --- a/src/callback.c +++ b/src/callback.c @@ -284,14 +284,12 @@ static int matchQuality( ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ static FuncDef *functionSearch( - FuncDefHash *pHash, /* Hash table to search */ int h, /* Hash of the name */ - const char *zFunc, /* Name of function */ - int nFunc /* Number of bytes in zFunc */ + const char *zFunc /* Name of function */ ){ FuncDef *p; - for(p=pHash->a[h]; p; p=p->pHash){ - if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 && p->zName[nFunc]==0 ){ + for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ + if( sqlite3StrICmp(p->zName, zFunc)==0 ){ return p; } } @@ -301,23 +299,26 @@ static FuncDef *functionSearch( /* ** Insert a new FuncDef into a FuncDefHash hash table. */ -void sqlite3FuncDefInsert( - FuncDefHash *pHash, /* The hash table into which to insert */ - FuncDef *pDef /* The function definition to insert */ +void sqlite3InsertBuiltinFuncs( + FuncDef *aDef, /* List of global functions to be inserted */ + int nDef /* Length of the apDef[] list */ ){ - FuncDef *pOther; - int nName = sqlite3Strlen30(pDef->zName); - u8 c1 = (u8)pDef->zName[0]; - int h = (sqlite3UpperToLower[c1] + nName) % ArraySize(pHash->a); - pOther = functionSearch(pHash, h, pDef->zName, nName); - if( pOther ){ - assert( pOther!=pDef && pOther->pNext!=pDef ); - pDef->pNext = pOther->pNext; - pOther->pNext = pDef; - }else{ - pDef->pNext = 0; - pDef->pHash = pHash->a[h]; - pHash->a[h] = pDef; + int i; + for(i=0; ipNext!=&aDef[i] ); + aDef[i].pNext = pOther->pNext; + pOther->pNext = &aDef[i]; + }else{ + aDef[i].pNext = 0; + aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h]; + sqlite3BuiltinFunctions.a[h] = &aDef[i]; + } } } @@ -344,8 +345,7 @@ void sqlite3FuncDefInsert( */ FuncDef *sqlite3FindFunction( sqlite3 *db, /* An open database */ - const char *zName, /* Name of the function. Not null-terminated */ - int nName, /* Number of characters in the name */ + const char *zName, /* Name of the function. zero-terminated */ int nArg, /* Number of arguments. -1 means any number */ u8 enc, /* Preferred text encoding */ u8 createFlag /* Create new entry if true and does not otherwise exist */ @@ -354,14 +354,15 @@ FuncDef *sqlite3FindFunction( FuncDef *pBest = 0; /* Best match found so far */ int bestScore = 0; /* Score of best match */ int h; /* Hash value */ + int nName; /* Length of the name */ assert( nArg>=(-2) ); assert( nArg>=(-1) || createFlag==0 ); - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % ArraySize(db->aFunc.a); + nName = sqlite3Strlen30(zName); /* First search for a match amongst the application-defined functions. */ - p = functionSearch(&db->aFunc, h, zName, nName); + p = (FuncDef*)sqlite3HashFind(&db->aFunc, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -384,9 +385,9 @@ FuncDef *sqlite3FindFunction( ** So we must not search for built-ins when creating a new function. */ if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); bestScore = 0; - p = functionSearch(pHash, h, zName, nName); + h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; + p = functionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -403,12 +404,19 @@ FuncDef *sqlite3FindFunction( */ if( createFlag && bestScorezName = (char *)&pBest[1]; + FuncDef *pOther; + pBest->zName = (const char*)&pBest[1]; pBest->nArg = (u16)nArg; pBest->funcFlags = enc; - memcpy(pBest->zName, zName, nName); - pBest->zName[nName] = 0; - sqlite3FuncDefInsert(&db->aFunc, pBest); + memcpy((char*)&pBest[1], zName, nName+1); + pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); + if( pOther==pBest ){ + sqlite3DbFree(db, pBest); + sqlite3OomFault(db); + return 0; + }else{ + pBest->pNext = pOther; + } } if( pBest && (pBest->xSFunc || createFlag) ){ diff --git a/src/complete.c b/src/complete.c index b120b7e8..bb2c0309 100644 --- a/src/complete.c +++ b/src/complete.c @@ -281,7 +281,7 @@ int sqlite3_complete16(const void *zSql){ if( zSql8 ){ rc = sqlite3_complete(zSql8); }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3ValueFree(pVal); return rc & 0xff; diff --git a/src/ctime.c b/src/ctime.c index f1bb69c1..09587ea8 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -45,6 +45,15 @@ static const char * const azCompileOpt[] = { #if SQLITE_CHECK_PAGES "CHECK_PAGES", #endif +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif #if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif @@ -64,7 +73,7 @@ static const char * const azCompileOpt[] = { "DISABLE_LFS", #endif #if SQLITE_ENABLE_8_3_NAMES - "ENABLE_8_3_NAMES", + "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), #endif #if SQLITE_ENABLE_API_ARMOR "ENABLE_API_ARMOR", diff --git a/src/date.c b/src/date.c index 0a1d0e07..8b2f2fa7 100644 --- a/src/date.c +++ b/src/date.c @@ -50,6 +50,15 @@ #ifndef SQLITE_OMIT_DATETIME_FUNCS +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So declare a substitute. The substitute function itself is +** defined in "os_win.c". +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +struct tm *__cdecl localtime(const time_t *); +#endif /* ** A structure for holding a single date and time. @@ -418,6 +427,7 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +#ifndef SQLITE_OMIT_LOCALTIME /* ** On recent Windows platforms, the localtime_s() function is available ** as part of the "Secure CRT". It is essentially equivalent to @@ -436,7 +446,6 @@ static void clearYMD_HMS_TZ(DateTime *p){ #define HAVE_LOCALTIME_S 1 #endif -#ifndef SQLITE_OMIT_LOCALTIME /* ** The following routine implements the rough equivalent of localtime_r() ** using whatever operating-system specific localtime facility that @@ -1103,7 +1112,6 @@ static void currentTimeFunc( ){ time_t t; char *zFormat = (char *)sqlite3_user_data(context); - sqlite3 *db; sqlite3_int64 iT; struct tm *pTm; struct tm sNow; @@ -1136,7 +1144,7 @@ static void currentTimeFunc( ** external linkage. */ void sqlite3RegisterDateTimeFunctions(void){ - static SQLITE_WSD FuncDef aDateTimeFuncs[] = { + static FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), DFUNCTION(date, -1, 0, 0, dateFunc ), @@ -1152,11 +1160,5 @@ void sqlite3RegisterDateTimeFunctions(void){ STR_FUNCTION(current_timestamp, 0, "%Y-%m-%d %H:%M:%S", 0, currentTimeFunc), #endif }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aDateTimeFuncs); - - for(i=0; ibase.pVtab = pVTab; @@ -349,7 +349,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){ nUsable = szPage - sqlite3BtreeGetReserveNoMutex(pBt); sqlite3BtreeLeave(pBt); p->aCell = sqlite3_malloc64((p->nCell+1) * sizeof(StatCell)); - if( p->aCell==0 ) return SQLITE_NOMEM; + if( p->aCell==0 ) return SQLITE_NOMEM_BKPT; memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell)); for(i=0; inCell; i++){ @@ -382,7 +382,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){ pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); pCell->nOvfl = nOvfl; pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl); - if( pCell->aOvfl==0 ) return SQLITE_NOMEM; + if( pCell->aOvfl==0 ) return SQLITE_NOMEM_BKPT; pCell->aOvfl[0] = sqlite3Get4byte(&aData[iOff+nLocal]); for(j=1; jaPage[0].iCell = 0; pCsr->aPage[0].zPath = z = sqlite3_mprintf("/"); pCsr->iPage = 0; - if( z==0 ) rc = SQLITE_NOMEM; + if( z==0 ) rc = SQLITE_NOMEM_BKPT; }else{ pCsr->isEof = 1; return sqlite3_reset(pCsr->pStmt); @@ -496,7 +496,7 @@ statNextRestart: } pCell->iOvfl++; statSizeAndOffset(pCsr); - return z==0 ? SQLITE_NOMEM : SQLITE_OK; + return z==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; } if( p->iRightChildPg ) break; p->iCell++; @@ -520,7 +520,7 @@ statNextRestart: p[1].iCell = 0; p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); p->iCell++; - if( z==0 ) rc = SQLITE_NOMEM; + if( z==0 ) rc = SQLITE_NOMEM_BKPT; } @@ -554,7 +554,7 @@ statNextRestart: pCsr->nUnused = p->nUnused; pCsr->nMxPayload = p->nMxPayload; pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath); - if( z==0 ) rc = SQLITE_NOMEM; + if( z==0 ) rc = SQLITE_NOMEM_BKPT; nPayload = 0; for(i=0; inCell; i++){ nPayload += p->aCell[i].nLocal; @@ -588,7 +588,7 @@ static int statFilter( if( pCsr->iDb<0 ){ sqlite3_free(pCursor->pVtab->zErrMsg); pCursor->pVtab->zErrMsg = sqlite3_mprintf("no such schema: %s", zDbase); - return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM_BKPT; } }else{ pCsr->iDb = pTab->iDb; @@ -602,9 +602,9 @@ static int statFilter( " UNION ALL " "SELECT name, rootpage, type" " FROM \"%w\".%s WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster); + " ORDER BY name", pTab->db->aDb[pCsr->iDb].zDbSName, zMaster); if( zSql==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; }else{ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); @@ -656,7 +656,7 @@ static int statColumn( default: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); int iDb = pCsr->iDb; - sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, db->aDb[iDb].zDbSName, -1, SQLITE_STATIC); break; } } diff --git a/src/delete.c b/src/delete.c index 0fe064bc..ec857182 100644 --- a/src/delete.c +++ b/src/delete.c @@ -102,7 +102,7 @@ void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } @@ -143,7 +143,7 @@ Expr *sqlite3LimitWhere( */ if( pOrderBy && (pLimit == 0) ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - goto limit_where_cleanup_2; + goto limit_where_cleanup; } /* We only need to generate a select expression if there @@ -165,16 +165,16 @@ Expr *sqlite3LimitWhere( */ pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup_2; + if( pSelectRowid == 0 ) goto limit_where_cleanup; pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup_2; + if( pEList == 0 ) goto limit_where_cleanup; /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); if( pSelectSrc == 0 ) { sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup_2; + goto limit_where_cleanup; } /* generate the SELECT expression tree. */ @@ -184,21 +184,11 @@ Expr *sqlite3LimitWhere( /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pWhereRowid == 0 ) goto limit_where_cleanup_1; - pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); - if( pInClause == 0 ) goto limit_where_cleanup_1; - - pInClause->x.pSelect = pSelect; - pInClause->flags |= EP_xIsSelect; - sqlite3ExprSetHeightAndFlags(pParse, pInClause); + pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0) : 0; + sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; - /* something went wrong. clean up anything allocated. */ -limit_where_cleanup_1: - sqlite3SelectDelete(pParse->db, pSelect); - return 0; - -limit_where_cleanup_2: +limit_where_cleanup: sqlite3ExprDelete(pParse->db, pWhere); sqlite3ExprListDelete(pParse->db, pOrderBy); sqlite3ExprDelete(pParse->db, pLimit); @@ -222,7 +212,6 @@ void sqlite3DeleteFrom( ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ - const char *zDb; /* Name of database holding pTab */ int i; /* Loop counter */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ @@ -249,11 +238,12 @@ void sqlite3DeleteFrom( int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ + int bComplex; /* True if there are triggers or FKs or + ** subqueries in the WHERE clause */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ - int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -281,7 +271,6 @@ void sqlite3DeleteFrom( #else # define pTrigger 0 # define isView 0 -# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -299,8 +288,8 @@ void sqlite3DeleteFrom( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - zDb = db->aDb[iDb].zName; - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + db->aDb[iDb].zDbSName); assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); if( rcauth==SQLITE_DENY ){ goto delete_from_cleanup; @@ -366,6 +355,9 @@ void sqlite3DeleteFrom( && pWhere==0 && !bComplex && !IsVirtual(pTab) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + && db->xPreUpdateCallback==0 +#endif ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -380,7 +372,8 @@ void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { - u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK|WHERE_SEEK_TABLE; + if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ @@ -480,7 +473,7 @@ void sqlite3DeleteFrom( if( !isView ){ int iAddrOnce = 0; if( eOnePass==ONEPASS_MULTI ){ - iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, @@ -715,14 +708,19 @@ void sqlite3GenerateRowDelete( /* Delete the index and table entries. Skip this step if pTab is really ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). */ + ** fire the INSTEAD OF triggers). + ** + ** If variable 'count' is non-zero, then this OP_Delete instruction should + ** invoke the update-hook. The pre-update-hook, on the other hand should + ** be invoked unless table pTab is a system table. The difference is that + ** the update-hook is not invoked for rows removed by REPLACE, but the + ** pre-update-hook is. + */ if( pTab->pSelect==0 ){ u8 p5 = 0; sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - if( count ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); - } + sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); if( eMode!=ONEPASS_OFF ){ sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } diff --git a/src/expr.c b/src/expr.c index 8d96ba10..75d0485d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -14,6 +14,18 @@ */ #include "sqliteInt.h" +/* Forward declarations */ +static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); +static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); + +/* +** Return the affinity character for a single column of a table. +*/ +char sqlite3TableColumnAffinity(Table *pTab, int iCol){ + assert( iColnCol ); + return iCol>=0 ? pTab->aCol[iCol].affinity : SQLITE_AFF_INTEGER; +} + /* ** Return the 'affinity' of the expression pExpr if any. ** @@ -39,21 +51,21 @@ char sqlite3ExprAffinity(Expr *pExpr){ assert( pExpr->flags&EP_xIsSelect ); return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); } + if( op==TK_REGISTER ) op = pExpr->op2; #ifndef SQLITE_OMIT_CAST if( op==TK_CAST ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif - if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) - && pExpr->pTab!=0 - ){ - /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = pExpr->iColumn; - if( j<0 ) return SQLITE_AFF_INTEGER; - assert( pExpr->pTab && jpTab->nCol ); - return pExpr->pTab->aCol[j].affinity; + if( op==TK_AGG_COLUMN || op==TK_COLUMN ){ + return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); + } + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft->flags&EP_xIsSelect ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); } return pExpr->affinity; } @@ -219,7 +231,7 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( !aff ){ + }else if( NEVER(aff==0) ){ aff = SQLITE_AFF_BLOB; } return aff; @@ -309,6 +321,270 @@ static int codeCompare( return addr; } +/* +** Return true if expression pExpr is a vector, or false otherwise. +** +** A vector is defined as any expression that results in two or more +** columns of result. Every TK_VECTOR node is an vector because the +** parser will not generate a TK_VECTOR with fewer than two entries. +** But a TK_SELECT might be either a vector or a scalar. It is only +** considered a vector if it has two or more result columns. +*/ +int sqlite3ExprIsVector(Expr *pExpr){ + return sqlite3ExprVectorSize(pExpr)>1; +} + +/* +** If the expression passed as the only argument is of type TK_VECTOR +** return the number of expressions in the vector. Or, if the expression +** is a sub-select, return the number of columns in the sub-select. For +** any other type of expression, return 1. +*/ +int sqlite3ExprVectorSize(Expr *pExpr){ + u8 op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( op==TK_VECTOR ){ + return pExpr->x.pList->nExpr; + }else if( op==TK_SELECT ){ + return pExpr->x.pSelect->pEList->nExpr; + }else{ + return 1; + } +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Return a pointer to a subexpression of pVector that is the i-th +** column of the vector (numbered starting with 0). The caller must +** ensure that i is within range. +** +** If pVector is really a scalar (and "scalar" here includes subqueries +** that return a single column!) then return pVector unmodified. +** +** pVector retains ownership of the returned subexpression. +** +** If the vector is a (SELECT ...) then the expression returned is +** just the expression for the i-th term of the result set, and may +** not be ready for evaluation because the table cursor has not yet +** been positioned. +*/ +Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ + assert( iop2==0 || pVector->op==TK_REGISTER ); + if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ + return pVector->x.pSelect->pEList->a[i].pExpr; + }else{ + return pVector->x.pList->a[i].pExpr; + } + } + return pVector; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Compute and return a new Expr object which when passed to +** sqlite3ExprCode() will generate all necessary code to compute +** the iField-th column of the vector expression pVector. +** +** It is ok for pVector to be a scalar (as long as iField==0). +** In that case, this routine works like sqlite3ExprDup(). +** +** The caller owns the returned Expr object and is responsible for +** ensuring that the returned value eventually gets freed. +** +** The caller retains ownership of pVector. If pVector is a TK_SELECT, +** then the returned object will reference pVector and so pVector must remain +** valid for the life of the returned object. If pVector is a TK_VECTOR +** or a scalar expression, then it can be deleted as soon as this routine +** returns. +** +** A trick to cause a TK_SELECT pVector to be deleted together with +** the returned Expr object is to attach the pVector to the pRight field +** of the returned TK_SELECT_COLUMN Expr object. +*/ +Expr *sqlite3ExprForVectorField( + Parse *pParse, /* Parsing context */ + Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ + int iField /* Which column of the vector to return */ +){ + Expr *pRet; + if( pVector->op==TK_SELECT ){ + assert( pVector->flags & EP_xIsSelect ); + /* The TK_SELECT_COLUMN Expr node: + ** + ** pLeft: pVector containing TK_SELECT + ** pRight: not used. But recursively deleted. + ** iColumn: Index of a column in pVector + ** pLeft->iTable: First in an array of register holding result, or 0 + ** if the result is not yet computed. + ** + ** sqlite3ExprDelete() specifically skips the recursive delete of + ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector + ** can be attached to pRight to cause this node to take ownership of + ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes + ** with the same pLeft pointer to the pVector, but only one of them + ** will own the pVector. + */ + pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0, 0); + if( pRet ){ + pRet->iColumn = iField; + pRet->pLeft = pVector; + } + assert( pRet==0 || pRet->iTable==0 ); + }else{ + if( pVector->op==TK_VECTOR ) pVector = pVector->x.pList->a[iField].pExpr; + pRet = sqlite3ExprDup(pParse->db, pVector, 0); + } + return pRet; +} +#endif /* !define(SQLITE_OMIT_SUBQUERY) */ + +/* +** If expression pExpr is of type TK_SELECT, generate code to evaluate +** it. Return the register in which the result is stored (or, if the +** sub-select returns more than one column, the first in an array +** of registers in which the result is stored). +** +** If pExpr is not a TK_SELECT expression, return 0. +*/ +static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ + int reg = 0; +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->op==TK_SELECT ){ + reg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + } +#endif + return reg; +} + +/* +** Argument pVector points to a vector expression - either a TK_VECTOR +** or TK_SELECT that returns more than one column. This function returns +** the register number of a register that contains the value of +** element iField of the vector. +** +** If pVector is a TK_SELECT expression, then code for it must have +** already been generated using the exprCodeSubselect() routine. In this +** case parameter regSelect should be the first in an array of registers +** containing the results of the sub-select. +** +** If pVector is of type TK_VECTOR, then code for the requested field +** is generated. In this case (*pRegFree) may be set to the number of +** a temporary register to be freed by the caller before returning. +** +** Before returning, output parameter (*ppExpr) is set to point to the +** Expr object corresponding to element iElem of the vector. +*/ +static int exprVectorRegister( + Parse *pParse, /* Parse context */ + Expr *pVector, /* Vector to extract element from */ + int iField, /* Field to extract from pVector */ + int regSelect, /* First in array of registers */ + Expr **ppExpr, /* OUT: Expression element */ + int *pRegFree /* OUT: Temp register to free */ +){ + u8 op = pVector->op; + assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT ); + if( op==TK_REGISTER ){ + *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); + return pVector->iTable+iField; + } + if( op==TK_SELECT ){ + *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; + return regSelect+iField; + } + *ppExpr = pVector->x.pList->a[iField].pExpr; + return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); +} + +/* +** Expression pExpr is a comparison between two vector values. Compute +** the result of the comparison (1, 0, or NULL) and write that +** result into register dest. +** +** The caller must satisfy the following preconditions: +** +** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ +** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ +** otherwise: op==pExpr->op and p5==0 +*/ +static void codeVectorCompare( + Parse *pParse, /* Code generator context */ + Expr *pExpr, /* The comparison operation */ + int dest, /* Write results into this register */ + u8 op, /* Comparison operator */ + u8 p5 /* SQLITE_NULLEQ or zero */ +){ + Vdbe *v = pParse->pVdbe; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + int nLeft = sqlite3ExprVectorSize(pLeft); + int i; + int regLeft = 0; + int regRight = 0; + u8 opx = op; + int addrDone = sqlite3VdbeMakeLabel(v); + + assert( nLeft==sqlite3ExprVectorSize(pRight) ); + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE + ); + assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) + || (pExpr->op==TK_ISNOT && op==TK_NE) ); + assert( p5==0 || pExpr->op!=op ); + assert( p5==SQLITE_NULLEQ || pExpr->op==op ); + + p5 |= SQLITE_STOREP2; + if( opx==TK_LE ) opx = TK_LT; + if( opx==TK_GE ) opx = TK_GT; + + regLeft = exprCodeSubselect(pParse, pLeft); + regRight = exprCodeSubselect(pParse, pRight); + + for(i=0; 1 /*Loop exits by "break"*/; i++){ + int regFree1 = 0, regFree2 = 0; + Expr *pL, *pR; + int r1, r2; + assert( i>=0 && i0 ) sqlite3ExprCachePush(pParse); + r1 = exprVectorRegister(pParse, pLeft, i, regLeft, &pL, ®Free1); + r2 = exprVectorRegister(pParse, pRight, i, regRight, &pR, ®Free2); + codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5); + testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + if( i>0 ) sqlite3ExprCachePop(pParse); + if( i==nLeft-1 ){ + break; + } + if( opx==TK_EQ ){ + sqlite3VdbeAddOp2(v, OP_IfNot, dest, addrDone); VdbeCoverage(v); + p5 |= SQLITE_KEEPNULL; + }else if( opx==TK_NE ){ + sqlite3VdbeAddOp2(v, OP_If, dest, addrDone); VdbeCoverage(v); + p5 |= SQLITE_KEEPNULL; + }else{ + assert( op==TK_LT || op==TK_GT || op==TK_LE || op==TK_GE ); + sqlite3VdbeAddOp2(v, OP_ElseNotEq, 0, addrDone); + VdbeCoverageIf(v, op==TK_LT); + VdbeCoverageIf(v, op==TK_GT); + VdbeCoverageIf(v, op==TK_LE); + VdbeCoverageIf(v, op==TK_GE); + if( i==nLeft-2 ) opx = op; + } + } + sqlite3VdbeResolveLabel(v, addrDone); +} + #if SQLITE_MAX_EXPR_DEPTH>0 /* ** Check that argument nHeight is less than or equal to the maximum @@ -444,7 +720,7 @@ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is allocated to hold the integer text and the dequote flag is ignored. */ Expr *sqlite3ExprAlloc( - sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ int op, /* Expression opcode */ const Token *pToken, /* Token argument. Might be NULL */ int dequote /* True to dequote */ @@ -471,15 +747,13 @@ Expr *sqlite3ExprAlloc( pNew->flags |= EP_IntValue; pNew->u.iValue = iValue; }else{ - int c; pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; - if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + if( pNew->u.zToken[0]=='"' ) pNew->flags |= EP_DblQuoted; sqlite3Dequote(pNew->u.zToken); - if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } @@ -562,6 +836,22 @@ Expr *sqlite3PExpr( return p; } +/* +** Add pSelect to the Expr.x.pSelect field. Or, if pExpr is NULL (due +** do a memory allocation failure) then delete the pSelect object. +*/ +void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ + if( pExpr ){ + pExpr->x.pSelect = pSelect; + ExprSetProperty(pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, pExpr); + }else{ + assert( pParse->db->mallocFailed ); + sqlite3SelectDelete(pParse->db, pSelect); + } +} + + /* ** If the expression is always either TRUE or FALSE (respectively), ** then return 1. If one cannot determine the truth value of the @@ -648,7 +938,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ ** instance of the wildcard, the next sequential variable number is ** assigned. */ -void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ +void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ sqlite3 *db = pParse->db; const char *z; @@ -657,19 +947,19 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ z = pExpr->u.zToken; assert( z!=0 ); assert( z[0]!=0 ); + assert( n==sqlite3Strlen30(z) ); if( z[1]==0 ){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); }else{ - ynVar x = 0; - u32 n = sqlite3Strlen30(z); + ynVar x; if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); - pExpr->iColumn = x = (ynVar)i; + x = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -677,7 +967,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - x = 0; + return; } if( i>pParse->nVar ){ pParse->nVar = (int)i; @@ -688,33 +978,31 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ ** has never appeared before, reuse the same variable number */ ynVar i; - for(i=0; inzVar; i++){ + for(i=x=0; inzVar; i++){ if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ - pExpr->iColumn = x = (ynVar)i+1; + x = (ynVar)i+1; break; } } - if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); + if( x==0 ) x = (ynVar)(++pParse->nVar); } - if( x>0 ){ - if( x>pParse->nzVar ){ - char **a; - a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ){ - assert( db->mallocFailed ); /* Error reported through mallocFailed */ - return; - } - pParse->azVar = a; - memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); - pParse->nzVar = x; - } - if( z[0]!='?' || pParse->azVar[x-1]==0 ){ - sqlite3DbFree(db, pParse->azVar[x-1]); - pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); + pExpr->iColumn = x; + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ){ + assert( db->mallocFailed ); /* Error reported through mallocFailed */ + return; } + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; + } + if( pParse->azVar[x-1]==0 ){ + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } - if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "too many SQL variables"); } } @@ -722,26 +1010,36 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* ** Recursively delete an expression tree. */ -void sqlite3ExprDelete(sqlite3 *db, Expr *p){ - if( p==0 ) return; +static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ + assert( p!=0 ); /* Sanity check: Assert that the IntValue is non-negative if it exists */ assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); - if( !ExprHasProperty(p, EP_TokenOnly) ){ +#ifdef SQLITE_DEBUG + if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ + assert( p->pLeft==0 ); + assert( p->pRight==0 ); + assert( p->x.pSelect==0 ); + } +#endif + if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( p->x.pList==0 || p->pRight==0 ); - sqlite3ExprDelete(db, p->pLeft); + if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); sqlite3ExprDelete(db, p->pRight); - if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( ExprHasProperty(p, EP_xIsSelect) ){ sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); } } + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbFree(db, p); } } +void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p ) sqlite3ExprDeleteNN(db, p); +} /* ** Return the number of bytes allocated for the expression structure @@ -793,7 +1091,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==(flags&EXPRDUP_REDUCE) ){ + if( 0==flags ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -855,88 +1153,92 @@ static int dupedExprSize(Expr *p, int flags){ ** if any. Before returning, *pzBuffer is set to the first byte past the ** portion of the buffer copied into by this function. */ -static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ - Expr *pNew = 0; /* Value to return */ - assert( flags==0 || flags==EXPRDUP_REDUCE ); +static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ + Expr *pNew; /* Value to return */ + u8 *zAlloc; /* Memory space from which to build Expr object */ + u32 staticFlag; /* EP_Static if space not obtained from malloc */ + assert( db!=0 ); - if( p ){ - const int isReduced = (flags&EXPRDUP_REDUCE); - u8 *zAlloc; - u32 staticFlag = 0; + assert( p ); + assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || isReduced ); + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + }else{ + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + staticFlag = 0; + } + pNew = (Expr *)zAlloc; - /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; - staticFlag = EP_Static; + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, dupFlags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, flags)); + nToken = 0; + } + if( dupFlags ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + u32 nSize = (u32)exprStructSize(p); + memcpy(zAlloc, p, nSize); + if( nSizeu.zToken string (if any). - */ - const unsigned nStructSize = dupedExprStructSize(p, flags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; + /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; + + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); + } + + if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - nToken = 0; + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); } - if( isReduced ){ - assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); - }else{ - u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); - if( nSizepLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + zAlloc += dupedExprNodeSize(p, dupFlags); + if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; } - - /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); - pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); - pNew->flags |= staticFlag; - - /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; - memcpy(zToken, p->u.zToken, nToken); + if( pzBuffer ){ + *pzBuffer = zAlloc; } - - if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ - /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ - if( ExprHasProperty(p, EP_xIsSelect) ){ - pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); + }else{ + if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + if( pNew->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); - } - } - - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, flags); - if( ExprHasProperty(pNew, EP_Reduced) ){ - pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); - pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); - } - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly) ){ pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); - pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } - } } return pNew; @@ -988,7 +1290,7 @@ static With *withDup(sqlite3 *db, With *p){ */ Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ assert( flags==0 || flags==EXPRDUP_REDUCE ); - return exprDup(db, p, flags, 0); + return p ? exprDup(db, p, flags, 0) : 0; } ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ ExprList *pNew; @@ -1175,6 +1477,60 @@ no_mem: return 0; } +/* +** pColumns and pExpr form a vector assignment which is part of the SET +** clause of an UPDATE statement. Like this: +** +** (a,b,c) = (expr1,expr2,expr3) +** Or: (a,b,c) = (SELECT x,y,z FROM ....) +** +** For each term of the vector assignment, append new entries to the +** expression list pList. In the case of a subquery on the LHS, append +** TK_SELECT_COLUMN expressions. +*/ +ExprList *sqlite3ExprListAppendVector( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + IdList *pColumns, /* List of names of LHS of the assignment */ + Expr *pExpr /* Vector expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + int n; + int i; + int iFirst = pList ? pList->nExpr : 0; + /* pColumns can only be NULL due to an OOM but an OOM will cause an + ** exit prior to this routine being invoked */ + if( NEVER(pColumns==0) ) goto vector_append_error; + if( pExpr==0 ) goto vector_append_error; + n = sqlite3ExprVectorSize(pExpr); + if( pColumns->nId!=n ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pColumns->nId, n); + goto vector_append_error; + } + for(i=0; inExpr==iFirst+i+1 ); + pList->a[pList->nExpr-1].zName = pColumns->a[i].zName; + pColumns->a[i].zName = 0; + } + } + if( pExpr->op==TK_SELECT ){ + if( pList && pList->a[iFirst].pExpr ){ + assert( pList->a[iFirst].pExpr->op==TK_SELECT_COLUMN ); + pList->a[iFirst].pExpr->pRight = pExpr; + pExpr = 0; + } + } + +vector_append_error: + sqlite3ExprDelete(db, pExpr); + sqlite3IdListDelete(db, pColumns); + return pList; +} + /* ** Set the sort order for the last element on the given ExprList. */ @@ -1210,7 +1566,7 @@ void sqlite3ExprListSetName( pItem = &pList->a[pList->nExpr-1]; assert( pItem->zName==0 ); pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); + if( dequote ) sqlite3Dequote(pItem->zName); } } @@ -1259,10 +1615,9 @@ void sqlite3ExprListCheckLength( /* ** Delete an entire expression list. */ -void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ +static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i; struct ExprList_item *pItem; - if( pList==0 ) return; assert( pList->a!=0 || pList->nExpr==0 ); for(pItem=pList->a, i=0; inExpr; i++, pItem++){ sqlite3ExprDelete(db, pItem->pExpr); @@ -1272,6 +1627,9 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ sqlite3DbFree(db, pList->a); sqlite3DbFree(db, pList); } +void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ + if( pList ) exprListDeleteNN(db, pList); +} /* ** Return the bitwise-OR of all Expr.flags fields in the given @@ -1283,7 +1641,8 @@ u32 sqlite3ExprListFlags(const ExprList *pList){ if( pList ){ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( ALWAYS(pExpr) ) m |= pExpr->flags; + assert( pExpr!=0 ); + m |= pExpr->flags; } } return m; @@ -1568,23 +1927,22 @@ int sqlite3IsRowid(const char *z){ } /* -** Return true if we are able to the IN operator optimization on a -** query of the form -** -** x IN (SELECT ...) -** -** Where the SELECT... clause is as specified by the parameter to this -** routine. -** -** The Select object passed in has already been preprocessed and no -** errors have been found. +** pX is the RHS of an IN operator. If pX is a SELECT statement +** that can be simplified to a direct table access, then return +** a pointer to the SELECT statement. If pX is not a SELECT statement, +** or if the SELECT statement needs to be manifested into a transient +** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY -static int isCandidateForInOpt(Select *p){ +static Select *isCandidateForInOpt(Expr *pX){ + Select *p; SrcList *pSrc; ExprList *pEList; Table *pTab; - if( p==0 ) return 0; /* right-hand side of IN is SELECT */ + int i; + if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ + if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ + p = pX->x.pSelect; if( p->pPrior ) return 0; /* Not a compound SELECT */ if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); @@ -1600,25 +1958,22 @@ static int isCandidateForInOpt(Select *p){ if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ pTab = pSrc->a[0].pTab; - if( NEVER(pTab==0) ) return 0; + assert( pTab!=0 ); assert( pTab->pSelect==0 ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ pEList = p->pEList; - if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ - if( pEList->a[0].pExpr->op!=TK_COLUMN ) return 0; /* Result is a column */ - return 1; + assert( pEList!=0 ); + /* All SELECT results must be columns. */ + for(i=0; inExpr; i++){ + Expr *pRes = pEList->a[i].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + } + return p; } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Code an OP_Once instruction and allocate space for its flag. Return the -** address of the new instruction. -*/ -int sqlite3CodeOnce(Parse *pParse){ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); -} - +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that checks the left-most column of index table iCur to see if ** it contains any NULL entries. Cause the register at regHasNull to be set @@ -1634,6 +1989,7 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ VdbeComment((v, "first_entry_in(%d)", iCur)); sqlite3VdbeJumpHere(v, addr1); } +#endif #ifndef SQLITE_OMIT_SUBQUERY @@ -1678,7 +2034,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: ** -** SELECT FROM +** SELECT , ... FROM
    ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then @@ -1694,14 +2050,14 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table must be used unless the selected is guaranteed -** to be unique - either because it is an INTEGER PRIMARY KEY or it -** has a UNIQUE constraint or UNIQUE index. +** An epheremal table must be used unless the selected columns are guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or due to +** a UNIQUE constraint or index. ** ** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used ** for fast set membership tests) then an epheremal table must -** be used unless is an INTEGER PRIMARY KEY or an index can -** be found with as its left-most column. +** be used unless is a single INTEGER PRIMARY KEY column or an +** index can be found with the specified as its left-most. ** ** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and ** if the RHS of the IN operator is a list (not a subquery) then this @@ -1722,9 +2078,26 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** the value in that register will be NULL if the b-tree contains one or more ** NULL values, and it will be some non-NULL value if the b-tree contains no ** NULL values. +** +** If the aiMap parameter is not NULL, it must point to an array containing +** one element for each column returned by the SELECT statement on the RHS +** of the IN(...) operator. The i'th entry of the array is populated with the +** offset of the index column that matches the i'th column returned by the +** SELECT. For example, if the expression and selected index are: +** +** (?,?,?) IN (SELECT a, b, c FROM t1) +** CREATE INDEX i1 ON t1(b, c, a); +** +** then aiMap[] is populated with {2, 0, 1}. */ #ifndef SQLITE_OMIT_SUBQUERY -int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ +int sqlite3FindInIndex( + Parse *pParse, /* Parsing context */ + Expr *pX, /* The right-hand side (RHS) of the IN operator */ + u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ + int *prRhsHasNull, /* Register holding NULL status. See notes */ + int *aiMap /* Mapping from Index fields to RHS fields */ +){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ @@ -1734,38 +2107,46 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ assert( pX->op==TK_IN ); mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + /* If the RHS of this IN(...) operator is a SELECT, and if it matters + ** whether or not the SELECT result contains NULL values, check whether + ** or not NULL is actually possible (it may not be, for example, due + ** to NOT NULL constraints in the schema). If no NULL values are possible, + ** set prRhsHasNull to 0 before continuing. */ + if( prRhsHasNull && (pX->flags & EP_xIsSelect) ){ + int i; + ExprList *pEList = pX->x.pSelect->pEList; + for(i=0; inExpr; i++){ + if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; + } + if( i==pEList->nExpr ){ + prRhsHasNull = 0; + } + } + /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new - ** ephemeral table. - */ - p = (ExprHasProperty(pX, EP_xIsSelect) ? pX->x.pSelect : 0); - if( pParse->nErr==0 && isCandidateForInOpt(p) ){ + ** ephemeral table. */ + if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table
    . */ - Expr *pExpr; /* Expression */ - i16 iCol; /* Index of column */ i16 iDb; /* Database idx for pTab */ + ExprList *pEList = p->pEList; + int nExpr = pEList->nExpr; - assert( p ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ pTab = p->pSrc->a[0].pTab; - pExpr = p->pEList->a[0].pExpr; - iCol = (i16)pExpr->iColumn; - + /* Code an OP_Transaction and OP_TableLock for
    . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* This function is only called from two places. In both cases the vdbe - ** has already been allocated. So assume sqlite3GetVdbe() is always - ** successful here. - */ - assert(v); - if( iCol<0 ){ - int iAddr = sqlite3CodeOnce(pParse); + assert(v); /* sqlite3GetVdbe() has always been previously called */ + if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ + /* The "x IN (SELECT rowid FROM table)" case */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); @@ -1774,44 +2155,114 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ + int affinity_ok = 1; + int i; - /* The collation sequence used by the comparison. If an index is to - ** be used in place of a temp-table, it must be ordered according - ** to this collation sequence. */ - CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); - - /* Check that the affinity that will be used to perform the - ** comparison is the same as the affinity of the column. If - ** it is not, it is not possible to use any index. - */ - int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); - - for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ - if( (pIdx->aiColumn[0]==iCol) - && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) - ){ - int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); - eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - - if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ - *prRhsHasNull = ++pParse->nMem; - sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); - } - sqlite3VdbeJumpHere(v, iAddr); + /* Check that the affinity that will be used to perform each + ** comparison is the same as the affinity of each column in table + ** on the RHS of the IN operator. If it not, it is not possible to + ** use any index of the RHS table. */ + for(i=0; ipLeft, i); + int iCol = pEList->a[i].pExpr->iColumn; + char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ + char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); + testcase( cmpaff==SQLITE_AFF_BLOB ); + testcase( cmpaff==SQLITE_AFF_TEXT ); + switch( cmpaff ){ + case SQLITE_AFF_BLOB: + break; + case SQLITE_AFF_TEXT: + /* sqlite3CompareAffinity() only returns TEXT if one side or the + ** other has no affinity and the other side is TEXT. Hence, + ** the only way for cmpaff to be TEXT is for idxaff to be TEXT + ** and for the term on the LHS of the IN to have no affinity. */ + assert( idxaff==SQLITE_AFF_TEXT ); + break; + default: + affinity_ok = sqlite3IsNumericAffinity(idxaff); } } - } - } + + if( affinity_ok ){ + /* Search for an existing index that will work for this IN operator */ + for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ + Bitmask colUsed; /* Columns of the index used */ + Bitmask mCol; /* Mask for the current column */ + if( pIdx->nColumnnColumn==BMS-2 ); + testcase( pIdx->nColumn==BMS-1 ); + if( pIdx->nColumn>=BMS-1 ) continue; + if( mustBeUnique ){ + if( pIdx->nKeyCol>nExpr + ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) + ){ + continue; /* This index is not unique over the IN RHS columns */ + } + } + + colUsed = 0; /* Columns of index used so far */ + for(i=0; ipLeft, i); + Expr *pRhs = pEList->a[i].pExpr; + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + int j; + + assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; + assert( pIdx->azColl[j] ); + if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ + continue; + } + break; + } + if( j==nExpr ) break; + mCol = MASKBIT(j); + if( mCol & colUsed ) break; /* Each column used only once */ + colUsed |= mCol; + if( aiMap ) aiMap[i] = j; + } + + assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); + if( colUsed==(MASKBIT(nExpr)-1) ){ + /* If we reach this point, that means the index pIdx is usable */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); +#ifndef SQLITE_OMIT_EXPLAIN + sqlite3VdbeAddOp4(v, OP_Explain, 0, 0, 0, + sqlite3MPrintf(db, "USING INDEX %s FOR IN-OPERATOR",pIdx->zName), + P4_DYNAMIC); +#endif + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull ){ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + i64 mask = (1<nMem; + if( nExpr==1 ){ + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + } + sqlite3VdbeJumpHere(v, iAddr); + } + } /* End loop over indexes */ + } /* End if( affinity_ok ) */ + } /* End if not an rowid index */ + } /* End attempt to optimize using an index */ /* If no preexisting index is available for the IN clause ** and IN_INDEX_NOOP is an allowed reply ** and the RHS of the IN operator is a list, not a subquery - ** and the RHS is not contant or has two or fewer terms, + ** and the RHS is not constant or has two or fewer terms, ** then it is not worth creating an ephemeral table to evaluate ** the IN operator so return IN_INDEX_NOOP. */ @@ -1822,7 +2273,6 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ ){ eType = IN_INDEX_NOOP; } - if( eType==0 ){ /* Could not find an existing table or index to use as the RHS b-tree. @@ -1844,10 +2294,63 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ }else{ pX->iTable = iTab; } + + if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ + int i, n; + n = sqlite3ExprVectorSize(pX->pLeft); + for(i=0; ipLeft; + int nVal = sqlite3ExprVectorSize(pLeft); + Select *pSelect = (pExpr->flags & EP_xIsSelect) ? pExpr->x.pSelect : 0; + char *zRet; + + assert( pExpr->op==TK_IN ); + zRet = sqlite3DbMallocZero(pParse->db, nVal+1); + if( zRet ){ + int i; + for(i=0; ipEList->a[i].pExpr, a); + }else{ + zRet[i] = a; + } + } + zRet[nVal] = '\0'; + } + return zRet; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Load the Parse object passed as the first argument with an error +** message of the form: +** +** "sub-select returns N columns - expected M" +*/ +void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); +} +#endif + /* ** Generate code for scalar subqueries used as a subquery expression, EXISTS, ** or IN operators. Examples: @@ -1873,7 +2376,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ ** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the -** result. For IN operators or if an error occurs, the return value is 0. +** result. For a multi-column SELECT, the result is stored in a contiguous +** array of registers and the return value is the register of the left-most +** result column. Return 0 for IN operators or if an error occurs. */ #ifndef SQLITE_OMIT_SUBQUERY int sqlite3CodeSubselect( @@ -1888,8 +2393,8 @@ int sqlite3CodeSubselect( if( NEVER(v==0) ) return 0; sqlite3ExprCachePush(pParse); - /* This code must be run in its entirety every time it is encountered - ** if any of the following is true: + /* The evaluation of the IN/EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: ** ** * The right-hand side is a correlated subquery ** * The right-hand side is an expression list containing variables @@ -1899,7 +2404,7 @@ int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); + jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN @@ -1915,17 +2420,18 @@ int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { - char affinity; /* Affinity of the LHS of the IN */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ KeyInfo *pKeyInfo = 0; /* Key information */ - - affinity = sqlite3ExprAffinity(pLeft); + int nVal; /* Size of vector pLeft */ + + nVal = sqlite3ExprVectorSize(pLeft); + assert( !isRowid || nVal==1 ); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' ** expression it is handled the same way. An ephemeral table is - ** filled with single-field index keys representing the results - ** from the SELECT or the . + ** filled with index keys representing the results from the + ** SELECT or the . ** ** If the 'x' expression is a column value, or the SELECT... ** statement returns a column value, then the affinity of that @@ -1936,8 +2442,9 @@ int sqlite3CodeSubselect( ** is used. */ pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, + pExpr->iTable, (isRowid?0:nVal)); + pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, nVal, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -1946,27 +2453,37 @@ int sqlite3CodeSubselect( ** table allocated and opened above. */ Select *pSelect = pExpr->x.pSelect; - SelectDest dest; - ExprList *pEList; + ExprList *pEList = pSelect->pEList; assert( !isRowid ); - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affSdst = (u8)affinity; - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pSelect->iLimit = 0; - testcase( pSelect->selFlags & SF_Distinct ); - testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pSelect, &dest) ){ - sqlite3KeyInfoUnref(pKeyInfo); - return 0; + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + SelectDest dest; + int i; + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3DbFree(pParse->db, dest.zAffSdst); + sqlite3KeyInfoUnref(pKeyInfo); + return 0; + } + sqlite3DbFree(pParse->db, dest.zAffSdst); + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); + } } - pEList = pSelect->pEList; - assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ - assert( pEList!=0 ); - assert( pEList->nExpr>0 ); - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, - pEList->a[0].pExpr); }else if( ALWAYS(pExpr->x.pList!=0) ){ /* Case 2: expr IN (exprlist) ** @@ -1975,11 +2492,13 @@ int sqlite3CodeSubselect( ** that columns affinity when building index keys. If is not ** a column, use numeric affinity. */ + char affinity; /* Affinity of the LHS of the IN */ int i; ExprList *pList = pExpr->x.pList; struct ExprList_item *pItem; int r1, r2, r3; + affinity = sqlite3ExprAffinity(pLeft); if( !affinity ){ affinity = SQLITE_AFF_BLOB; } @@ -2035,26 +2554,37 @@ int sqlite3CodeSubselect( case TK_EXISTS: case TK_SELECT: default: { - /* If this has to be a scalar SELECT. Generate code to put the - ** value of this select in a memory cell and record the number - ** of the memory cell in iColumn. If this is an EXISTS, write - ** an integer 0 (not exists) or 1 (exists) into a memory cell - ** and record that memory cell in iColumn. + /* Case 3: (SELECT ... FROM ...) + ** or: EXISTS(SELECT ... FROM ...) + ** + ** For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. */ Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECt result */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ testcase( pExpr->op==TK_EXISTS ); testcase( pExpr->op==TK_SELECT ); assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; - sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; dest.iSdst = dest.iSDParm; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; @@ -2062,8 +2592,8 @@ int sqlite3CodeSubselect( VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, - &sqlite3IntTokens[1]); + pSel->pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER, + &sqlite3IntTokens[1], 0); pSel->iLimit = 0; pSel->selFlags &= ~SF_MultiValue; if( sqlite3Select(pParse, pSel, &dest) ){ @@ -2088,6 +2618,32 @@ int sqlite3CodeSubselect( } #endif /* SQLITE_OMIT_SUBQUERY */ +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** a sub-query, that the LHS is a vector of size 1. +*/ +int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ + int nVector = sqlite3ExprVectorSize(pIn->pLeft); + if( (pIn->flags & EP_xIsSelect) ){ + if( nVector!=pIn->x.pSelect->pEList->nExpr ){ + sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); + return 1; + } + }else if( nVector!=1 ){ + if( (pIn->pLeft->flags & EP_xIsSelect) ){ + sqlite3SubselectError(pParse, nVector, 1); + }else{ + sqlite3ErrorMsg(pParse, "row value misused"); + } + return 1; + } + return 0; +} +#endif + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code for an IN expression. @@ -2095,16 +2651,24 @@ int sqlite3CodeSubselect( ** x IN (SELECT ...) ** x IN (value, value, ...) ** -** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) -** is an array of zero or more values. The expression is true if the LHS is -** contained within the RHS. The value of the expression is unknown (NULL) -** if the LHS is NULL or if the LHS is not contained within the RHS and the -** RHS contains one or more NULL values. +** The left-hand side (LHS) is a scalar or vector expression. The +** right-hand side (RHS) is an array of zero or more scalar values, or a +** subquery. If the RHS is a subquery, the number of result columns must +** match the number of columns in the vector on the LHS. If the RHS is +** a list of values, the LHS must be a scalar. +** +** The IN operator is true if the LHS value is contained within the RHS. +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be +** determined due to NULLs. ** ** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. +** +** See the separate in-operator.md documentation file in the canonical +** SQLite source tree for additional information. */ static void sqlite3ExprCodeIN( Parse *pParse, /* Parsing and code generating context */ @@ -2113,36 +2677,83 @@ static void sqlite3ExprCodeIN( int destIfNull /* Jump here if the results are unknown due to NULLs */ ){ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ - char affinity; /* Comparison affinity to use */ int eType; /* Type of the RHS */ - int r1; /* Temporary use register */ + int rLhs; /* Register(s) holding the LHS values */ + int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ + int *aiMap = 0; /* Map from vector field to index column */ + char *zAff = 0; /* Affinity string for comparisons */ + int nVector; /* Size of vectors for this IN operator */ + int iDummy; /* Dummy parameter to exprCodeVector() */ + Expr *pLeft; /* The LHS of the IN operator */ + int i; /* loop counter */ + int destStep2; /* Where to jump when NULLs seen in step 2 */ + int destStep6 = 0; /* Start of code for Step 6 */ + int addrTruthOp; /* Address of opcode that determines the IN is true */ + int destNotNull; /* Jump here if a comparison is not true in step 6 */ + int addrTop; /* Top of the step-6 loop */ - /* Compute the RHS. After this step, the table with cursor - ** pExpr->iTable will contains the values that make up the RHS. - */ + pLeft = pExpr->pLeft; + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + zAff = exprINAffinity(pParse, pExpr); + nVector = sqlite3ExprVectorSize(pExpr->pLeft); + aiMap = (int*)sqlite3DbMallocZero( + pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 + ); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + + /* Attempt to compute the RHS. After this step, if anything other than + ** IN_INDEX_NOOP is returned, the table opened ith cursor pExpr->iTable + ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, + ** the RHS has not yet been coded. */ v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, - destIfFalse==destIfNull ? 0 : &rRhsHasNull); + destIfFalse==destIfNull ? 0 : &rRhsHasNull, aiMap); - /* Figure out the affinity to use to create a key from the results - ** of the expression. affinityStr stores a static string suitable for - ** P4 of OP_MakeRecord. - */ - affinity = comparisonAffinity(pExpr); + assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + ); +#ifdef SQLITE_DEBUG + /* Confirm that aiMap[] contains nVector integer values between 0 and + ** nVector-1. */ + for(i=0; i from " IN (...)". + /* Code the LHS, the from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting + ** at r1. + ** + ** sqlite3FindInIndex() might have reordered the fields of the LHS vector + ** so that the fields are in the same order as an existing index. The + ** aiMap[] array contains a mapping from the original LHS field order to + ** the field order that matches the RHS index. */ sqlite3ExprCachePush(pParse); - r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCode(pParse, pExpr->pLeft, r1); + rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + for(i=0; ix.pList; @@ -2154,7 +2765,7 @@ static void sqlite3ExprCodeIN( assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); } for(ii=0; iinExpr; ii++){ r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); @@ -2162,16 +2773,16 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } if( iinExpr-1 || destIfNull!=destIfFalse ){ - sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + sqlite3VdbeAddOp4(v, OP_Eq, rLhs, labelOk, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverageIf(v, iinExpr-1); VdbeCoverageIf(v, ii==pList->nExpr-1); - sqlite3VdbeChangeP5(v, affinity); + sqlite3VdbeChangeP5(v, zAff[0]); }else{ assert( destIfNull==destIfFalse ); - sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + sqlite3VdbeAddOp4(v, OP_Ne, rLhs, destIfFalse, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); } sqlite3ReleaseTempReg(pParse, regToFree); } @@ -2181,78 +2792,113 @@ static void sqlite3ExprCodeIN( } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); + goto sqlite3ExprCodeIN_finished; + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the + ** LHS does contain NULLs then the result must be either FALSE or NULL. + ** We will then skip the binary search of the RHS. + */ + if( destIfNull==destIfFalse ){ + destStep2 = destIfFalse; }else{ - - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. - */ - if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree - */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + destStep2 = destStep6 = sqlite3VdbeMakeLabel(v); + } + for(i=0; ipLeft, i); + if( sqlite3ExprCanBeNull(p) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); VdbeCoverage(v); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); - if( rRhsHasNull==0 ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. - */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - VdbeCoverage(v); - }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. - */ - int addr1; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the answer is TRUE the presence of NULLs in the RHS does - ** not matter. If the LHS is not contained in the RHS, then the - ** answer is NULL if the RHS contains NULLs and the answer is - ** FALSE if the RHS is NULL-free. - */ - addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); - VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfFalse); - sqlite3VdbeJumpHere(v, addr1); - } } } - sqlite3ReleaseTempReg(pParse, r1); + + /* Step 3. The LHS is now known to be non-NULL. Do the binary search + ** of the RHS using the LHS as a probe. If found, the result is + ** true. + */ + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree and so we also + ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 + ** into a single opcode. */ + sqlite3VdbeAddOp3(v, OP_SeekRowid, pExpr->iTable, destIfFalse, rLhs); + VdbeCoverage(v); + addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ + }else{ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + if( destIfFalse==destIfNull ){ + /* Combine Step 3 and Step 5 into a single opcode */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + goto sqlite3ExprCodeIN_finished; + } + /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, + rLhs, nVector); VdbeCoverage(v); + } + + /* Step 4. If the RHS is known to be non-NULL and we did not find + ** an match on the search above, then the result must be FALSE. + */ + if( rRhsHasNull && nVector==1 ){ + sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); + VdbeCoverage(v); + } + + /* Step 5. If we do not care about the difference between NULL and + ** FALSE, then just return false. + */ + if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); + + /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. + ** If any comparison is NULL, then the result is NULL. If all + ** comparisons are FALSE then the final result is FALSE. + ** + ** For a scalar LHS, it is sufficient to check just the first row + ** of the RHS. + */ + if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + if( nVector>1 ){ + destNotNull = sqlite3VdbeMakeLabel(v); + }else{ + /* For nVector==1, combine steps 6 and 7 by immediately returning + ** FALSE if the first comparison is not NULL */ + destNotNull = destIfFalse; + } + for(i=0; iiTable, i, r3); + sqlite3VdbeAddOp4(v, OP_Ne, rLhs+i, destNotNull, r3, + (void*)pColl, P4_COLLSEQ); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r3); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + if( nVector>1 ){ + sqlite3VdbeResolveLabel(v, destNotNull); + sqlite3VdbeAddOp2(v, OP_Next, pExpr->iTable, addrTop+1); + VdbeCoverage(v); + + /* Step 7: If we reach this point, we know that the result must + ** be false. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + + /* Jumps here in order to return true. */ + sqlite3VdbeJumpHere(v, addrTruthOp); + +sqlite3ExprCodeIN_finished: + if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); +sqlite3ExprCodeIN_oom_error: + sqlite3DbFree(pParse->db, aiMap); + sqlite3DbFree(pParse->db, zAff); } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -2317,14 +2963,17 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ } /* -** Clear a cache entry. +** Erase column-cache entry number i */ -static void cacheEntryClear(Parse *pParse, struct yColCache *p){ - if( p->tempReg ){ +static void cacheEntryClear(Parse *pParse, int i){ + if( pParse->aColCache[i].tempReg ){ if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = p->iReg; + pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; } - p->tempReg = 0; + } + pParse->nColCache--; + if( inColCache ){ + pParse->aColCache[i] = pParse->aColCache[pParse->nColCache]; } } @@ -2355,43 +3004,33 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** that the object will never already be in cache. Verify this guarantee. */ #ifndef NDEBUG - for(i=0, p=pParse->aColCache; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + assert( p->iTable!=iTab || p->iColumn!=iCol ); } #endif - /* Find an empty slot and replace it */ - for(i=0, p=pParse->aColCache; iiReg==0 ){ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; + /* If the cache is already full, delete the least recently used entry */ + if( pParse->nColCache>=SQLITE_N_COLCACHE ){ + minLru = 0x7fffffff; + idxLru = -1; + for(i=0, p=pParse->aColCache; ilrulru; + } } + p = &pParse->aColCache[idxLru]; + }else{ + p = &pParse->aColCache[pParse->nColCache++]; } - /* Replace the last recently used */ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - if( ALWAYS(idxLru>=0) ){ - p = &pParse->aColCache[idxLru]; - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; - } + /* Add the new entry to the end of the cache */ + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; } /* @@ -2399,14 +3038,13 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** Purge the range of registers from the column cache. */ void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i; - int iLast = iReg + nReg - 1; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iReg && r<=iLast ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + int i = 0; + while( inColCache ){ + struct yColCache *p = &pParse->aColCache[i]; + if( p->iReg >= iReg && p->iReg < iReg+nReg ){ + cacheEntryClear(pParse, i); + }else{ + i++; } } } @@ -2431,8 +3069,7 @@ void sqlite3ExprCachePush(Parse *pParse){ ** the cache to the state it was in prior the most recent Push. */ void sqlite3ExprCachePop(Parse *pParse){ - int i; - struct yColCache *p; + int i = 0; assert( pParse->iCacheLevel>=1 ); pParse->iCacheLevel--; #ifdef SQLITE_DEBUG @@ -2440,10 +3077,11 @@ void sqlite3ExprCachePop(Parse *pParse){ printf("POP to %d\n", pParse->iCacheLevel); } #endif - for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + while( inColCache ){ + if( pParse->aColCache[i].iLevel>pParse->iCacheLevel ){ + cacheEntryClear(pParse, i); + }else{ + i++; } } } @@ -2457,7 +3095,7 @@ void sqlite3ExprCachePop(Parse *pParse){ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ if( p->iReg==iReg ){ p->tempReg = 0; } @@ -2501,7 +3139,7 @@ void sqlite3ExprCodeGetColumnOfTable( }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; int x = iCol; - if( !HasRowid(pTab) ){ + if( !HasRowid(pTab) && !IsVirtual(pTab) ){ x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); @@ -2535,8 +3173,8 @@ int sqlite3ExprCodeGetColumn( int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + if( p->iTable==iTable && p->iColumn==iColumn ){ p->lru = pParse->iCacheCnt++; sqlite3ExprCachePinRegister(pParse, p->iReg); return p->iReg; @@ -2568,19 +3206,20 @@ void sqlite3ExprCodeGetColumnToReg( */ void sqlite3ExprCacheClear(Parse *pParse){ int i; - struct yColCache *p; #if SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("CLEAR\n"); } #endif - for(i=0, p=pParse->aColCache; iiReg ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + for(i=0; inColCache; i++){ + if( pParse->aColCache[i].tempReg + && pParse->nTempRegaTempReg) + ){ + pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; } } + pParse->nColCache = 0; } /* @@ -2612,7 +3251,7 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ int r = p->iReg; if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ } @@ -2620,8 +3259,11 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ + /* -** Convert an expression node to a TK_REGISTER +** Convert a scalar expression node to a TK_REGISTER referencing +** register iReg. The caller must ensure that iReg already contains +** the correct value for the expression. */ static void exprToRegister(Expr *p, int iReg){ p->op2 = p->op; @@ -2630,6 +3272,38 @@ static void exprToRegister(Expr *p, int iReg){ ExprClearProperty(p, EP_Skip); } +/* +** Evaluate an expression (either a vector or a scalar expression) and store +** the result in continguous temporary registers. Return the index of +** the first register used to store the result. +** +** If the returned result register is a temporary scalar, then also write +** that register number into *piFreeable. If the returned result register +** is not a temporary or if the expression is a vector set *piFreeable +** to 0. +*/ +static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ + int iResult; + int nResult = sqlite3ExprVectorSize(p); + if( nResult==1 ){ + iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); + }else{ + *piFreeable = 0; + if( p->op==TK_SELECT ){ + iResult = sqlite3CodeSubselect(pParse, p, 0, 0); + }else{ + int i; + iResult = pParse->nMem+1; + pParse->nMem += nResult; + for(i=0; ix.pList->a[i].pExpr, i+iResult); + } + } + } + return iResult; +} + + /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -2647,9 +3321,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ int inReg = target; /* Results stored in register inReg */ int regFree1 = 0; /* If non-zero free this temporary register */ int regFree2 = 0; /* If non-zero free this temporary register */ - int r1, r2, r3, r4; /* Various register numbers */ - sqlite3 *db = pParse->db; /* The database connection */ + int r1, r2; /* Various register numbers */ Expr tempX; /* Temporary expression node */ + int p5 = 0; assert( target>0 && target<=pParse->nMem ); if( v==0 ){ @@ -2668,12 +3342,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ assert( pCol->iMem>0 ); - inReg = pCol->iMem; - break; + return pCol->iMem; }else if( pAggInfo->useSortingIdx ){ sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - break; + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ } @@ -2682,38 +3355,36 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( iTab<0 ){ if( pParse->ckBase>0 ){ /* Generating CHECK constraints or inserting into partial index */ - inReg = pExpr->iColumn + pParse->ckBase; - break; + return pExpr->iColumn + pParse->ckBase; }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ iTab = pParse->iSelfTab; } } - inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + return sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, pExpr->iColumn, iTab, target, pExpr->op2); - break; } case TK_INTEGER: { codeInteger(pParse, pExpr, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pExpr->u.zToken, 0, target); - break; + return target; } #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeLoadString(v, target, pExpr->u.zToken); - break; + return target; } case TK_NULL: { sqlite3VdbeAddOp2(v, OP_Null, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { @@ -2728,7 +3399,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( z[n]=='\'' ); zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); - break; + return target; } #endif case TK_VARIABLE: { @@ -2741,11 +3412,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } - break; + return target; } case TK_REGISTER: { - inReg = pExpr->iTable; - break; + return pExpr->iTable; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { @@ -2759,42 +3429,37 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); - break; + return inReg; } #endif /* SQLITE_OMIT_CAST */ + case TK_IS: + case TK_ISNOT: + op = (op==TK_IS) ? TK_EQ : TK_NE; + p5 = SQLITE_NULLEQ; + /* fall-through */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2); - assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); - assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); - assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); - assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); - testcase( regFree1==0 ); - testcase( regFree2==0 ); + Expr *pLeft = pExpr->pLeft; + if( sqlite3ExprIsVector(pLeft) ){ + codeVectorCompare(pParse, pExpr, target, op, p5); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | p5); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + } break; } case TK_AND: @@ -2832,10 +3497,12 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pLeft ); if( pLeft->op==TK_INTEGER ){ codeInteger(pParse, pLeft, 1, target); + return target; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( pLeft->op==TK_FLOAT ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pLeft->u.zToken, 1, target); + return target; #endif }else{ tempX.op = TK_INTEGER; @@ -2846,7 +3513,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); testcase( regFree2==0 ); } - inReg = target; break; } case TK_BITNOT: @@ -2855,7 +3521,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - inReg = target; sqlite3VdbeAddOp2(v, op, r1, inReg); break; } @@ -2880,7 +3545,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); }else{ - inReg = pInfo->aFunc[pExpr->iAgg].iMem; + return pInfo->aFunc[pExpr->iAgg].iMem; } break; } @@ -2888,10 +3553,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ExprList *pFarg; /* List of function arguments */ int nFarg; /* Number of function arguments */ FuncDef *pDef; /* The function definition object */ - int nId; /* Length of the function name in bytes */ const char *zId; /* The function name */ u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* The database connection */ u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ @@ -2904,10 +3569,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); + pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 && pParse->explain ){ + pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); + } +#endif if( pDef==0 || pDef->xFinalize!=0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); + sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); break; } @@ -2936,8 +3605,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ */ if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ assert( nFarg>=1 ); - inReg = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); - break; + return sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); } for(i=0; ix.pSelect->pEList->nExpr)!=1 ){ + sqlite3SubselectError(pParse, nCol, 1); + }else{ + return sqlite3CodeSubselect(pParse, pExpr, 0, 0); + } break; } + case TK_SELECT_COLUMN: { + if( pExpr->pLeft->iTable==0 ){ + pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft, 0, 0); + } + return pExpr->pLeft->iTable + pExpr->iColumn; + } case TK_IN: { int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = sqlite3VdbeMakeLabel(v); @@ -3031,7 +3710,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeResolveLabel(v, destIfFalse); sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); sqlite3VdbeResolveLabel(v, destIfNull); - break; + return target; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -3048,34 +3727,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** Z is stored in pExpr->pList->a[1].pExpr. */ case TK_BETWEEN: { - Expr *pLeft = pExpr->pLeft; - struct ExprList_item *pLItem = pExpr->x.pList->a; - Expr *pRight = pLItem->pExpr; - - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - r3 = sqlite3GetTempReg(pParse); - r4 = sqlite3GetTempReg(pParse); - codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); - pLItem++; - pRight = pLItem->pExpr; - sqlite3ReleaseTempReg(pParse, regFree2); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree2==0 ); - codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); - sqlite3ReleaseTempReg(pParse, r3); - sqlite3ReleaseTempReg(pParse, r4); - break; + exprCodeBetween(pParse, pExpr, target, 0, 0); + return target; } + case TK_SPAN: case TK_COLLATE: case TK_UPLUS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; + return sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); } case TK_TRIGGER: { @@ -3134,6 +3792,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } + case TK_VECTOR: { + sqlite3ErrorMsg(pParse, "row value misused"); + break; + } /* ** Form A: @@ -3177,8 +3839,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( (pX = pExpr->pLeft)!=0 ){ tempX = *pX; testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); + exprToRegister(&tempX, exprCodeVector(pParse, &tempX, ®Free1)); testcase( regFree1==0 ); + memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; opCompare.pLeft = &tempX; pTest = &opCompare; @@ -3212,7 +3875,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( db->mallocFailed || pParse->nErr>0 + assert( pParse->db->mallocFailed || pParse->nErr>0 || pParse->iCacheLevel==iCacheLevel ); sqlite3VdbeResolveLabel(v, endLabel); break; @@ -3457,20 +4120,33 @@ int sqlite3ExprCodeExprList( ** ** Code it as such, taking care to do the common subexpression ** elimination of x. +** +** The xJumpIf parameter determines details: +** +** NULL: Store the boolean result in reg[dest] +** sqlite3ExprIfTrue: Jump to dest if true +** sqlite3ExprIfFalse: Jump to dest if false +** +** The jumpIfNull parameter is ignored if xJumpIf is NULL. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ Expr *pExpr, /* The BETWEEN expression */ - int dest, /* Jump here if the jump is taken */ - int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int dest, /* Jump destination or storage location */ + void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ + + memset(&compLeft, 0, sizeof(Expr)); + memset(&compRight, 0, sizeof(Expr)); + memset(&exprAnd, 0, sizeof(Expr)); + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); exprX = *pExpr->pLeft; exprAnd.op = TK_AND; @@ -3482,23 +4158,25 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = &exprX; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); - if( jumpIfTrue ){ - sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + exprToRegister(&exprX, exprCodeVector(pParse, &exprX, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); }else{ - sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + exprX.flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); } sqlite3ReleaseTempReg(pParse, regFree1); /* Ensure adequate test coverage */ - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==0 ); } /* @@ -3550,12 +4228,20 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -3565,23 +4251,12 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -3599,7 +4274,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -3613,6 +4288,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { + default_expr: if( exprAlwaysTrue(pExpr) ){ sqlite3VdbeGoto(v, dest); }else if( exprAlwaysFalse(pExpr) ){ @@ -3706,12 +4382,20 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -3721,23 +4405,12 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -3753,7 +4426,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -3769,6 +4442,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { + default_expr: if( exprAlwaysFalse(pExpr) ){ sqlite3VdbeGoto(v, dest); }else if( exprAlwaysTrue(pExpr) ){ @@ -3935,6 +4609,61 @@ int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ return 0; } +/* +** An instance of the following structure is used by the tree walker +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. +*/ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ +}; + +/* +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. +*/ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + /* ** An instance of the following structure is used by the tree walker ** to count references to table columns in the arguments of an @@ -4132,7 +4861,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ pItem->iMem = ++pParse->nMem; assert( !ExprHasProperty(pExpr, EP_IntValue) ); pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, sqlite3Strlen30(pExpr->u.zToken), + pExpr->u.zToken, pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); if( pExpr->flags & EP_Distinct ){ pItem->iDistinct = pParse->nTab++; @@ -4218,7 +4947,7 @@ void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ if( iReg && pParse->nTempRegaTempReg) ){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ if( p->iReg==iReg ){ p->tempReg = 1; return; @@ -4229,10 +4958,11 @@ void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ } /* -** Allocate or deallocate a block of nReg consecutive registers +** Allocate or deallocate a block of nReg consecutive registers. */ int sqlite3GetTempRange(Parse *pParse, int nReg){ int i, n; + if( nReg==1 ) return sqlite3GetTempReg(pParse); i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ @@ -4246,6 +4976,10 @@ int sqlite3GetTempRange(Parse *pParse, int nReg){ return i; } void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, iReg); + return; + } sqlite3ExprCacheRemove(pParse, iReg, nReg); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; @@ -4260,3 +4994,26 @@ void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nTempReg = 0; pParse->nRangeReg = 0; } + +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeRegiRangeReg>=iFirst + ){ + return 0; + } + for(i=0; inTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ diff --git a/src/fkey.c b/src/fkey.c index 38fd4f75..883cc641 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -871,7 +871,7 @@ void sqlite3FkCheck( if( (db->flags&SQLITE_ForeignKeys)==0 ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the ** child table (the table that the foreign key definition is part of). */ @@ -1162,6 +1162,9 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ + return 0; + } pTrigger = pFKey->apTrigger[iAction]; if( action!=OE_None && !pTrigger ){ @@ -1239,10 +1242,10 @@ static Trigger *fkActionTrigger( if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } pList = sqlite3ExprListAppend(pParse, pList, pNew); sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); @@ -1369,7 +1372,8 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ - assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + assert( db==0 || IsVirtual(pTab) + || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ diff --git a/src/func.c b/src/func.c index d615cf90..6ec0962d 100644 --- a/src/func.c +++ b/src/func.c @@ -208,6 +208,8 @@ static void instrFunc( zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; + if( zNeedle==0 ) return; + assert( zHaystack ); } while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ N++; @@ -740,7 +742,7 @@ static int patternCompare( } c2 = Utf8Read(zString); if( c==c2 ) continue; - if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ continue; } if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; @@ -1316,6 +1318,26 @@ static void trimFunc( } +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +/* +** The "unknown" function is automatically substituted in place of +** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN +** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** When the "sqlite3" command-line shell is built using this functionality, +** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries +** involving application-defined functions to be examined in a generic +** sqlite3 shell. +*/ +static void unknownFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + /* no-op */ +} +#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ + + /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE_SOUNDEX compile-time option is used ** when SQLite is built. @@ -1386,6 +1408,14 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); }else{ @@ -1611,7 +1641,7 @@ static void groupConcatFinalize(sqlite3_context *context){ ** of the built-in functions above are part of the global function set. ** This routine only deals with those that are not global. */ -void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ +void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ @@ -1624,8 +1654,7 @@ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ */ static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, sqlite3Strlen30(zName), - 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); if( ALWAYS(pDef) ){ pDef->funcFlags |= flagVal; } @@ -1673,9 +1702,7 @@ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ return 0; } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, - sqlite3Strlen30(pExpr->u.zToken), - 2, SQLITE_UTF8, 0); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, 2, SQLITE_UTF8, 0); if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } @@ -1699,7 +1726,7 @@ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ ** ** After this routine runs */ -void sqlite3RegisterGlobalFunctions(void){ +void sqlite3RegisterBuiltinFunctions(void){ /* ** The following array holds FuncDef structures for all of the functions ** defined in this file. @@ -1707,8 +1734,27 @@ void sqlite3RegisterGlobalFunctions(void){ ** The array cannot be constant since changes are made to the ** FuncDef.pHash elements at start-time. The elements of this array ** are read-only after initialization is complete. + ** + ** For peak efficiency, put the most frequently used function last. */ - static SQLITE_WSD FuncDef aBuiltinFunc[] = { + static FuncDef aBuiltinFunc[] = { +#ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), +#endif +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), + FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), FUNCTION(rtrim, 1, 2, 0, trimFunc ), @@ -1726,8 +1772,6 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(instr, 2, 0, 0, instrFunc ), - FUNCTION(substr, 2, 0, 0, substrFunc ), - FUNCTION(substr, 3, 0, 0, substrFunc ), FUNCTION(printf, -1, 0, 0, printfFunc ), FUNCTION(unicode, 1, 0, 0, unicodeFunc ), FUNCTION(char, -1, 0, 0, charFunc ), @@ -1738,40 +1782,22 @@ void sqlite3RegisterGlobalFunctions(void){ #endif FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), - FUNCTION(coalesce, 1, 0, 0, 0 ), - FUNCTION(coalesce, 0, 0, 0, 0 ), - FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), FUNCTION(hex, 1, 0, 0, hexFunc ), FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), - FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), -#if SQLITE_USER_AUTHENTICATION - FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), -#endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), VFUNCTION(changes, 0, 0, 0, changes ), VFUNCTION(total_changes, 0, 0, 0, total_changes ), FUNCTION(replace, 3, 0, 0, replaceFunc ), FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), - #ifdef SQLITE_SOUNDEX - FUNCTION(soundex, 1, 0, 0, soundexFunc ), - #endif - #ifndef SQLITE_OMIT_LOAD_EXTENSION - VFUNCTION(load_extension, 1, 0, 0, loadExt ), - VFUNCTION(load_extension, 2, 0, 0, loadExt ), - #endif + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), @@ -1782,27 +1808,42 @@ void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #ifdef SQLITE_CASE_SENSITIVE_LIKE +#ifdef SQLITE_CASE_SENSITIVE_LIKE LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #else +#else LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), - #endif +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + FUNCTION(unknown, -1, 0, 0, unknownFunc ), +#endif + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), }; - - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aBuiltinFunc); - - for(i=0; iu.pHash){ + int n = sqlite3Strlen30(p->zName); + int h = p->zName[0] + n; + printf(" %s(%d)", p->zName, h); + } + printf("\n"); + } + } #endif } diff --git a/src/global.c b/src/global.c index 64966b35..f7865c95 100644 --- a/src/global.c +++ b/src/global.c @@ -70,6 +70,7 @@ const unsigned char sqlite3UpperToLower[] = { ** isxdigit() 0x08 ** toupper() 0x20 ** SQLite identifier character 0x40 +** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper ** case. i.e. if the character is a lower-case ASCII character. @@ -78,16 +79,13 @@ const unsigned char sqlite3UpperToLower[] = { ** ** (x & ~(map[x]&0x20)) ** -** Standard function tolower() is implemented using the sqlite3UpperToLower[] +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] ** array. tolower() is used more often than toupper() by SQLite. ** -** Bit 0x40 is set if the character non-alphanumeric and can be used in an +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an ** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. -** -** SQLite's versions are identical to the standard versions assuming a -** locale of "C". They are implemented as macros in sqliteInt.h. */ #ifdef SQLITE_ASCII const unsigned char sqlite3CtypeMap[256] = { @@ -95,7 +93,7 @@ const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ - 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ @@ -103,8 +101,8 @@ const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ - 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ - 0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ @@ -159,6 +157,18 @@ const unsigned char sqlite3CtypeMap[256] = { # define SQLITE_SORTER_PMASZ 250 #endif +/* Statement journals spill to disk when their size exceeds the following +** threshold (in bytes). 0 means that statement journals are created and +** written to disk immediately (the default behavior for SQLite versions +** before 3.12.0). -1 means always keep the entire statement journal in +** memory. (The statement journal is also always held entirely in memory +** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this +** setting.) +*/ +#ifndef SQLITE_STMTJRNL_SPILL +# define SQLITE_STMTJRNL_SPILL (64*1024) +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -173,6 +183,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* neverCorrupt */ 128, /* szLookaside */ 500, /* nLookaside */ + SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ @@ -211,7 +222,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { #ifndef SQLITE_OMIT_BUILTIN_TEST 0, /* xTestCallback */ #endif - 0 /* bLocaltimeFault */ + 0, /* bLocaltimeFault */ + 0x7ffffffe /* iOnceResetThreshold */ }; /* @@ -219,7 +231,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { ** database connections. After initialization, this table is ** read-only. */ -SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; +FuncDefHash sqlite3BuiltinFunctions; /* ** Constant tokens for values 0 and 1. @@ -234,7 +246,7 @@ const Token sqlite3IntTokens[] = { ** The value of the "pending" byte must be 0x40000000 (1 byte past the ** 1-gibabyte boundary) in a compatible database. SQLite never uses ** the database page that contains the pending byte. It never attempts -** to read or write that page. The pending byte page is set assign +** to read or write that page. The pending byte page is set aside ** for use by the VFS layers as space for managing file locks. ** ** During testing, it is often desirable to move the pending byte to diff --git a/src/hash.c b/src/hash.c index b5886e06..62b695a1 100644 --- a/src/hash.c +++ b/src/hash.c @@ -55,8 +55,12 @@ void sqlite3HashClear(Hash *pH){ static unsigned int strHash(const char *z){ unsigned int h = 0; unsigned char c; - while( (c = (unsigned char)*z++)!=0 ){ - h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). + ** 0x9e3779b1 is 2654435761 which is the closest prime number to + ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ + h += sqlite3UpperToLower[c]; + h *= 0x9e3779b1; } return h; } @@ -148,7 +152,7 @@ static HashElem *findElementWithHash( int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - if( pH->ht ){ + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; h = strHash(pKey) % pH->htsize; pEntry = &pH->ht[h]; diff --git a/src/hash.h b/src/hash.h index 6dfa4e03..90540eda 100644 --- a/src/hash.h +++ b/src/hash.h @@ -12,8 +12,8 @@ ** This is the header file for the generic hash-table implementation ** used in SQLite. */ -#ifndef _SQLITE_HASH_H_ -#define _SQLITE_HASH_H_ +#ifndef SQLITE_HASH_H +#define SQLITE_HASH_H /* Forward declarations of structures. */ typedef struct Hash Hash; @@ -93,4 +93,4 @@ void sqlite3HashClear(Hash*); */ /* #define sqliteHashCount(H) ((H)->count) // NOT USED */ -#endif /* _SQLITE_HASH_H_ */ +#endif /* SQLITE_HASH_H */ diff --git a/src/hwtime.h b/src/hwtime.h index b8bc5a29..5b209db8 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -13,8 +13,8 @@ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H /* ** The following routine only works on pentium-class (or newer) processors. @@ -82,4 +82,4 @@ #endif -#endif /* !defined(_HWTIME_H_) */ +#endif /* !defined(SQLITE_HWTIME_H) */ diff --git a/src/in-operator.md b/src/in-operator.md new file mode 100644 index 00000000..e9ad2101 --- /dev/null +++ b/src/in-operator.md @@ -0,0 +1,107 @@ +IN-Operator Implementation Notes +================================ + +## Definitions: + +An IN operator has one of the following formats: + +> + x IN (y1,y2,y3,...,yN) + x IN (subquery) + +The "x" is referred to as the LHS (left-hand side). The list or subquery +on the right is called the RHS (right-hand side). If the RHS is a list +it must be a non-empty list. But if the RHS is a subquery, it can be an +empty set. + +The LHS can be a scalar (a single quantity) or a vector (a list of +two or or more values) or a subquery that returns one or more columns. +We use the term "vector" to mean an actually list of values or a +subquery that returns two or more columns. An isolated value or +a subquery that returns a single columns is called a scalar. + +The RHS can be a subquery that returns a single column, a subquery +that returns two or more columns, or a list of scalars. It is not +currently support for the RHS to be a list of vectors. + +The number of columns for LHS must match the number of columns for +the RHS. If the RHS is a list of values, then the LHS must be a +scalar. If the RHS is a subquery returning N columns, then the LHS +must be a vector of size N. + +NULL values can occur in either or both of the LHS and RHS. +If the LHS contains only +NULL values then we say that it is a "total-NULL". If the LHS contains +some NULL values and some non-NULL values, then it is a "partial-NULL". +For a scalar, there is no difference between a partial-NULL and a total-NULL. +The RHS is a partial-NULL if any row contains a NULL value. The RHS is +a total-NULL if it contains one or more rows that contain only NULL values. +The LHS is called "non-NULL" if it contains no NULL values. The RHS is +called "non-NULL" if it contains no NULL values in any row. + +The result of an IN operator is one of TRUE, FALSE, or NULL. A NULL result +means that it cannot be determined if the LHS is contained in the RHS due +to the presence of NULL values. In some contexts (for example, when the IN +operator occurs in a WHERE clause) +the system only needs a binary result: TRUE or NOT-TRUE. One can also +to define a binary result of FALSE and NOT-FALSE, but +it turns out that no extra optimizations are possible in that case, so if +the FALSE/NOT-FALSE binary is needed, we have to compute the three-state +TRUE/FALSE/NULL result and then combine the TRUE and NULL values into +NOT-FALSE. + +A "NOT IN" operator is computed by first computing the equivalent IN +operator, then interchanging the TRUE and FALSE results. + +## Simple Full-Scan Algorithm + +The following algorithm always compute the correct answer. However, this +algorithm is suboptimal, especially if there are many rows on the RHS. + + 1. Set the null-flag to false + 2. For each row in the RHS: +
      +
    1. Compare the LHS against the RHS +
    2. If the LHS exactly matches the RHS, immediately return TRUE +
    3. If the comparison result is NULL, set the null-flag to true +
    + 3. If the null-flag is true, return NULL. + 4. Return FALSE + +## Optimized Algorithm + +The following procedure computes the same answer as the simple full-scan +algorithm, though it does so with less work in the common case. This +is the algorithm that is implemented in SQLite. + + 1. If the RHS is a constant list of length 1 or 2, then rewrite the + IN operator as a simple expression. Implement + + x IN (y1,y2) + + as if it were + + x=y1 OR x=y2 + + This is the INDEX_NOOP optimization and is only undertaken if the + IN operator is used for membership testing. If the IN operator is + driving a loop, then skip this step entirely. + + 2. Check the LHS to see if it is a partial-NULL and if it is, jump + ahead to step 5. + + 3. Do a binary search of the RHS using the LHS as a probe. If + an exact match is found, return TRUE. + + 4. If the RHS is non-NULL then return FALSE. + + 5. If we do not need to distinguish between FALSE and NULL, + then return FALSE. + + 6. For each row in the RHS, compare that row against the LHS and + if the result is NULL, immediately return NULL. In the case + of a scalar IN operator, we only need to look at the very first + row the RHS because for a scalar RHS, all NULLs will always come + first. If the RHS is empty, this step is a no-op. + + 7. Return FALSE. diff --git a/src/insert.c b/src/insert.c index 7ff884b8..8167ff78 100644 --- a/src/insert.c +++ b/src/insert.c @@ -200,7 +200,9 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ /* ** Locate or create an AutoincInfo structure associated with table pTab ** which is in database iDb. Return the register number for the register -** that holds the maximum rowid. +** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT +** table. (Also return zero when doing a VACUUM since we do not want to +** update the AUTOINCREMENT counters during a VACUUM.) ** ** There is at most one AutoincInfo structure per table even if the ** same table is autoincremented multiple times due to inserts within @@ -223,7 +225,9 @@ static int autoIncBegin( Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ - if( pTab->tabFlags & TF_Autoincrement ){ + if( (pTab->tabFlags & TF_Autoincrement)!=0 + && (pParse->db->flags & SQLITE_Vacuum)==0 + ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); AutoincInfo *pInfo; @@ -481,7 +485,6 @@ void sqlite3Insert( sqlite3 *db; /* The main database structure */ Table *pTab; /* The table to insert into. aka TABLE */ char *zTab; /* Name of the table into which we are inserting */ - const char *zDb; /* Name of the database holding this table */ int i, j, idx; /* Loop counters */ Vdbe *v; /* Generate code into this virtual machine */ Index *pIdx; /* For looping over indices of the table */ @@ -496,7 +499,6 @@ void sqlite3Insert( int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ - Db *pDb; /* The database containing table being inserted into */ u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ @@ -546,9 +548,8 @@ void sqlite3Insert( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - pDb = &db->aDb[iDb]; - zDb = pDb->zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, + db->aDb[iDb].zDbSName) ){ goto insert_cleanup; } withoutRowid = !HasRowid(pTab); @@ -1431,9 +1432,18 @@ void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, - ONEPASS_SINGLE, -1); + regNewData, 1, 0, OE_Replace, 1, -1); }else{ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( HasRowid(pTab) ){ + /* This OP_Delete opcode fires the pre-update-hook only. It does + ** not modify the b-tree. It is more efficient to let the coming + ** OP_Insert replace the existing entry than it is to delete the + ** existing entry and then insert a new one. */ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); + sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); + } +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ if( pTab->pIndex ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); @@ -1703,7 +1713,7 @@ void sqlite3CompleteInsertion( } sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); } sqlite3VdbeChangeP5(v, pik_flags); } @@ -1768,15 +1778,15 @@ int sqlite3OpenTableAndIndices( for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + if( piDataCur ) *piDataCur = iIdxCur; + p5 = 0; + } if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - } - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; - }else{ sqlite3VdbeChangeP5(v, p5); + VdbeComment((v, "%s", pIdx->zName)); } } if( iBase>pParse->nTab ) pParse->nTab = iBase; @@ -1999,11 +2009,15 @@ static int xferOptimization( return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 - && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) - || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) - ){ - return 0; /* Default values must be the same for all columns */ + if( i>0 ){ + assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); + assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); + if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) + || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, + pSrcCol->pDflt->u.zToken)!=0) + ){ + return 0; /* Default values must be the same for all columns */ + } } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ @@ -2099,7 +2113,7 @@ static int xferOptimization( } sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, - pDest->zName, 0); + (char*)pDest, P4_TABLE); sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -2163,6 +2177,7 @@ static int xferOptimization( sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); if( emptyDestTest ){ + sqlite3AutoincrementEnd(pParse); sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); diff --git a/src/journal.c b/src/journal.c deleted file mode 100644 index fed27be3..00000000 --- a/src/journal.c +++ /dev/null @@ -1,256 +0,0 @@ -/* -** 2007 August 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a special kind of sqlite3_file object used -** by SQLite to create journal files if the atomic-write optimization -** is enabled. -** -** The distinctive characteristic of this sqlite3_file is that the -** actual on disk file is created lazily. When the file is created, -** the caller specifies a buffer size for an in-memory buffer to -** be used to service read() and write() requests. The actual file -** on disk is not created or populated until either: -** -** 1) The in-memory representation grows too large for the allocated -** buffer, or -** 2) The sqlite3JournalCreate() function is called. -*/ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE -#include "sqliteInt.h" - - -/* -** A JournalFile object is a subclass of sqlite3_file used by -** as an open file handle for journal files. -*/ -struct JournalFile { - sqlite3_io_methods *pMethod; /* I/O methods on journal files */ - int nBuf; /* Size of zBuf[] in bytes */ - char *zBuf; /* Space to buffer journal writes */ - int iSize; /* Amount of zBuf[] currently used */ - int flags; /* xOpen flags */ - sqlite3_vfs *pVfs; /* The "real" underlying VFS */ - sqlite3_file *pReal; /* The "real" underlying file descriptor */ - const char *zJournal; /* Name of the journal file */ -}; -typedef struct JournalFile JournalFile; - -/* -** If it does not already exists, create and populate the on-disk file -** for JournalFile p. -*/ -static int createFile(JournalFile *p){ - int rc = SQLITE_OK; - if( !p->pReal ){ - sqlite3_file *pReal = (sqlite3_file *)&p[1]; - rc = sqlite3OsOpen(p->pVfs, p->zJournal, pReal, p->flags, 0); - if( rc==SQLITE_OK ){ - p->pReal = pReal; - if( p->iSize>0 ){ - assert(p->iSize<=p->nBuf); - rc = sqlite3OsWrite(p->pReal, p->zBuf, p->iSize, 0); - } - if( rc!=SQLITE_OK ){ - /* If an error occurred while writing to the file, close it before - ** returning. This way, SQLite uses the in-memory journal data to - ** roll back changes made to the internal page-cache before this - ** function was called. */ - sqlite3OsClose(pReal); - p->pReal = 0; - } - } - } - return rc; -} - -/* -** Close the file. -*/ -static int jrnlClose(sqlite3_file *pJfd){ - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - sqlite3OsClose(p->pReal); - } - sqlite3_free(p->zBuf); - return SQLITE_OK; -} - -/* -** Read data from the file. -*/ -static int jrnlRead( - sqlite3_file *pJfd, /* The journal file from which to read */ - void *zBuf, /* Put the results here */ - int iAmt, /* Number of bytes to read */ - sqlite_int64 iOfst /* Begin reading at this offset */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsRead(p->pReal, zBuf, iAmt, iOfst); - }else if( (iAmt+iOfst)>p->iSize ){ - rc = SQLITE_IOERR_SHORT_READ; - }else{ - memcpy(zBuf, &p->zBuf[iOfst], iAmt); - } - return rc; -} - -/* -** Write data to the file. -*/ -static int jrnlWrite( - sqlite3_file *pJfd, /* The journal file into which to write */ - const void *zBuf, /* Take data to be written from here */ - int iAmt, /* Number of bytes to write */ - sqlite_int64 iOfst /* Begin writing at this offset into the file */ -){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( !p->pReal && (iOfst+iAmt)>p->nBuf ){ - rc = createFile(p); - } - if( rc==SQLITE_OK ){ - if( p->pReal ){ - rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst); - }else{ - memcpy(&p->zBuf[iOfst], zBuf, iAmt); - if( p->iSize<(iOfst+iAmt) ){ - p->iSize = (iOfst+iAmt); - } - } - } - return rc; -} - -/* -** Truncate the file. -*/ -static int jrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsTruncate(p->pReal, size); - }else if( sizeiSize ){ - p->iSize = size; - } - return rc; -} - -/* -** Sync the file. -*/ -static int jrnlSync(sqlite3_file *pJfd, int flags){ - int rc; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsSync(p->pReal, flags); - }else{ - rc = SQLITE_OK; - } - return rc; -} - -/* -** Query the size of the file in bytes. -*/ -static int jrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ - int rc = SQLITE_OK; - JournalFile *p = (JournalFile *)pJfd; - if( p->pReal ){ - rc = sqlite3OsFileSize(p->pReal, pSize); - }else{ - *pSize = (sqlite_int64) p->iSize; - } - return rc; -} - -/* -** Table of methods for JournalFile sqlite3_file object. -*/ -static struct sqlite3_io_methods JournalFileMethods = { - 1, /* iVersion */ - jrnlClose, /* xClose */ - jrnlRead, /* xRead */ - jrnlWrite, /* xWrite */ - jrnlTruncate, /* xTruncate */ - jrnlSync, /* xSync */ - jrnlFileSize, /* xFileSize */ - 0, /* xLock */ - 0, /* xUnlock */ - 0, /* xCheckReservedLock */ - 0, /* xFileControl */ - 0, /* xSectorSize */ - 0, /* xDeviceCharacteristics */ - 0, /* xShmMap */ - 0, /* xShmLock */ - 0, /* xShmBarrier */ - 0 /* xShmUnmap */ -}; - -/* -** Open a journal file. -*/ -int sqlite3JournalOpen( - sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ - const char *zName, /* Name of the journal file */ - sqlite3_file *pJfd, /* Preallocated, blank file handle */ - int flags, /* Opening flags */ - int nBuf /* Bytes buffered before opening the file */ -){ - JournalFile *p = (JournalFile *)pJfd; - memset(p, 0, sqlite3JournalSize(pVfs)); - if( nBuf>0 ){ - p->zBuf = sqlite3MallocZero(nBuf); - if( !p->zBuf ){ - return SQLITE_NOMEM; - } - }else{ - return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); - } - p->pMethod = &JournalFileMethods; - p->nBuf = nBuf; - p->flags = flags; - p->zJournal = zName; - p->pVfs = pVfs; - return SQLITE_OK; -} - -/* -** If the argument p points to a JournalFile structure, and the underlying -** file has not yet been created, create it now. -*/ -int sqlite3JournalCreate(sqlite3_file *p){ - if( p->pMethods!=&JournalFileMethods ){ - return SQLITE_OK; - } - return createFile((JournalFile *)p); -} - -/* -** The file-handle passed as the only argument is guaranteed to be an open -** file. It may or may not be of class JournalFile. If the file is a -** JournalFile, and the underlying file on disk has not yet been opened, -** return 0. Otherwise, return 1. -*/ -int sqlite3JournalExists(sqlite3_file *p){ - return (p->pMethods!=&JournalFileMethods || ((JournalFile *)p)->pReal!=0); -} - -/* -** Return the number of bytes required to store a JournalFile that uses vfs -** pVfs to create the underlying on-disk files. -*/ -int sqlite3JournalSize(sqlite3_vfs *pVfs){ - return (pVfs->szOsFile+sizeof(JournalFile)); -} -#endif diff --git a/src/legacy.c b/src/legacy.c index 1b5e518d..bd34512d 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -131,7 +131,7 @@ exec_out: if( *pzErrMsg ){ memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ diff --git a/src/loadext.c b/src/loadext.c index 94298c47..6011fd2b 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -21,7 +21,6 @@ #include #ifndef SQLITE_OMIT_LOAD_EXTENSION - /* ** Some API routines are omitted when various features are ** excluded from a build of SQLite. Substitute a NULL pointer @@ -91,7 +90,7 @@ # define sqlite3_enable_shared_cache 0 #endif -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) # define sqlite3_profile 0 # define sqlite3_trace 0 #endif @@ -111,6 +110,10 @@ #define sqlite3_blob_reopen 0 #endif +#if defined(SQLITE_OMIT_TRACE) +# define sqlite3_trace_v2 0 +#endif + /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -414,7 +417,12 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.10.0 and later */ sqlite3_status64, sqlite3_strlike, - sqlite3_db_cacheflush + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno, + /* Version 3.14.0 and later */ + sqlite3_trace_v2, + sqlite3_expanded_sql }; /* @@ -437,13 +445,14 @@ static int sqlite3LoadExtension( ){ sqlite3_vfs *pVfs = db->pVfs; void *handle; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; char *zErrmsg = 0; const char *zEntry; char *zAltEntry = 0; void **aHandle; u64 nMsg = 300 + sqlite3Strlen30(zFile); int ii; + int rc; /* Shared library endings to try if zFile cannot be loaded as written */ static const char *azEndings[] = { @@ -462,8 +471,9 @@ static int sqlite3LoadExtension( /* Ticket #1863. To avoid a creating security problems for older ** applications that relink against newer versions of SQLite, the ** ability to run load_extension is turned off by default. One - ** must call sqlite3_enable_load_extension() to turn on extension - ** loading. Otherwise you get the following error. + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. */ if( (db->flags & SQLITE_LoadExtension)==0 ){ if( pzErrMsg ){ @@ -478,7 +488,7 @@ static int sqlite3LoadExtension( #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii=0 && zFile[iFile]!='/'; iFile--){} @@ -527,8 +536,7 @@ static int sqlite3LoadExtension( } memcpy(zAltEntry+iEntry, "_init", 6); zEntry = zAltEntry; - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zEntry); + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ @@ -545,7 +553,9 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } sqlite3_free(zAltEntry); - if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + rc = xInit(db, &zErrmsg, &sqlite3Apis); + if( rc ){ + if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -557,7 +567,7 @@ static int sqlite3LoadExtension( /* Append the new shared library handle to the db->aExtension array. */ aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); if( aHandle==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( db->nExtension>0 ){ memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); @@ -602,26 +612,15 @@ void sqlite3CloseExtensions(sqlite3 *db){ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ - db->flags |= SQLITE_LoadExtension; + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~SQLITE_LoadExtension; + db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ - -/* -** The auto-extension code added regardless of whether or not extension -** loading is supported. We need a dummy sqlite3Apis pointer for that -** code if regular extension loading is not available. This is that -** dummy pointer. -*/ -#ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; -#endif - +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ /* ** The following object holds the list of automatically loaded @@ -656,7 +655,9 @@ static SQLITE_WSD struct sqlite3AutoExtList { ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -int sqlite3_auto_extension(void (*xInit)(void)){ +int sqlite3_auto_extension( + void (*xInit)(void) +){ int rc = SQLITE_OK; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -679,7 +680,7 @@ int sqlite3_auto_extension(void (*xInit)(void)){ void (**aNew)(void); aNew = sqlite3_realloc64(wsdAutoext.aExt, nByte); if( aNew==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ wsdAutoext.aExt = aNew; wsdAutoext.aExt[wsdAutoext.nExt] = xInit; @@ -701,7 +702,9 @@ int sqlite3_auto_extension(void (*xInit)(void)){ ** Return 1 if xInit was found on the list and removed. Return 0 if xInit ** was not on the list. */ -int sqlite3_cancel_auto_extension(void (*xInit)(void)){ +int sqlite3_cancel_auto_extension( + void (*xInit)(void) +){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -750,7 +753,7 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ u32 i; int go = 1; int rc; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; wsdAutoextInit; if( wsdAutoext.nExt==0 ){ @@ -761,18 +764,22 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ char *zErrmsg; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + const sqlite3_api_routines *pThunk = 0; +#else + const sqlite3_api_routines *pThunk = &sqlite3Apis; #endif sqlite3_mutex_enter(mutex); if( i>=wsdAutoext.nExt ){ xInit = 0; go = 0; }else{ - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - wsdAutoext.aExt[i]; + xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ + if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; diff --git a/src/main.c b/src/main.c index 922af131..392a0042 100644 --- a/src/main.c +++ b/src/main.c @@ -187,7 +187,7 @@ int sqlite3_initialize(void){ sqlite3GlobalConfig.pInitMutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } } @@ -218,7 +218,6 @@ int sqlite3_initialize(void){ */ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); sqlite3GlobalConfig.inProgress = 1; #ifdef SQLITE_ENABLE_SQLLOG { @@ -226,8 +225,8 @@ int sqlite3_initialize(void){ sqlite3_init_sqllog(); } #endif - memset(pHash, 0, sizeof(sqlite3GlobalFunctions)); - sqlite3RegisterGlobalFunctions(); + memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); + sqlite3RegisterBuiltinFunctions(); if( sqlite3GlobalConfig.isPCacheInit==0 ){ rc = sqlite3PcacheInitialize(); } @@ -634,6 +633,11 @@ int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_STMTJRNL_SPILL: { + sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); + break; + } + default: { rc = SQLITE_ERROR; break; @@ -785,6 +789,11 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ int rc; va_start(ap, op); switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; + } case SQLITE_DBCONFIG_LOOKASIDE: { void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ @@ -797,8 +806,10 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ int op; /* The opcode */ u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -958,7 +969,7 @@ void sqlite3CloseSavepoints(sqlite3 *db){ ** with SQLITE_ANY as the encoding. */ static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->pDestructor; + FuncDestructor *pDestructor = p->u.pDestructor; if( pDestructor ){ pDestructor->nRef--; if( pDestructor->nRef==0 ){ @@ -1027,6 +1038,9 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->xTrace(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } /* Force xDisconnect calls on all virtual tables */ disconnectAllVtab(db); @@ -1140,18 +1154,17 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ */ sqlite3ConnectionClosed(db); - for(j=0; jaFunc.a); j++){ - FuncDef *pNext, *pHash, *p; - for(p=db->aFunc.a[j]; p; p=pHash){ - pHash = p->pHash; - while( p ){ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - } - } + for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ + FuncDef *pNext, *p; + p = sqliteHashData(i); + do{ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + }while( p ); } + sqlite3HashClear(&db->aFunc); for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ CollSeq *pColl = (CollSeq *)sqliteHashData(i); /* Invoke any destructors registered for collation sequence user data. */ @@ -1630,7 +1643,7 @@ int sqlite3CreateFunc( ** is being overridden/deleted but there are no active VMs, allow the ** operation to continue but invalidate all precompiled statements. */ - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ if( db->nVdbeActive ){ sqlite3ErrorWithMsg(db, SQLITE_BUSY, @@ -1642,10 +1655,10 @@ int sqlite3CreateFunc( } } - p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); assert(p || db->mallocFailed); if( !p ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* If an older version of the function with a configured destructor is @@ -1655,7 +1668,7 @@ int sqlite3CreateFunc( if( pDestructor ){ pDestructor->nRef++; } - p->pDestructor = pDestructor; + p->u.pDestructor = pDestructor; p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; testcase( p->funcFlags & SQLITE_DETERMINISTIC ); p->xSFunc = xSFunc ? xSFunc : xStep; @@ -1770,7 +1783,6 @@ int sqlite3_overload_function( const char *zName, int nArg ){ - int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; #ifdef SQLITE_ENABLE_API_ARMOR @@ -1779,7 +1791,7 @@ int sqlite3_overload_function( } #endif sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ + if( sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, 0, sqlite3InvalidFunction, 0, 0, 0); } @@ -1797,7 +1809,8 @@ int sqlite3_overload_function( ** trace is a pointer to a function that is invoked at the start of each ** SQL statement. */ -void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ +#ifndef SQLITE_OMIT_DEPRECATED +void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ void *pOld; #ifdef SQLITE_ENABLE_API_ARMOR @@ -1808,11 +1821,38 @@ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ #endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; - db->xTrace = xTrace; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ + +/* Register a trace callback using the version-2 interface. +*/ +int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED /* ** Register a profile function. The pArg from the previously registered ** profile function is returned. @@ -1841,6 +1881,7 @@ void *sqlite3_profile( sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ #endif /* SQLITE_OMIT_TRACE */ /* @@ -1919,6 +1960,27 @@ void *sqlite3_rollback_hook( return pRet; } +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +void *sqlite3_preupdate_hook( + sqlite3 *db, /* Attach the hook to this database */ + void(*xCallback)( /* Callback function */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), + void *pArg /* First callback argument */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pPreUpdateArg; + db->xPreUpdateCallback = xCallback; + db->pPreUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + #ifndef SQLITE_OMIT_WAL /* ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). @@ -2149,14 +2211,14 @@ int sqlite3TempInMemory(const sqlite3 *db){ const char *sqlite3_errmsg(sqlite3 *db){ const char *z; if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM); + return sqlite3ErrStr(SQLITE_NOMEM_BKPT); } if( !sqlite3SafetyCheckSickOrOk(db) ){ return sqlite3ErrStr(SQLITE_MISUSE_BKPT); } sqlite3_mutex_enter(db->mutex); if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM); + z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); }else{ testcase( db->pErr==0 ); z = (char*)sqlite3_value_text(db->pErr); @@ -2224,7 +2286,7 @@ int sqlite3_errcode(sqlite3 *db){ return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return db->errCode & db->errMask; } @@ -2233,10 +2295,13 @@ int sqlite3_extended_errcode(sqlite3 *db){ return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return db->errCode; } +int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} /* ** Return a string that describes the kind of error specified in the @@ -2313,7 +2378,7 @@ static int createCollation( } pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM; + if( pColl==0 ) return SQLITE_NOMEM_BKPT; pColl->xCmp = xCompare; pColl->pUser = pCtx; pColl->xDel = xDel; @@ -2361,8 +2426,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_VDBE_OP<40 # error SQLITE_MAX_VDBE_OP must be at least 40 #endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 #endif #if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 # error SQLITE_MAX_ATTACHED must be between 0 and 125 @@ -2492,7 +2557,7 @@ int sqlite3ParseUri( for(iIn=0; iInaCollSeq); @@ -2857,7 +2925,7 @@ static int openDatabase( flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3Error(db, rc); goto opendb_out; @@ -2868,13 +2936,13 @@ static int openDatabase( sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - /* The default safety_level for the main database is 'full'; for the temp - ** database it is 'NONE'. This matches the pager layer defaults. + /* The default safety_level for the main database is FULL; for the temp + ** database it is OFF. This matches the pager layer defaults. */ - db->aDb[0].zName = "main"; - db->aDb[0].safety_level = 3; - db->aDb[1].zName = "temp"; - db->aDb[1].safety_level = 1; + db->aDb[0].zDbSName = "main"; + db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + db->aDb[1].zDbSName = "temp"; + db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; db->magic = SQLITE_MAGIC_OPEN; if( db->mallocFailed ){ @@ -2886,12 +2954,21 @@ static int openDatabase( ** is accessed. */ sqlite3Error(db, SQLITE_OK); - sqlite3RegisterBuiltinFunctions(db); + sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + +#ifdef SQLITE_ENABLE_FTS5 + /* Register any built-in FTS5 module before loading the automatic + ** extensions. This allows automatic extensions to register FTS5 + ** tokenizers and auxiliary functions. */ + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif /* Load automatic extensions - extensions that have been registered ** using the sqlite3_automatic_extension() API. */ - rc = sqlite3_errcode(db); if( rc==SQLITE_OK ){ sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); @@ -2920,12 +2997,6 @@ static int openDatabase( } #endif -#ifdef SQLITE_ENABLE_FTS5 - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts5Init(db); - } -#endif - #ifdef SQLITE_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); @@ -3060,7 +3131,7 @@ int sqlite3_open16( SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; } }else{ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } sqlite3ValueFree(pVal); @@ -3205,7 +3276,7 @@ int sqlite3_get_autocommit(sqlite3 *db){ /* ** The following routines are substitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error ** constants. They serve two purposes: ** ** 1. Serve as a convenient place to set a breakpoint in a debugger @@ -3214,28 +3285,33 @@ int sqlite3_get_autocommit(sqlite3 *db){ ** 2. Invoke sqlite3_log() to provide the source code location where ** a low-level error is first detected. */ +static int reportError(int iErr, int lineno, const char *zType){ + sqlite3_log(iErr, "%s at line %d of [%.10s]", + zType, lineno, 20+sqlite3_sourceid()); + return iErr; +} int sqlite3CorruptError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CORRUPT, - "database corruption at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CORRUPT; + return reportError(SQLITE_CORRUPT, lineno, "database corruption"); } int sqlite3MisuseError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_MISUSE, - "misuse at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_MISUSE; + return reportError(SQLITE_MISUSE, lineno, "misuse"); } int sqlite3CantopenError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(SQLITE_CANTOPEN, - "cannot open file at line %d of [%.10s]", - lineno, 20+sqlite3_sourceid()); - return SQLITE_CANTOPEN; + return reportError(SQLITE_CANTOPEN, lineno, "cannot open file"); } - +#ifdef SQLITE_DEBUG +int sqlite3NomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return reportError(SQLITE_NOMEM, lineno, "OOM"); +} +int sqlite3IoerrnomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); +} +#endif #ifndef SQLITE_OMIT_DEPRECATED /* @@ -3329,7 +3405,7 @@ int sqlite3_table_column_metadata( ** explicitly declared column. Copy meta information from *pCol. */ if( pCol ){ - zDataType = pCol->zType; + zDataType = sqlite3ColumnType(pCol,0); zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; @@ -3708,6 +3784,15 @@ int sqlite3_test_control(int op, ...){ break; } + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; + } /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); ** @@ -3829,7 +3914,7 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ int i; for(i=0; inDb; i++){ if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) + && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zDbSName)==0) ){ return db->aDb[i].pBt; } diff --git a/src/malloc.c b/src/malloc.c index ebe04403..462d78e6 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -795,7 +795,7 @@ void sqlite3OomClear(sqlite3 *db){ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ sqlite3OomClear(db); sqlite3Error(db, SQLITE_NOMEM); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* diff --git a/src/memjournal.c b/src/memjournal.c index 6452cecc..cd8b87d8 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -13,6 +13,15 @@ ** This file contains code use to implement an in-memory rollback journal. ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +** +** Update: The in-memory journal is also used to temporarily cache +** smaller journals that are not critical for power-loss recovery. +** For example, statement journals that are not too big will be held +** entirely in memory, thus reducing the number of file I/O calls, and +** more importantly, reducing temporary file creation events. If these +** journals become too large for memory, they are spilled to disk. But +** in the common case, they are usually small and no file I/O needs to +** occur. */ #include "sqliteInt.h" @@ -21,24 +30,28 @@ typedef struct MemJournal MemJournal; typedef struct FilePoint FilePoint; typedef struct FileChunk FileChunk; -/* Space to hold the rollback journal is allocated in increments of -** this many bytes. -** -** The size chosen is a little less than a power of two. That way, -** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This minimizes wasted space in power-of-two -** memory allocators. -*/ -#define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) - /* ** The rollback journal is composed of a linked list of these structures. +** +** The zChunk array is always at least 8 bytes in size - usually much more. +** Its actual size is stored in the MemJournal.nChunkSize variable. */ struct FileChunk { FileChunk *pNext; /* Next chunk in the journal */ - u8 zChunk[JOURNAL_CHUNKSIZE]; /* Content of this chunk */ + u8 zChunk[8]; /* Content of this chunk */ }; +/* +** By default, allocate this many bytes of memory for each FileChunk object. +*/ +#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 + +/* +** For chunk size nChunkSize, return the number of bytes that should +** be allocated for each FileChunk structure. +*/ +#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) + /* ** An instance of this object serves as a cursor into the rollback journal. ** The cursor can be either for reading or writing. @@ -49,14 +62,22 @@ struct FilePoint { }; /* -** This subclass is a subclass of sqlite3_file. Each open memory-journal +** This structure is a subclass of sqlite3_file. Each open memory-journal ** is an instance of this class. */ struct MemJournal { - sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + int nChunkSize; /* In-memory chunk-size */ + + int nSpill; /* Bytes of data before flushing */ + int nSize; /* Bytes of data currently in memory */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ + + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + const char *zJournal; /* Name of the journal file */ }; /* @@ -75,36 +96,94 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; - /* SQLite never tries to read past the end of a rollback journal file */ - assert( iOfst+iAmt<=p->endpoint.iOffset ); +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( (iAmt+iOfst)>p->endpoint.iOffset ){ + return SQLITE_IOERR_SHORT_READ; + } +#endif + assert( (iAmt+iOfst)<=p->endpoint.iOffset ); + assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; for(pChunk=p->pFirst; - ALWAYS(pChunk) && (iOff+JOURNAL_CHUNKSIZE)<=iOfst; + ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; pChunk=pChunk->pNext ){ - iOff += JOURNAL_CHUNKSIZE; + iOff += p->nChunkSize; } }else{ pChunk = p->readpoint.pChunk; + assert( pChunk!=0 ); } - iChunkOffset = (int)(iOfst%JOURNAL_CHUNKSIZE); + iChunkOffset = (int)(iOfst%p->nChunkSize); do { - int iSpace = JOURNAL_CHUNKSIZE - iChunkOffset; - int nCopy = MIN(nRead, (JOURNAL_CHUNKSIZE - iChunkOffset)); - memcpy(zOut, &pChunk->zChunk[iChunkOffset], nCopy); + int iSpace = p->nChunkSize - iChunkOffset; + int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); + memcpy(zOut, (u8*)pChunk->zChunk + iChunkOffset, nCopy); zOut += nCopy; nRead -= iSpace; iChunkOffset = 0; } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; p->readpoint.pChunk = pChunk; return SQLITE_OK; } +/* +** Free the list of FileChunk structures headed at MemJournal.pFirst. +*/ +static void memjrnlFreeChunks(MemJournal *p){ + FileChunk *pIter; + FileChunk *pNext; + for(pIter=p->pFirst; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } + p->pFirst = 0; +} + +/* +** Flush the contents of memory to a real file on disk. +*/ +static int memjrnlCreateFile(MemJournal *p){ + int rc; + sqlite3_file *pReal = (sqlite3_file*)p; + MemJournal copy = *p; + + memset(p, 0, sizeof(MemJournal)); + rc = sqlite3OsOpen(copy.pVfs, copy.zJournal, pReal, copy.flags, 0); + if( rc==SQLITE_OK ){ + int nChunk = copy.nChunkSize; + i64 iOff = 0; + FileChunk *pIter; + for(pIter=copy.pFirst; pIter; pIter=pIter->pNext){ + if( iOff + nChunk > copy.endpoint.iOffset ){ + nChunk = copy.endpoint.iOffset - iOff; + } + rc = sqlite3OsWrite(pReal, (u8*)pIter->zChunk, nChunk, iOff); + if( rc ) break; + iOff += nChunk; + } + if( rc==SQLITE_OK ){ + /* No error has occurred. Free the in-memory buffers. */ + memjrnlFreeChunks(©); + } + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while creating or writing to the file, restore + ** the original before returning. This way, SQLite uses the in-memory + ** journal data to roll back changes made to the internal page-cache + ** before this function was called. */ + sqlite3OsClose(pReal); + *p = copy; + } + return rc; +} + + /* ** Write data to the file. */ @@ -118,38 +197,62 @@ static int memjrnlWrite( int nWrite = iAmt; u8 *zWrite = (u8 *)zBuf; - /* An in-memory journal file should only ever be appended to. Random - ** access writes are not required by sqlite. - */ - assert( iOfst==p->endpoint.iOffset ); - UNUSED_PARAMETER(iOfst); - - while( nWrite>0 ){ - FileChunk *pChunk = p->endpoint.pChunk; - int iChunkOffset = (int)(p->endpoint.iOffset%JOURNAL_CHUNKSIZE); - int iSpace = MIN(nWrite, JOURNAL_CHUNKSIZE - iChunkOffset); - - if( iChunkOffset==0 ){ - /* New chunk is required to extend the file. */ - FileChunk *pNew = sqlite3_malloc(sizeof(FileChunk)); - if( !pNew ){ - return SQLITE_IOERR_NOMEM; - } - pNew->pNext = 0; - if( pChunk ){ - assert( p->pFirst ); - pChunk->pNext = pNew; - }else{ - assert( !p->pFirst ); - p->pFirst = pNew; - } - p->endpoint.pChunk = pNew; + /* If the file should be created now, create it and write the new data + ** into the file on disk. */ + if( p->nSpill>0 && (iAmt+iOfst)>p->nSpill ){ + int rc = memjrnlCreateFile(p); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pJfd, zBuf, iAmt, iOfst); } + return rc; + } - memcpy(&p->endpoint.pChunk->zChunk[iChunkOffset], zWrite, iSpace); - zWrite += iSpace; - nWrite -= iSpace; - p->endpoint.iOffset += iSpace; + /* If the contents of this write should be stored in memory */ + else{ + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required. The only exception to this is when + ** the in-memory journal is being used by a connection using the + ** atomic-write optimization. In this case the first 28 bytes of the + ** journal file may be written as part of committing the transaction. */ + assert( iOfst==p->endpoint.iOffset || iOfst==0 ); +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + if( iOfst==0 && p->pFirst ){ + assert( p->nChunkSize>iAmt ); + memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); + }else +#else + assert( iOfst>0 || p->pFirst==0 ); +#endif + { + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); + int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + p->nSize = iAmt + iOfst; + } } return SQLITE_OK; @@ -157,19 +260,21 @@ static int memjrnlWrite( /* ** Truncate the file. +** +** If the journal file is already on disk, truncate it there. Or, if it +** is still in main memory but is being truncated to zero bytes in size, +** ignore */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; - FileChunk *pChunk; - assert(size==0); - UNUSED_PARAMETER(size); - pChunk = p->pFirst; - while( pChunk ){ - FileChunk *pTmp = pChunk; - pChunk = pChunk->pNext; - sqlite3_free(pTmp); + if( ALWAYS(size==0) ){ + memjrnlFreeChunks(p); + p->nSize = 0; + p->endpoint.pChunk = 0; + p->endpoint.iOffset = 0; + p->readpoint.pChunk = 0; + p->readpoint.iOffset = 0; } - sqlite3MemJournalOpen(pJfd); return SQLITE_OK; } @@ -177,21 +282,19 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ ** Close the file. */ static int memjrnlClose(sqlite3_file *pJfd){ - memjrnlTruncate(pJfd, 0); + MemJournal *p = (MemJournal *)pJfd; + memjrnlFreeChunks(p); return SQLITE_OK; } - /* ** Sync the file. ** -** Syncing an in-memory journal is a no-op. And, in fact, this routine -** is never called in a working implementation. This implementation -** exists purely as a contingency, in case some malfunction in some other -** part of SQLite causes Sync to be called by mistake. +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. */ -static int memjrnlSync(sqlite3_file *NotUsed, int NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); +static int memjrnlSync(sqlite3_file *pJfd, int flags){ + UNUSED_PARAMETER2(pJfd, flags); return SQLITE_OK; } @@ -230,26 +333,86 @@ static const struct sqlite3_io_methods MemJournalMethods = { }; /* -** Open a journal file. +** Open a journal file. +** +** The behaviour of the journal file depends on the value of parameter +** nSpill. If nSpill is 0, then the journal file is always create and +** accessed using the underlying VFS. If nSpill is less than zero, then +** all content is always stored in main-memory. Finally, if nSpill is a +** positive value, then the journal file is initially created in-memory +** but may be flushed to disk later on. In this case the journal file is +** flushed to disk either when it grows larger than nSpill bytes in size, +** or when sqlite3JournalCreate() is called. */ -void sqlite3MemJournalOpen(sqlite3_file *pJfd){ - MemJournal *p = (MemJournal *)pJfd; - assert( EIGHT_BYTE_ALIGNMENT(p) ); - memset(p, 0, sqlite3MemJournalSize()); - p->pMethod = (sqlite3_io_methods*)&MemJournalMethods; +int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nSpill /* Bytes buffered before opening the file */ +){ + MemJournal *p = (MemJournal*)pJfd; + + /* Zero the file-handle object. If nSpill was passed zero, initialize + ** it using the sqlite3OsOpen() function of the underlying VFS. In this + ** case none of the code in this module is executed as a result of calls + ** made on the journal file-handle. */ + memset(p, 0, sizeof(MemJournal)); + if( nSpill==0 ){ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + + if( nSpill>0 ){ + p->nChunkSize = nSpill; + }else{ + p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); + assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); + } + + p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; + p->nSpill = nSpill; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; } /* -** Return true if the file-handle passed as an argument is -** an in-memory journal +** Open an in-memory journal file. */ -int sqlite3IsMemJournal(sqlite3_file *pJfd){ - return pJfd->pMethods==&MemJournalMethods; +void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + sqlite3JournalOpen(0, 0, pJfd, 0, -1); +} + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE +/* +** If the argument p points to a MemJournal structure that is not an +** in-memory-only journal file (i.e. is one that was opened with a +ve +** nSpill parameter), and the underlying file has not yet been created, +** create it now. +*/ +int sqlite3JournalCreate(sqlite3_file *p){ + int rc = SQLITE_OK; + if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){ + rc = memjrnlCreateFile((MemJournal*)p); + } + return rc; +} +#endif + +/* +** The file-handle passed as the only argument is open on a journal file. +** Return true if this "journal file" is currently stored in heap memory, +** or false otherwise. +*/ +int sqlite3JournalIsInMemory(sqlite3_file *p){ + return p->pMethods==&MemJournalMethods; } /* -** Return the number of bytes required to store a MemJournal file descriptor. +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. */ -int sqlite3MemJournalSize(void){ - return sizeof(MemJournal); +int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); } diff --git a/src/msvc.h b/src/msvc.h index 01ebf2b4..3914b05b 100644 --- a/src/msvc.h +++ b/src/msvc.h @@ -12,8 +12,8 @@ ** ** This file contains code that is specific to MSVC. */ -#ifndef _MSVC_H_ -#define _MSVC_H_ +#ifndef SQLITE_MSVC_H +#define SQLITE_MSVC_H #if defined(_MSC_VER) #pragma warning(disable : 4054) @@ -33,4 +33,4 @@ #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ -#endif /* _MSVC_H_ */ +#endif /* SQLITE_MSVC_H */ diff --git a/src/os.c b/src/os.c index 90130d0e..5cf00147 100644 --- a/src/os.c +++ b/src/os.c @@ -13,9 +13,7 @@ ** This file contains OS interface code that is common to all ** architectures. */ -#define _SQLITE_OS_C_ 1 #include "sqliteInt.h" -#undef _SQLITE_OS_C_ /* ** If we compile with the SQLITE_TEST macro set, then the following block @@ -66,9 +64,9 @@ int sqlite3_open_file_count = 0; #if defined(SQLITE_TEST) int sqlite3_memdebug_vfs_oom_test = 1; #define DO_OS_MALLOC_TEST(x) \ - if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3IsMemJournal(x))) { \ + if (sqlite3_memdebug_vfs_oom_test && (!x || !sqlite3JournalIsInMemory(x))) { \ void *pTstAlloc = sqlite3Malloc(10); \ - if (!pTstAlloc) return SQLITE_IOERR_NOMEM; \ + if (!pTstAlloc) return SQLITE_IOERR_NOMEM_BKPT; \ sqlite3_free(pTstAlloc); \ } #else @@ -81,13 +79,11 @@ int sqlite3_memdebug_vfs_oom_test = 1; ** of this would be completely automatic if SQLite were coded using ** C++ instead of plain old C. */ -int sqlite3OsClose(sqlite3_file *pId){ - int rc = SQLITE_OK; +void sqlite3OsClose(sqlite3_file *pId){ if( pId->pMethods ){ - rc = pId->pMethods->xClose(pId); + pId->pMethods->xClose(pId); pId->pMethods = 0; } - return rc; } int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ DO_OS_MALLOC_TEST(id); @@ -262,6 +258,9 @@ int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } +int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() @@ -287,7 +286,7 @@ int sqlite3OsOpenMalloc( int flags, int *pOutFlags ){ - int rc = SQLITE_NOMEM; + int rc; sqlite3_file *pFile; pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); if( pFile ){ @@ -297,15 +296,15 @@ int sqlite3OsOpenMalloc( }else{ *ppFile = pFile; } + }else{ + rc = SQLITE_NOMEM_BKPT; } return rc; } -int sqlite3OsCloseFree(sqlite3_file *pFile){ - int rc = SQLITE_OK; +void sqlite3OsCloseFree(sqlite3_file *pFile){ assert( pFile ); - rc = sqlite3OsClose(pFile); + sqlite3OsClose(pFile); sqlite3_free(pFile); - return rc; } /* @@ -316,7 +315,7 @@ int sqlite3OsCloseFree(sqlite3_file *pFile){ */ int sqlite3OsInit(void){ void *p = sqlite3_malloc(10); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; sqlite3_free(p); return sqlite3_os_init(); } diff --git a/src/os.h b/src/os.h index 2c1b86f9..947f88b3 100644 --- a/src/os.h +++ b/src/os.h @@ -160,7 +160,7 @@ int sqlite3OsInit(void); /* ** Functions for accessing sqlite3_file methods */ -int sqlite3OsClose(sqlite3_file*); +void sqlite3OsClose(sqlite3_file*); int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); int sqlite3OsTruncate(sqlite3_file*, i64 size); @@ -197,6 +197,7 @@ void sqlite3OsDlClose(sqlite3_vfs *, void *); #endif /* SQLITE_OMIT_LOAD_EXTENSION */ int sqlite3OsRandomness(sqlite3_vfs *, int, char *); int sqlite3OsSleep(sqlite3_vfs *, int); +int sqlite3OsGetLastError(sqlite3_vfs*); int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* @@ -204,6 +205,6 @@ int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); ** sqlite3_malloc() to obtain space for the file-handle structure. */ int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -int sqlite3OsCloseFree(sqlite3_file *); +void sqlite3OsCloseFree(sqlite3_file *); #endif /* _SQLITE_OS_H_ */ diff --git a/src/os_setup.h b/src/os_setup.h index 68de1446..08aaa119 100644 --- a/src/os_setup.h +++ b/src/os_setup.h @@ -13,8 +13,8 @@ ** This file contains pre-processor directives related to operating system ** detection and/or setup. */ -#ifndef _OS_SETUP_H_ -#define _OS_SETUP_H_ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H /* ** Figure out if we are dealing with Unix, Windows, or some other operating @@ -54,4 +54,4 @@ # endif #endif -#endif /* _OS_SETUP_H_ */ +#endif /* SQLITE_OS_SETUP_H */ diff --git a/src/os_unix.c b/src/os_unix.c index fe1fc6af..08d4cb5d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -71,6 +71,19 @@ # endif #endif +/* Use pread() and pwrite() if they are available */ +#if defined(__APPLE__) +# define HAVE_PREAD 1 +# define HAVE_PWRITE 1 +#endif +#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64) +# undef USE_PREAD +# define USE_PREAD64 1 +#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE) +# undef USE_PREAD64 +# define USE_PREAD 1 +#endif + /* ** standard include files. */ @@ -392,7 +405,7 @@ static struct unix_syscall { #else { "pread64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) +#define osPread64 ((ssize_t(*)(int,void*,size_t,off64_t))aSyscall[10].pCurrent) { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) @@ -410,7 +423,7 @@ static struct unix_syscall { #else { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ aSyscall[13].pCurrent) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, @@ -1302,7 +1315,7 @@ static int findInodeInfo( if( pInode==0 ){ pInode = sqlite3_malloc64( sizeof(*pInode) ); if( pInode==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); @@ -1344,12 +1357,16 @@ static int fileHasMoved(unixFile *pFile){ static void verifyDbFile(unixFile *pFile){ struct stat buf; int rc; + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + rc = osFstat(pFile->h, &buf); if( rc!=0 ){ sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); return; } - if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + if( buf.st_nlink==0 ){ sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); return; } @@ -1485,7 +1502,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** lock transitions in terms of the POSIX advisory shared and exclusive ** lock primitives (called read-locks and write-locks below, to avoid ** confusion with SQLite lock names). The algorithms are complicated - ** slightly in order to be compatible with windows systems simultaneously + ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved @@ -1493,8 +1510,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range', a range of 510 bytes at a well known offset. ** ** To obtain a SHARED lock, a read-lock is obtained on the 'pending - ** byte'. If this is successful, a random byte from the 'shared byte - ** range' is read-locked and the lock on the 'pending byte' released. + ** byte'. If this is successful, 'shared byte range' is read-locked + ** and the lock on the 'pending byte' released. (Legacy note: When + ** SQLite was first developed, Windows95 systems were still very common, + ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** single randomly selected by from the 'shared byte range' is locked. + ** Windows95 is now pretty much extinct, but this work-around for the + ** lack of shared-locks on Windows95 lives on, for backwards + ** compatibility.) ** ** A process may only obtain a RESERVED lock after it has a SHARED lock. ** A RESERVED lock is implemented by grabbing a write-lock on the @@ -1513,11 +1536,6 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range'. Since all other locks require a read-lock on one of the bytes ** within this range, this ensures that no other locks are held on the ** database. - ** - ** The reason a single byte cannot be used instead of the 'shared byte - ** range' is that some versions of windows do not support read-locks. By - ** locking a random byte from a range, concurrent SHARED locks may exist - ** even if the locking primitive used is always a write-lock. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -4222,7 +4240,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Allocate space for the new unixShm object. */ p = sqlite3_malloc64( sizeof(*p) ); - if( p==0 ) return SQLITE_NOMEM; + if( p==0 ) return SQLITE_NOMEM_BKPT; memset(p, 0, sizeof(*p)); assert( pDbFd->pShm==0 ); @@ -4254,7 +4272,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ #endif pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); @@ -4270,10 +4288,12 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } } if( pInode->bProcessLock==0 ){ @@ -4445,7 +4465,7 @@ static int unixShmMap( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->apRegion = apNew; @@ -4465,7 +4485,7 @@ static int unixShmMap( }else{ pMem = sqlite3_malloc64(szRegion); if( pMem==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto shmpage_out; } memset(pMem, 0, szRegion); @@ -5243,7 +5263,7 @@ static int fillInUnixFile( pNew->pId = vxworksFindFileId(zFilename); if( pNew->pId==0 ){ ctrlFlags |= UNIXFILE_NOLOCK; - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } #endif @@ -5299,7 +5319,7 @@ static int fillInUnixFile( afpLockingContext *pCtx; pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ /* NB: zFilename exists and remains valid until the file is closed ** according to requirement F11141. So we do not need to make a @@ -5329,7 +5349,7 @@ static int fillInUnixFile( nFilename = (int)strlen(zFilename) + 6; zLockFile = (char *)sqlite3_malloc64(nFilename); if( zLockFile==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); } @@ -5352,7 +5372,7 @@ static int fillInUnixFile( if( zSemName[n]=='/' ) zSemName[n] = '_'; pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); if( pNew->pInode->pSem == SEM_FAILED ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; pNew->pInode->aSemName[0] = '\0'; } } @@ -5392,20 +5412,24 @@ static const char *unixTempFileDir(void){ "/tmp", "." }; - unsigned int i; + unsigned int i = 0; struct stat buf; const char *zDir = sqlite3_temp_directory; if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - for(i=0; i=sizeof(azDirs)/sizeof(azDirs[0]) ) break; + zDir = azDirs[i++]; } - return zDir; + return 0; } /* @@ -5421,9 +5445,11 @@ static int unixGetTempname(int nBuf, char *zBuf){ ** using the io-error infrastructure to test that SQLite handles this ** function failing. */ + zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; do{ u64 r; sqlite3_randomness(sizeof(r), &r); @@ -5503,6 +5529,27 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ return pUnused; } +/* +** Find the mode, uid and gid of file zFile. +*/ +static int getFileMode( + const char *zFile, /* File name */ + mode_t *pMode, /* OUT: Permissions of zFile */ + uid_t *pUid, /* OUT: uid of zFile. */ + gid_t *pGid /* OUT: gid of zFile. */ +){ + struct stat sStat; /* Output of stat() on database file */ + int rc = SQLITE_OK; + if( 0==osStat(zFile, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + return rc; +} + /* ** This function is called by unixOpen() to determine the unix permissions ** to create new files with. If no error occurs, then SQLITE_OK is returned @@ -5538,7 +5585,6 @@ static int findCreateFileMode( if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ - struct stat sStat; /* Output of stat() on database file */ /* zPath is a path to a WAL or journal file. The following block derives ** the path to the associated database file from zPath. This block handles @@ -5569,15 +5615,18 @@ static int findCreateFileMode( memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; - if( 0==osStat(zDb, &sStat) ){ - *pMode = sStat.st_mode & 0777; - *pUid = sStat.st_uid; - *pGid = sStat.st_gid; - }else{ - rc = SQLITE_IOERR_FSTAT; - } + rc = getFileMode(zDb, pMode, pUid, pGid); }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; + }else if( flags & SQLITE_OPEN_URI ){ + /* If this is a main database file and the file was opened using a URI + ** filename, check for the "modeof" parameter. If present, interpret + ** its value as a filename and try to copy the mode, uid and gid from + ** that file. */ + const char *z = sqlite3_uri_parameter(zPath, "modeof"); + if( z ){ + rc = getFileMode(z, pMode, pUid, pGid); + } } return rc; } @@ -5693,7 +5742,7 @@ static int unixOpen( }else{ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } p->pUnused = pUnused; @@ -5779,7 +5828,7 @@ static int unixOpen( zPath = sqlite3_mprintf("%s", zName); if( zPath==0 ){ robust_close(p, fd, __LINE__); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } #else osUnlink(zName); @@ -5790,9 +5839,6 @@ static int unixOpen( p->openFlags = openFlags; } #endif - - noLock = eType!=SQLITE_OPEN_MAIN_DB; - #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ @@ -5811,6 +5857,7 @@ static int unixOpen( /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; @@ -6011,7 +6058,7 @@ static int unixFullPathname( if( bLink ){ if( zDel==0 ){ zDel = sqlite3_malloc(nOut); - if( zDel==0 ) rc = SQLITE_NOMEM; + if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ rc = SQLITE_CANTOPEN_BKPT; } @@ -6249,23 +6296,18 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ # define unixCurrentTime 0 #endif -#ifndef SQLITE_OMIT_DEPRECATED /* -** We added the xGetLastError() method with the intention of providing -** better low-level error messages when operating-system problems come up -** during SQLite operation. But so far, none of that has been implemented -** in the core. So this routine is never called. For now, it is merely -** a place-holder. +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. */ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ UNUSED_PARAMETER(NotUsed); UNUSED_PARAMETER(NotUsed2); UNUSED_PARAMETER(NotUsed3); - return 0; + return errno; } -#else -# define unixGetLastError 0 -#endif /* @@ -6555,7 +6597,7 @@ static int proxyCreateUnixFile( }else{ pUnused = sqlite3_malloc64(sizeof(*pUnused)); if( !pUnused ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } if( fd<0 ){ @@ -6588,7 +6630,7 @@ static int proxyCreateUnixFile( pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto end_create_proxy; } memset(pNew, 0, sizeof(unixFile)); @@ -7001,7 +7043,7 @@ static int proxyTakeConch(unixFile *pFile){ if( tempLockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); if( !pCtx->lockProxyPath ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } } @@ -7066,7 +7108,7 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ ** the name of the original database file. */ *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memcpy(conchPath, dbPath, len+1); @@ -7182,7 +7224,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx = sqlite3_malloc64( sizeof(*pCtx) ); if( pCtx==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(pCtx, 0, sizeof(*pCtx)); @@ -7218,7 +7260,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { if( rc==SQLITE_OK ){ pCtx->dbPath = sqlite3DbStrDup(0, dbPath); if( pCtx->dbPath==NULL ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; } } if( rc==SQLITE_OK ){ diff --git a/src/os_win.c b/src/os_win.c index eda6cf59..2cb5f7b0 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -290,6 +290,17 @@ struct winFile { #endif }; +/* +** The winVfsAppData structure is used for the pAppData member for all of the +** Win32 VFS variants. +*/ +typedef struct winVfsAppData winVfsAppData; +struct winVfsAppData { + const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ + void *pAppData; /* The extra pAppData, if any. */ + BOOL bNoLock; /* Non-zero if locking is disabled. */ +}; + /* ** Allowed values for winFile.ctrlFlags */ @@ -344,11 +355,23 @@ struct winFile { # define SQLITE_WIN32_HEAP_CREATE (TRUE) #endif +/* + * This is cache size used in the calculation of the initial size of the + * Win32-specific heap. It cannot be negative. + */ +#ifndef SQLITE_WIN32_CACHE_SIZE +# if SQLITE_DEFAULT_CACHE_SIZE>=0 +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# else +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# endif +#endif + /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) #endif @@ -1222,7 +1245,7 @@ int sqlite3_win32_compact_heap(LPUINT pnLargest){ if( lastErrno==NO_ERROR ){ sqlite3_log(SQLITE_NOMEM, "failed to HeapCompact (no space), heap=%p", (void*)hHeap); - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ sqlite3_log(SQLITE_ERROR, "failed to HeapCompact (%lu), heap=%p", osGetLastError(), (void*)hHeap); @@ -1248,8 +1271,8 @@ int sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); @@ -1294,6 +1317,12 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ assert( nMin==-1 || nMin==0 || nMin0 ){ memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); @@ -1542,7 +1571,7 @@ static int winMemInit(void *pAppData){ "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, dwMaximumSize); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pWinMemData->bOwned = TRUE; assert( pWinMemData->bOwned ); @@ -1552,7 +1581,7 @@ static int winMemInit(void *pAppData){ if( !pWinMemData->hHeap ){ sqlite3_log(SQLITE_NOMEM, "failed to GetProcessHeap (%lu)", osGetLastError()); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pWinMemData->bOwned = FALSE; assert( !pWinMemData->bOwned ); @@ -1619,147 +1648,244 @@ void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). +** Convert a UTF-8 string to Microsoft Unicode. ** -** Space to hold the returned string is obtained from malloc. +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static LPWSTR winUtf8ToUnicode(const char *zFilename){ +static LPWSTR winUtf8ToUnicode(const char *zText){ int nChar; - LPWSTR zWideFilename; + LPWSTR zWideText; - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); if( nChar==0 ){ return 0; } - zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); - if( zWideFilename==0 ){ + zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) ); + if( zWideText==0 ){ return 0; } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); if( nChar==0 ){ - sqlite3_free(zWideFilename); - zWideFilename = 0; + sqlite3_free(zWideText); + zWideText = 0; } - return zWideFilename; + return zWideText; } /* -** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is -** obtained from sqlite3_malloc(). +** Convert a Microsoft Unicode string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ +static char *winUnicodeToUtf8(LPCWSTR zWideText){ int nByte; - char *zFilename; + char *zText; - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zFilename = sqlite3MallocZero( nByte ); - if( zFilename==0 ){ + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ return 0; } - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zFilename); - zFilename = 0; + sqlite3_free(zText); + zText = 0; } - return zFilename; + return zText; } /* -** Convert an ANSI string to Microsoft Unicode, based on the -** current codepage settings for file apis. +** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM +** code page. ** -** Space to hold the returned string is obtained -** from sqlite3_malloc. +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static LPWSTR winMbcsToUnicode(const char *zFilename){ +static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ int nByte; - LPWSTR zMbcsFilename; - int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + LPWSTR zMbcsText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL, + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, 0)*sizeof(WCHAR); if( nByte==0 ){ return 0; } - zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); - if( zMbcsFilename==0 ){ + zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + if( zMbcsText==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, nByte); if( nByte==0 ){ - sqlite3_free(zMbcsFilename); - zMbcsFilename = 0; + sqlite3_free(zMbcsText); + zMbcsText = 0; } - return zMbcsFilename; + return zMbcsText; } /* -** Convert Microsoft Unicode to multi-byte character string, based on the -** user's ANSI codepage. +** Convert a Microsoft Unicode string to a multi-byte character string, +** using the ANSI or OEM code page. ** -** Space to hold the returned string is obtained from -** sqlite3_malloc(). +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ +static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ int nByte; - char *zFilename; - int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + char *zText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zFilename = sqlite3MallocZero( nByte ); - if( zFilename==0 ){ + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ return 0; } - nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zFilename); - zFilename = 0; + sqlite3_free(zText); + zText = 0; } - return zFilename; + return zText; } /* -** Convert multibyte character string to UTF-8. Space to hold the -** returned string is obtained from sqlite3_malloc(). +** Convert a multi-byte character string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ - char *zFilenameUtf8; +static char *winMbcsToUtf8(const char *zText, int useAnsi){ + char *zTextUtf8; LPWSTR zTmpWide; - zTmpWide = winMbcsToUnicode(zFilename); + zTmpWide = winMbcsToUnicode(zText, useAnsi); if( zTmpWide==0 ){ return 0; } - zFilenameUtf8 = winUnicodeToUtf8(zTmpWide); + zTextUtf8 = winUnicodeToUtf8(zTmpWide); sqlite3_free(zTmpWide); - return zFilenameUtf8; + return zTextUtf8; } /* -** Convert UTF-8 to multibyte character string. Space to hold the -** returned string is obtained from sqlite3_malloc(). +** Convert a UTF-8 string to a multi-byte character string. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ - char *zFilenameMbcs; +static char *winUtf8ToMbcs(const char *zText, int useAnsi){ + char *zTextMbcs; LPWSTR zTmpWide; - zTmpWide = winUtf8ToUnicode(zFilename); + zTmpWide = winUtf8ToUnicode(zText); if( zTmpWide==0 ){ return 0; } - zFilenameMbcs = winUnicodeToMbcs(zTmpWide); + zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi); sqlite3_free(zTmpWide); - return zFilenameMbcs; + return zTextMbcs; +} + +/* +** This is a public wrapper for the winUtf8ToUnicode() function. +*/ +LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToUnicode(zText); +} + +/* +** This is a public wrapper for the winUnicodeToUtf8() function. +*/ +char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zWideText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUnicodeToUtf8(zWideText); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +char *sqlite3_win32_mbcs_to_utf8(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, useAnsi); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +char *sqlite3_win32_utf8_to_mbcs(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, useAnsi); } /* @@ -1789,7 +1915,7 @@ int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ if( zValue && zValue[0] ){ zValueUtf8 = winUnicodeToUtf8(zValue); if ( zValueUtf8==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } sqlite3_free(*ppDirectory); @@ -1861,7 +1987,7 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); sqlite3EndBenignMalloc(); /* free the system buffer allocated by FormatMessage */ osLocalFree(zTemp); @@ -2003,16 +2129,17 @@ static void winLogIoerr(int nRetry, int lineno){ } } -#if SQLITE_OS_WINCE -/************************************************************************* -** This section contains code for WinCE only. -*/ -#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** The MSVC CRT on Windows CE may not have a localtime() function. So -** create a substitute. +** This #if does not rely on the SQLITE_OS_WINCE define because the +** corresponding section in "date.c" cannot use it. */ -#include +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So define a substitute. +*/ +# include struct tm *__cdecl localtime(const time_t *t) { static struct tm y; @@ -2036,6 +2163,10 @@ struct tm *__cdecl localtime(const time_t *t) } #endif +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] /* @@ -2066,7 +2197,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ zName = winUtf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } /* Initialize the local lockdata */ @@ -2491,7 +2622,12 @@ static int winClose(sqlite3_file *id){ }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); #if SQLITE_OS_WINCE #define WINCE_DELETION_ATTEMPTS 3 - winceDestroyLock(pFile); + { + winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; + if( pAppData==NULL || !pAppData->bNoLock ){ + winceDestroyLock(pFile); + } + } if( pFile->zDeleteOnClose ){ int cnt = 0; while( @@ -3049,9 +3185,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( (pFile->locktype==NO_LOCK) - || ( (locktype==EXCLUSIVE_LOCK) - && (pFile->locktype==RESERVED_LOCK)) + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) ){ int cnt = 3; while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, @@ -3224,6 +3359,44 @@ static int winUnlock(sqlite3_file *id, int locktype){ return rc; } +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int winNolockLock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(pResOut); + return SQLITE_OK; +} + +static int winNolockUnlock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + /* ** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. @@ -3257,7 +3430,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { + case SQLITE_FCNTL_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -3315,6 +3488,12 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } + case SQLITE_FCNTL_WIN32_GET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + *phFile = pFile->h; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } #ifdef SQLITE_TEST case SQLITE_FCNTL_WIN32_SET_HANDLE: { LPHANDLE phFile = (LPHANDLE)pArg; @@ -3502,12 +3681,12 @@ struct winShm { /* ** Apply advisory locks for all n bytes beginning at ofst. */ -#define _SHM_UNLCK 1 -#define _SHM_RDLCK 2 -#define _SHM_WRLCK 3 +#define WINSHM_UNLCK 1 +#define WINSHM_RDLCK 2 +#define WINSHM_WRLCK 3 static int winShmSystemLock( winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ + int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ int ofst, /* Offset to first byte to be locked/unlocked */ int nByte /* Number of bytes to lock or unlock */ ){ @@ -3520,12 +3699,12 @@ static int winShmSystemLock( pFile->hFile.h, lockType, ofst, nByte)); /* Release/Acquire the system-level lock */ - if( lockType==_SHM_UNLCK ){ + if( lockType==WINSHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); }else{ /* Initialize the locking parameters */ DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } @@ -3537,7 +3716,7 @@ static int winShmSystemLock( } OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; @@ -3615,12 +3794,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); - if( p==0 ) return SQLITE_IOERR_NOMEM; + if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; nName = sqlite3Strlen30(pDbFd->zPath); pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); if( pNew==0 ){ sqlite3_free(p); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); @@ -3645,10 +3824,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } } rc = winOpen(pDbFd->pVfs, @@ -3663,7 +3844,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. */ - if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ + if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), @@ -3671,8 +3852,8 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } if( rc ) goto shm_open_err; } @@ -3701,7 +3882,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Jump here on any error */ shm_open_err: - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ sqlite3_free(p); sqlite3_free(pNew); @@ -3790,7 +3971,7 @@ static int winShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -3818,7 +3999,7 @@ static int winShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -3843,7 +4024,7 @@ static int winShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -3952,7 +4133,7 @@ static int winShmMap( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; + rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->aRegion = apNew; @@ -4286,6 +4467,44 @@ static const sqlite3_io_methods winIoMethod = { winUnfetch /* xUnfetch */ }; +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32 without performing any locking. +*/ +static const sqlite3_io_methods winIoNolockMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winNolockLock, /* xLock */ + winNolockUnlock, /* xUnlock */ + winNolockCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +static winVfsAppData winAppData = { + &winIoMethod, /* pMethod */ + 0, /* pAppData */ + 0 /* bNoLock */ +}; + +static winVfsAppData winNolockAppData = { + &winIoNolockMethod, /* pMethod */ + 0, /* pAppData */ + 1 /* bNoLock */ +}; + /**************************************************************************** **************************** sqlite3_vfs methods **************************** ** @@ -4306,7 +4525,7 @@ static char *winConvertToUtf8Filename(const void *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = sqlite3_win32_mbcs_to_utf8(zFilename); + zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); } #endif /* caller will handle out of memory */ @@ -4327,7 +4546,7 @@ static void *winConvertFromUtf8Filename(const char *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); } #endif /* caller will handle out of memory */ @@ -4382,7 +4601,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } /* Figure out the effective temporary directory. First, check if one @@ -4440,7 +4659,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( winIsDir(zConverted) ){ sqlite3_snprintf(nMax, zBuf, "%s", zDir); @@ -4453,7 +4672,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zConverted ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A, zDir, @@ -4474,7 +4693,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zConverted); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); @@ -4492,7 +4711,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zWidePath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osGetTempPathW(nMax, zWidePath)==0 ){ sqlite3_free(zWidePath); @@ -4510,7 +4729,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ sqlite3_free(zWidePath); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -4520,7 +4739,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ if( !zMbcsPath ){ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osGetTempPathA(nMax, zMbcsPath)==0 ){ sqlite3_free(zBuf); @@ -4528,14 +4747,14 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ return winLogError(SQLITE_IOERR_GETTEMPPATH, osGetLastError(), "winGetTempname3", 0); } - zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); + zUtf8 = winMbcsToUtf8(zMbcsPath, osAreFileApisANSI()); if( zUtf8 ){ sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); }else{ sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } } #endif /* SQLITE_WIN32_HAS_ANSI */ @@ -4618,7 +4837,7 @@ static int winIsDir(const void *zConverted){ ** Open a file. */ static int winOpen( - sqlite3_vfs *pVfs, /* Used to get maximum path name length */ + sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ const char *zName, /* Name of the file (UTF-8) */ sqlite3_file *id, /* Write the SQLite file handle here */ int flags, /* Open mode flags */ @@ -4633,6 +4852,7 @@ static int winOpen( #if SQLITE_OS_WINCE int isTemp = 0; #endif + winVfsAppData *pAppData; winFile *pFile = (winFile*)id; void *zConverted; /* Filename in OS encoding */ const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ @@ -4727,7 +4947,7 @@ static int winOpen( if( zConverted==0 ){ sqlite3_free(zTmpname); OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( winIsDir(zConverted) ){ @@ -4854,15 +5074,20 @@ static int winOpen( "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + pAppData = (winVfsAppData*)pVfs->pAppData; + #if SQLITE_OS_WINCE - if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK - ){ - osCloseHandle(h); - sqlite3_free(zConverted); - sqlite3_free(zTmpname); - OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); - return rc; + { + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && ((pAppData==NULL) || !pAppData->bNoLock) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -4873,7 +5098,7 @@ static int winOpen( } sqlite3_free(zTmpname); - pFile->pMethod = &winIoMethod; + pFile->pMethod = pAppData ? pAppData->pMethod : &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; if( isReadonly ){ @@ -4927,7 +5152,7 @@ static int winDelete( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ do { @@ -5035,7 +5260,7 @@ static int winAccess( zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ int cnt = 0; @@ -5148,6 +5373,18 @@ static int winFullPathname( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; +#endif + + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ + zRelative++; + } #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); @@ -5162,7 +5399,7 @@ static int winFullPathname( */ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | @@ -5174,7 +5411,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", sqlite3_data_directory, winGetDirSep(), zUtf8); @@ -5184,7 +5421,7 @@ static int winFullPathname( }else{ char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); if( !zOut ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( cygwin_conv_path( (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), @@ -5196,7 +5433,7 @@ static int winFullPathname( char *zUtf8 = winConvertToUtf8Filename(zOut); if( !zUtf8 ){ sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); sqlite3_free(zUtf8); @@ -5226,17 +5463,6 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; - void *zConverted; - char *zOut; - - /* If this path name begins with "/X:", where "X" is any alphabetic - ** character, discard the initial "/" from the pathname. - */ - if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ - zRelative++; - } - /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -5256,7 +5482,7 @@ static int winFullPathname( } zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } if( osIsNT() ){ LPWSTR zTemp; @@ -5270,7 +5496,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -5296,7 +5522,7 @@ static int winFullPathname( zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); if( nByte==0 ){ @@ -5306,7 +5532,7 @@ static int winFullPathname( "winFullPathname4", zRelative); } sqlite3_free(zConverted); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); sqlite3_free(zTemp); } #endif @@ -5315,7 +5541,7 @@ static int winFullPathname( sqlite3_free(zOut); return SQLITE_OK; }else{ - return SQLITE_IOERR_NOMEM; + return SQLITE_IOERR_NOMEM_BKPT; } #endif } @@ -5390,65 +5616,85 @@ static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ #define winDlClose 0 #endif +/* State information for the randomness gatherer. */ +typedef struct EntropyGatherer EntropyGatherer; +struct EntropyGatherer { + unsigned char *a; /* Gather entropy into this buffer */ + int na; /* Size of a[] in bytes */ + int i; /* XOR next input into a[i] */ + int nXor; /* Number of XOR operations done */ +}; + +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) +/* Mix sz bytes of entropy into p. */ +static void xorMemory(EntropyGatherer *p, unsigned char *x, int sz){ + int j, k; + for(j=0, k=p->i; ja[k++] ^= x[j]; + if( k>=p->na ) k = 0; + } + p->i = k; + p->nXor += sz; +} +#endif /* !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) */ /* ** Write up to nBuf bytes of randomness into zBuf. */ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ - int n = 0; - UNUSED_PARAMETER(pVfs); #if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) - n = nBuf; + UNUSED_PARAMETER(pVfs); memset(zBuf, 0, nBuf); + return nBuf; #else - if( sizeof(SYSTEMTIME)<=nBuf-n ){ + EntropyGatherer e; + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); +#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE + rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */ +#endif /* defined(_MSC_VER) && _MSC_VER>=1400 */ + e.a = (unsigned char*)zBuf; + e.na = nBuf; + e.nXor = 0; + e.i = 0; + { SYSTEMTIME x; osGetSystemTime(&x); - memcpy(&zBuf[n], &x, sizeof(x)); - n += sizeof(x); + xorMemory(&e, (unsigned char*)&x, sizeof(SYSTEMTIME)); } - if( sizeof(DWORD)<=nBuf-n ){ + { DWORD pid = osGetCurrentProcessId(); - memcpy(&zBuf[n], &pid, sizeof(pid)); - n += sizeof(pid); + xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } #if SQLITE_OS_WINRT - if( sizeof(ULONGLONG)<=nBuf-n ){ + { ULONGLONG cnt = osGetTickCount64(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); } #else - if( sizeof(DWORD)<=nBuf-n ){ + { DWORD cnt = osGetTickCount(); - memcpy(&zBuf[n], &cnt, sizeof(cnt)); - n += sizeof(cnt); + xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } -#endif - if( sizeof(LARGE_INTEGER)<=nBuf-n ){ +#endif /* SQLITE_OS_WINRT */ + { LARGE_INTEGER i; osQueryPerformanceCounter(&i); - memcpy(&zBuf[n], &i, sizeof(i)); - n += sizeof(i); + xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID - if( sizeof(UUID)<=nBuf-n ){ + { UUID id; memset(&id, 0, sizeof(UUID)); osUuidCreate(&id); - memcpy(&zBuf[n], &id, sizeof(UUID)); - n += sizeof(UUID); - } - if( sizeof(UUID)<=nBuf-n ){ - UUID id; + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); memset(&id, 0, sizeof(UUID)); osUuidCreateSequential(&id); - memcpy(&zBuf[n], &id, sizeof(UUID)); - n += sizeof(UUID); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); } -#endif -#endif /* defined(SQLITE_TEST) || defined(SQLITE_ZERO_PRNG_SEED) */ - return n; +#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ + return e.nXor>nBuf ? nBuf : e.nXor; +#endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -5564,8 +5810,10 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ ** sqlite3_errmsg(), possibly making IO errors easier to debug. */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); UNUSED_PARAMETER(pVfs); - return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; } /* @@ -5573,53 +5821,103 @@ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ */ int sqlite3_os_init(void){ static sqlite3_vfs winVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #if defined(SQLITE_WIN32_HAS_WIDE) static sqlite3_vfs winLongPathVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-longpath", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32-longpath", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + static sqlite3_vfs winNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #endif @@ -5643,6 +5941,12 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif + sqlite3_vfs_register(&winNolockVfs, 0); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathNolockVfs, 0); +#endif + return SQLITE_OK; } diff --git a/src/os_win.h b/src/os_win.h index 17d6a2be..27714ed0 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -12,8 +12,8 @@ ** ** This file contains code that is specific to Windows. */ -#ifndef _OS_WIN_H_ -#define _OS_WIN_H_ +#ifndef SQLITE_OS_WIN_H +#define SQLITE_OS_WIN_H /* ** Include the primary Windows SDK header file. @@ -85,4 +85,4 @@ # define SQLITE_OS_WIN_THREADS 0 #endif -#endif /* _OS_WIN_H_ */ +#endif /* SQLITE_OS_WIN_H */ diff --git a/src/pager.c b/src/pager.c index 5c61968e..6f9a1819 100644 --- a/src/pager.c +++ b/src/pager.c @@ -428,19 +428,6 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define MAX_SECTOR_SIZE 0x10000 -/* -** If the option SQLITE_EXTRA_DURABLE option is set at compile-time, then -** SQLite will do extra fsync() operations when synchronous==FULL to help -** ensure that transactions are durable across a power failure. Most -** applications are happy as long as transactions are consistent across -** a power failure, and are perfectly willing to lose the last transaction -** in exchange for the extra performance of avoiding directory syncs. -** And so the default SQLITE_EXTRA_DURABLE setting is off. -*/ -#ifndef SQLITE_EXTRA_DURABLE -# define SQLITE_EXTRA_DURABLE 0 -#endif - /* ** An instance of the following structure is allocated for each active @@ -830,9 +817,10 @@ static const unsigned char aJournalMagic[] = { ** rollback journal. Otherwise false. */ #ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ +int sqlite3PagerUseWal(Pager *pPager){ return (pPager->pWal!=0); } +# define pagerUseWal(x) sqlite3PagerUseWal(x) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 @@ -885,6 +873,7 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ + assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -971,7 +960,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); break; } @@ -1183,6 +1172,8 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#else +# define jrnlBufferSize(x) 0 #endif /* @@ -1343,6 +1334,7 @@ static i64 journalHdrOffset(Pager *pPager){ static int zeroJournalHdr(Pager *pPager, int doTruncate){ int rc = SQLITE_OK; /* Return code */ assert( isOpen(pPager->jfd) ); + assert( !sqlite3JournalIsInMemory(pPager->jfd) ); if( pPager->journalOff ){ const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ @@ -1724,7 +1716,7 @@ static void releaseAllSavepoints(Pager *pPager){ for(ii=0; iinSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } - if( !pPager->exclusiveMode || sqlite3IsMemJournal(pPager->sjfd) ){ + if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ sqlite3OsClose(pPager->sjfd); } sqlite3_free(pPager->aSavepoint); @@ -1830,13 +1822,17 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ - assert( !MEMDB ); - pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; } pPager->journalOff = 0; @@ -1880,6 +1876,29 @@ static int pager_error(Pager *pPager, int rc){ static int pager_truncate(Pager *pPager, Pgno nPage); +/* +** The write transaction open on pPager is being committed (bCommit==1) +** or rolled back (bCommit==0). +** +** Return TRUE if and only if all dirty pages should be flushed to disk. +** +** Rules: +** +** * For non-TEMP databases, always sync to disk. This is necessary +** for transactions to be durable. +** +** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing +** file has been created already (via a spill on pagerStress()) and +** when the number of dirty pages in memory exceeds 25% of the total +** cache size. +*/ +static int pagerFlushOnCommit(Pager *pPager, int bCommit){ + if( pPager->tempFile==0 ) return 1; + if( !bCommit ) return 0; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1962,8 +1981,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ - if( sqlite3IsMemJournal(pPager->jfd) ){ - assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); + if( sqlite3JournalIsInMemory(pPager->jfd) ){ + /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ sqlite3OsClose(pPager->jfd); }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ if( pPager->journalOff==0 ){ @@ -1983,15 +2002,16 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ - rc = zeroJournalHdr(pPager, hasMaster); + rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. + ** the database file, it will do so using an in-memory journal. */ - int bDelete = (!pPager->tempFile && sqlite3JournalExists(pPager->jfd)); + int bDelete = !pPager->tempFile; + assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_WAL @@ -2017,8 +2037,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + if( rc==SQLITE_OK ){ + if( pagerFlushOnCommit(pPager, bCommit) ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + } if( pagerUseWal(pPager) ){ /* Drop the WAL write-lock, if any. Also, if the connection was in @@ -2302,7 +2328,7 @@ static int pager_playback_one_page( pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -2324,9 +2350,9 @@ static int pager_playback_one_page( pPager->dbFileSize = pgno; } if( pPager->pBackup ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM); + CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData); + CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); } }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to @@ -2352,7 +2378,6 @@ static int pager_playback_one_page( assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; if( rc!=SQLITE_OK ) return rc; - pPg->flags &= ~PGHDR_NEED_READ; sqlite3PcacheMakeDirty(pPg); } if( pPg ){ @@ -2366,29 +2391,10 @@ static int pager_playback_one_page( pData = pPg->pData; memcpy(pData, (u8*)aData, pPager->pageSize); pPager->xReiniter(pPg); - if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ - /* If the contents of this page were just restored from the main - ** journal file, then its content must be as they were when the - ** transaction was first opened. In this case we can mark the page - ** as clean, since there will be no need to write it out to the - ** database. - ** - ** There is one exception to this rule. If the page is being rolled - ** back as part of a savepoint (or statement) rollback from an - ** unsynced portion of the main journal file, then it is not safe - ** to mark the page as clean. This is because marking the page as - ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is - ** already in the journal file (recorded in Pager.pInJournal) and - ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to - ** again within this transaction, it will be marked as dirty but - ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially - ** be written out into the database file before its journal file - ** segment is synced. If a crash occurs during or following this, - ** database corruption may ensue. - */ - assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); - } + /* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But + ** that call was dangerous and had no detectable benefit since the cache + ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so + ** has been removed. */ pager_set_pagehash(pPg); /* If this was page 1, then restore the value of Pager.dbFileVers. @@ -2398,7 +2404,7 @@ static int pager_playback_one_page( } /* Decode the page just read from disk */ - CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM); + CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3PcacheRelease(pPg); } return rc; @@ -2464,7 +2470,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); if( !pMaster ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; }else{ const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); @@ -2481,7 +2487,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ nMasterPtr = pVfs->mxPathname+1; zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); if( !zMasterJournal ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto delmaster_out; } zMasterPtr = &zMasterJournal[nMasterJournal+1]; @@ -2729,7 +2735,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** TODO: Technically the following is an error because it assumes that ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, - ** mxPathname is 512, which is the same as the minimum allowable value + ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ zMaster = pPager->pTmpSpace; @@ -2951,7 +2957,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); @@ -3179,6 +3185,8 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); /* If the number of pages in the database is not available from the @@ -3186,14 +3194,11 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** the database file. If the size of the database file is not an ** integer multiple of the page-size, round up the result. */ - if( nPage==0 ){ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -3311,7 +3316,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ if( pSavepoint ){ pDone = sqlite3BitvecCreate(pSavepoint->nOrig); if( !pDone ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } } @@ -3458,7 +3463,7 @@ void sqlite3PagerShrink(Pager *pPager){ ** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness ** of the database to damage due to OS crashes or power failures by ** changing the number of syncs()s when writing the journals. -** There are three levels: +** There are four levels: ** ** OFF sqlite3OsSync() is never called. This is the default ** for temporary and transient files. @@ -3478,6 +3483,10 @@ void sqlite3PagerShrink(Pager *pPager){ ** assurance that the journal will not be corrupted to the ** point of causing damage to the database during rollback. ** +** EXTRA This is like FULL except that is also syncs the directory +** that contains the rollback journal after the rollback +** journal is unlinked. +** ** The above is for a rollback-journal mode. For WAL mode, OFF continues ** to mean that no syncs ever occur. NORMAL means that the WAL is synced ** prior to the start of checkpoint and that the database file is synced @@ -3485,7 +3494,8 @@ void sqlite3PagerShrink(Pager *pPager){ ** was written back into the database. But no sync operations occur for ** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL ** file is synced following each commit operation, in addition to the -** syncs associated with NORMAL. +** syncs associated with NORMAL. There is no difference between FULL +** and EXTRA for WAL mode. ** ** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The ** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync @@ -3674,7 +3684,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ } if( rc==SQLITE_OK ){ pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM; + if( !pNew ) rc = SQLITE_NOMEM_BKPT; } if( rc==SQLITE_OK ){ @@ -3950,7 +3960,7 @@ static int pagerAcquireMapPage( *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; p->flags = PGHDR_MMAP; @@ -4264,8 +4274,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -4308,7 +4319,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pList->pgno==1 ) pager_write_changecounter(pList); /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); + CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); /* Write out the page data. */ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); @@ -4353,11 +4364,14 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + | SQLITE_OPEN_DELETEONCLOSE; + int nStmtSpill = sqlite3Config.nStmtSpill; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ - sqlite3MemJournalOpen(pPager->sjfd); - }else{ - rc = pagerOpentemp(pPager, pPager->sjfd, SQLITE_OPEN_SUBJOURNAL); + nStmtSpill = -1; } + rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nStmtSpill); } return rc; } @@ -4395,7 +4409,7 @@ static int subjournalPage(PgHdr *pPg){ i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ @@ -4578,18 +4592,8 @@ int sqlite3PagerOpen( int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle - ** (there are two of them, the main journal and the sub-journal). This - ** is the maximum space required for an in-memory journal file handle - ** and a regular journal file-handle. Note that a "regular journal-handle" - ** may be a wrapper capable of caching the first portion of the journal - ** file in memory to implement the atomic-write optimization (see - ** source file journal.c). - */ - if( sqlite3JournalSize(pVfs)>sqlite3MemJournalSize() ){ - journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); - }else{ - journalFileSize = ROUND8(sqlite3MemJournalSize()); - } + ** (there are two of them, the main journal and the sub-journal). */ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; @@ -4599,7 +4603,7 @@ int sqlite3PagerOpen( memDb = 1; if( zFilename && zFilename[0] ){ zPathname = sqlite3DbStrDup(0, zFilename); - if( zPathname==0 ) return SQLITE_NOMEM; + if( zPathname==0 ) return SQLITE_NOMEM_BKPT; nPathname = sqlite3Strlen30(zPathname); zFilename = 0; } @@ -4615,7 +4619,7 @@ int sqlite3PagerOpen( nPathname = pVfs->mxPathname+1; zPathname = sqlite3DbMallocRaw(0, nPathname*2); if( zPathname==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); @@ -4668,7 +4672,7 @@ int sqlite3PagerOpen( assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); if( !pPtr ){ sqlite3DbFree(0, zPathname); - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } pPager = (Pager*)(pPtr); pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); @@ -4823,11 +4827,7 @@ act_like_temp_file: assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; -#if SQLITE_EXTRA_DURABLE - pPager->extraSync = 1; -#else pPager->extraSync = 0; -#endif pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; @@ -4944,6 +4944,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ + assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -5036,17 +5037,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ + ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -5132,7 +5133,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); + rc = pager_playback(pPager, !pPager->tempFile); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -5228,7 +5229,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } @@ -5361,7 +5362,7 @@ int sqlite3PagerGet( ); if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ @@ -5388,7 +5389,7 @@ int sqlite3PagerGet( if( rc!=SQLITE_OK ) goto pager_acquire_err; if( pBase==0 ){ pPg = *ppPage = 0; - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto pager_acquire_err; } } @@ -5428,7 +5429,8 @@ int sqlite3PagerGet( goto pager_acquire_err; } - if( MEMDB || pPager->dbSizefd) ){ + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -5562,7 +5564,7 @@ static int pager_open_journal(Pager *pPager){ if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } /* Open the journal file if it is not already open. */ @@ -5570,24 +5572,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -5717,7 +5719,7 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); + CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); cksum = pager_cksum(pPager, (u8*)pData2); /* Even if an IO or diskfull error occurs while journalling the @@ -5958,6 +5960,7 @@ int sqlite3PagerWrite(PgHdr *pPg){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -5988,14 +5991,21 @@ int sqlite3PagerIswriteable(DbPage *pPg){ ** ** Tests show that this optimization can quadruple the speed of large ** DELETE operations. +** +** This optimization cannot be used with a temp-file, as the page may +** have been dirty at the start of the transaction. In that case, if +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the +** current transaction is rolled back. */ void sqlite3PagerDontWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; - if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + if( !pPager->tempFile && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; pPg->flags &= ~PGHDR_WRITEABLE; + testcase( pPg->flags & PGHDR_NEED_SYNC ); pager_set_pagehash(pPg); } } @@ -6074,7 +6084,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ if( DIRECT_MODE ){ const void *zBuf; assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM, zBuf); + CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); pPager->aStat[PAGER_STAT_WRITE]++; @@ -6190,17 +6200,21 @@ int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Provide the ability to easily simulate an I/O error during testing */ + if( sqlite3FaultSim(400) ) return SQLITE_IOERR; + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( 0==pagerFlushOnCommit(pPager, 1) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ + ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -6539,10 +6553,10 @@ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ } /* -** Return true if this is an in-memory pager. +** Return true if this is an in-memory or temp-file backed pager. */ int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; + return pPager->tempFile; } /* @@ -6573,7 +6587,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint ); if( !aNew ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); pPager->aSavepoint = aNew; @@ -6589,7 +6603,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ aNew[ii].iSubRec = pPager->nSubRec; aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); if( !aNew[ii].pInSavepoint ){ - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } if( pagerUseWal(pPager) ){ sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); @@ -6643,7 +6657,11 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = pPager->errCode; /* Return code */ + int rc = pPager->errCode; + +#ifdef SQLITE_ENABLE_ZIPVFS + if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; +#endif assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); @@ -6667,7 +6685,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ if( op==SAVEPOINT_RELEASE ){ if( nNew==0 && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ - if( sqlite3IsMemJournal(pPager->sjfd) ){ + if( sqlite3JournalIsInMemory(pPager->sjfd) ){ rc = sqlite3OsTruncate(pPager->sjfd, 0); assert( rc==SQLITE_OK ); } @@ -6684,6 +6702,20 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } + +#ifdef SQLITE_ENABLE_ZIPVFS + /* If the cache has been modified but the savepoint cannot be rolled + ** back journal_mode=off, put the pager in the error state. This way, + ** if the VFS used by this pager includes ZipVFS, the entire transaction + ** can be rolled back at the ZipVFS level. */ + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF + && pPager->eState>=PAGER_WRITER_CACHEMOD + ){ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + } +#endif } return rc; @@ -6738,14 +6770,6 @@ const char *sqlite3PagerJournalname(Pager *pPager){ return pPager->zJournal; } -/* -** Return true if fsync() calls are disabled for this pager. Return FALSE -** if fsync()s are executed normally. -*/ -int sqlite3PagerNosync(Pager *pPager){ - return pPager->noSync; -} - #ifdef SQLITE_HAS_CODEC /* ** Set or retrieve the codec for this pager @@ -6830,7 +6854,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - if( MEMDB ){ + assert( pPager->tempFile || !MEMDB ); + if( pPager->tempFile ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -6887,7 +6912,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ + if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -6904,8 +6929,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( MEMDB ){ - assert( pPgOld ); + if( pPager->tempFile && pPgOld ){ sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); } @@ -7157,10 +7181,12 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); } #endif + #ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA wal_checkpoint", @@ -7192,6 +7218,7 @@ int sqlite3PagerWalCallback(Pager *pPager){ */ int sqlite3PagerWalSupported(Pager *pPager){ const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + if( pPager->noLock ) return 0; return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); } @@ -7335,6 +7362,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; @@ -7384,5 +7412,4 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif - #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 8d9f0810..fe71fde5 100644 --- a/src/pager.h +++ b/src/pager.h @@ -14,8 +14,8 @@ ** at a time and provides a journal for rollback. */ -#ifndef _PAGER_H_ -#define _PAGER_H_ +#ifndef SQLITE_PAGER_H +#define SQLITE_PAGER_H /* ** Default maximum size for persistent journal files. A negative @@ -68,7 +68,11 @@ typedef struct PgHdr DbPage; #define PAGER_LOCKINGMODE_EXCLUSIVE 1 /* -** Numeric constants that encode the journalmode. +** Numeric constants that encode the journalmode. +** +** The numeric values encoded here (other than PAGER_JOURNALMODE_QUERY) +** are exposed in the API via the "PRAGMA journal_mode" command and +** therefore cannot be changed without a compatibility break. */ #define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ #define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ @@ -86,6 +90,11 @@ typedef struct PgHdr DbPage; /* ** Flags for sqlite3PagerSetFlags() +** +** Value constraints (enforced via assert()): +** PAGER_FULLFSYNC == SQLITE_FullFSync +** PAGER_CKPT_FULLFSYNC == SQLITE_CkptFullFSync +** PAGER_CACHE_SPILL == SQLITE_CacheSpill */ #define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ #define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ @@ -169,10 +178,13 @@ int sqlite3PagerSharedLock(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); int sqlite3PagerCloseWal(Pager *pPager); + int sqlite3PagerUseWal(Pager *pPager); # ifdef SQLITE_ENABLE_SNAPSHOT int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); # endif +#else +# define sqlite3PagerUseWal(x) 0 #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -191,11 +203,10 @@ sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); -int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); -void sqlite3PagerClearCache(Pager *); +void sqlite3PagerClearCache(Pager*); int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ @@ -222,4 +233,4 @@ void *sqlite3PagerCodec(DbPage *); # define enable_simulated_io_errors() #endif -#endif /* _PAGER_H_ */ +#endif /* SQLITE_PAGER_H */ diff --git a/src/parse.y b/src/parse.y index 0bfe4e47..1eff123d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -35,7 +35,6 @@ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); } %stack_overflow { - UNUSED_PARAMETER(yypMinor); /* Silence some compiler warnings */ sqlite3ErrorMsg(pParse, "parser stack overflow"); } @@ -81,15 +80,6 @@ struct LimitVal { Expr *pOffset; /* The OFFSET expression. NULL if there is none */ }; -/* -** An instance of this structure is used to store the LIKE, -** GLOB, NOT LIKE, and NOT GLOB operators. -*/ -struct LikeOp { - Token eOperator; /* "like" or "glob" or "regexp" */ - int bNot; /* True if the NOT keyword is present */ -}; - /* ** An instance of the following structure describes the event of a ** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, @@ -101,11 +91,6 @@ struct LikeOp { */ struct TrigEvent { int a; IdList * b; }; -/* -** An instance of this structure holds the ATTACH key and the key type. -*/ -struct AttachKey { int type; Token key; }; - /* ** Disable lookaside memory allocation for objects that might be ** shared across database connections. @@ -139,9 +124,9 @@ trans_opt ::= TRANSACTION. trans_opt ::= TRANSACTION nm. %type transtype {int} transtype(A) ::= . {A = TK_DEFERRED;} -transtype(A) ::= DEFERRED(X). {A = @X;} -transtype(A) ::= IMMEDIATE(X). {A = @X;} -transtype(A) ::= EXCLUSIVE(X). {A = @X;} +transtype(A) ::= DEFERRED(X). {A = @X; /*A-overwrites-X*/} +transtype(A) ::= IMMEDIATE(X). {A = @X; /*A-overwrites-X*/} +transtype(A) ::= EXCLUSIVE(X). {A = @X; /*A-overwrites-X*/} cmd ::= COMMIT trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= END trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= ROLLBACK trans_opt. {sqlite3RollbackTransaction(pParse);} @@ -164,10 +149,8 @@ cmd ::= create_table create_table_args. create_table ::= createkw temp(T) TABLE ifnotexists(E) nm(Y) dbnm(Z). { sqlite3StartTable(pParse,&Y,&Z,T,0,0,E); } -createkw(A) ::= CREATE(X). { - disableLookaside(pParse); - A = X; -} +createkw(A) ::= CREATE(A). {disableLookaside(pParse);} + %type ifnotexists {int} ifnotexists(A) ::= . {A = 0;} ifnotexists(A) ::= IF NOT EXISTS. {A = 1;} @@ -193,46 +176,9 @@ table_options(A) ::= WITHOUT nm(X). { sqlite3ErrorMsg(pParse, "unknown table option: %.*s", X.n, X.z); } } -columnlist ::= columnlist COMMA column. -columnlist ::= column. - -// A "column" is a complete description of a single column in a -// CREATE TABLE statement. This includes the column name, its -// datatype, and other keywords such as PRIMARY KEY, UNIQUE, REFERENCES, -// NOT NULL and so forth. -// -column(A) ::= columnid(X) type carglist. { - A.z = X.z; - A.n = (int)(pParse->sLastToken.z-X.z) + pParse->sLastToken.n; -} -columnid(A) ::= nm(X). { - sqlite3AddColumn(pParse,&X); - A = X; - pParse->constraintName.n = 0; -} - - -// An IDENTIFIER can be a generic identifier, or one of several -// keywords. Any non-standard keyword can also be an identifier. -// -%token_class id ID|INDEXED. - -// The following directive causes tokens ABORT, AFTER, ASC, etc. to -// fallback to ID if they will not parse as their original value. -// This obviates the need for the "id" nonterminal. -// -%fallback ID - ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW - CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR - IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW - ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT -%ifdef SQLITE_OMIT_COMPOUND_SELECT - EXCEPT INTERSECT UNION -%endif SQLITE_OMIT_COMPOUND_SELECT - REINDEX RENAME CTIME_KW IF - . -%wildcard ANY. +columnlist ::= columnlist COMMA columnname carglist. +columnlist ::= columnname carglist. +columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} // Define operator precedence early so that this is the first occurrence // of the operator tokens in the grammer. Keeping the operators together @@ -258,6 +204,29 @@ columnid(A) ::= nm(X). { %left COLLATE. %right BITNOT. +// An IDENTIFIER can be a generic identifier, or one of several +// keywords. Any non-standard keyword can also be an identifier. +// +%token_class id ID|INDEXED. + +// The following directive causes tokens ABORT, AFTER, ASC, etc. to +// fallback to ID if they will not parse as their original value. +// This obviates the need for the "id" nonterminal. +// +%fallback ID + ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW + CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR + IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT +%ifdef SQLITE_OMIT_COMPOUND_SELECT + EXCEPT INTERSECT UNION +%endif SQLITE_OMIT_COMPOUND_SELECT + REINDEX RENAME CTIME_KW IF + . +%wildcard ANY. + + // And "ids" is an identifer-or-string. // %token_class ids ID|STRING. @@ -265,29 +234,26 @@ columnid(A) ::= nm(X). { // The name of a column or table can be any of the following: // %type nm {Token} -nm(A) ::= id(X). {A = X;} -nm(A) ::= STRING(X). {A = X;} -nm(A) ::= JOIN_KW(X). {A = X;} +nm(A) ::= id(A). +nm(A) ::= STRING(A). +nm(A) ::= JOIN_KW(A). -// A typetoken is really one or more tokens that form a type name such +// A typetoken is really zero or more tokens that form a type name such // as can be found after the column name in a CREATE TABLE statement. // Multiple tokens are concatenated to form the value of the typetoken. // %type typetoken {Token} -type ::= . -type ::= typetoken(X). {sqlite3AddColumnType(pParse,&X);} -typetoken(A) ::= typename(X). {A = X;} -typetoken(A) ::= typename(X) LP signed RP(Y). { - A.z = X.z; - A.n = (int)(&Y.z[Y.n] - X.z); +typetoken(A) ::= . {A.n = 0; A.z = 0;} +typetoken(A) ::= typename(A). +typetoken(A) ::= typename(A) LP signed RP(Y). { + A.n = (int)(&Y.z[Y.n] - A.z); } -typetoken(A) ::= typename(X) LP signed COMMA signed RP(Y). { - A.z = X.z; - A.n = (int)(&Y.z[Y.n] - X.z); +typetoken(A) ::= typename(A) LP signed COMMA signed RP(Y). { + A.n = (int)(&Y.z[Y.n] - A.z); } %type typename {Token} -typename(A) ::= ids(X). {A = X;} -typename(A) ::= typename(X) ids(Y). {A.z=X.z; A.n=Y.n+(int)(Y.z-X.z);} +typename(A) ::= ids(A). +typename(A) ::= typename(A) ids(Y). {A.n=Y.n+(int)(Y.z-A.z);} signed ::= plus_num. signed ::= minus_num. @@ -309,7 +275,7 @@ ccons ::= DEFAULT MINUS(A) term(X). { } ccons ::= DEFAULT id(X). { ExprSpan v; - spanExpr(&v, pParse, TK_STRING, &X); + spanExpr(&v, pParse, TK_STRING, X); sqlite3AddDefaultValue(pParse,&v); } @@ -320,7 +286,8 @@ ccons ::= NULL onconf. ccons ::= NOT NULL onconf(R). {sqlite3AddNotNull(pParse, R);} ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I). {sqlite3AddPrimaryKey(pParse,0,R,I,Z);} -ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);} +ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X.pExpr);} ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} @@ -339,7 +306,7 @@ autoinc(X) ::= AUTOINCR. {X = 1;} // %type refargs {int} refargs(A) ::= . { A = OE_None*0x0101; /* EV: R-19803-45884 */} -refargs(A) ::= refargs(X) refarg(Y). { A = (X & ~Y.mask) | Y.value; } +refargs(A) ::= refargs(A) refarg(Y). { A = (A & ~Y.mask) | Y.value; } %type refarg {struct {int value; int mask;}} refarg(A) ::= MATCH nm. { A.value = 0; A.mask = 0x000000; } refarg(A) ::= ON INSERT refact. { A.value = 0; A.mask = 0x000000; } @@ -360,7 +327,7 @@ init_deferred_pred_opt(A) ::= INITIALLY DEFERRED. {A = 1;} init_deferred_pred_opt(A) ::= INITIALLY IMMEDIATE. {A = 0;} conslist_opt(A) ::= . {A.n = 0; A.z = 0;} -conslist_opt(A) ::= COMMA(X) conslist. {A = X;} +conslist_opt(A) ::= COMMA(A) conslist. conslist ::= conslist tconscomma tcons. conslist ::= tcons. tconscomma ::= COMMA. {pParse->constraintName.n = 0;} @@ -369,7 +336,8 @@ tcons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} tcons ::= UNIQUE LP sortlist(X) RP onconf(R). - {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);} + {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} tcons ::= CHECK LP expr(E) RP onconf. {sqlite3AddCheckConstraint(pParse,E.pExpr);} tcons ::= FOREIGN KEY LP eidlist(FA) RP @@ -379,7 +347,7 @@ tcons ::= FOREIGN KEY LP eidlist(FA) RP } %type defer_subclause_opt {int} defer_subclause_opt(A) ::= . {A = 0;} -defer_subclause_opt(A) ::= defer_subclause(X). {A = X;} +defer_subclause_opt(A) ::= defer_subclause(A). // The following is a non-standard extension that allows us to declare the // default behavior when there is a constraint conflict. @@ -391,7 +359,7 @@ onconf(A) ::= . {A = OE_Default;} onconf(A) ::= ON CONFLICT resolvetype(X). {A = X;} orconf(A) ::= . {A = OE_Default;} orconf(A) ::= OR resolvetype(X). {A = X;} -resolvetype(A) ::= raisetype(X). {A = X;} +resolvetype(A) ::= raisetype(A). resolvetype(A) ::= IGNORE. {A = OE_Ignore;} resolvetype(A) ::= REPLACE. {A = OE_Replace;} @@ -463,14 +431,14 @@ select(A) ::= with(W) selectnowith(X). { }else{ sqlite3WithDelete(pParse->db, W); } - A = p; + A = p; /*A-overwrites-W*/ } -selectnowith(A) ::= oneselect(X). {A = X;} +selectnowith(A) ::= oneselect(A). %ifndef SQLITE_OMIT_COMPOUND_SELECT -selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z). { +selectnowith(A) ::= selectnowith(A) multiselect_op(Y) oneselect(Z). { Select *pRhs = Z; - Select *pLhs = X; + Select *pLhs = A; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -491,12 +459,15 @@ selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z). { A = pRhs; } %type multiselect_op {int} -multiselect_op(A) ::= UNION(OP). {A = @OP;} +multiselect_op(A) ::= UNION(OP). {A = @OP; /*A-overwrites-OP*/} multiselect_op(A) ::= UNION ALL. {A = TK_ALL;} -multiselect_op(A) ::= EXCEPT|INTERSECT(OP). {A = @OP;} +multiselect_op(A) ::= EXCEPT|INTERSECT(OP). {A = @OP; /*A-overwrites-OP*/} %endif SQLITE_OMIT_COMPOUND_SELECT oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { +#if SELECTTRACE_ENABLED + Token s = S; /*A-overwrites-S*/ +#endif A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); #if SELECTTRACE_ENABLED /* Populate the Select.zSelName[] string that is used to help with @@ -509,7 +480,7 @@ oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) ** is an integer that is incremented with each SELECT statement seen. */ if( A!=0 ){ - const char *z = S.z+6; + const char *z = s.z+6; int i; sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "#%d", ++pParse->nSelect); @@ -523,20 +494,19 @@ oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) } #endif /* SELECTRACE_ENABLED */ } -oneselect(A) ::= values(X). {A = X;} +oneselect(A) ::= values(A). %type values {Select*} %destructor values {sqlite3SelectDelete(pParse->db, $$);} values(A) ::= VALUES LP nexprlist(X) RP. { A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0,0); } -values(A) ::= values(X) COMMA LP exprlist(Y) RP. { - Select *pRight, *pLeft = X; +values(A) ::= values(A) COMMA LP exprlist(Y) RP. { + Select *pRight, *pLeft = A; pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; - pLeft = X; pRight->pPrior = pLeft; A = pRight; }else{ @@ -561,22 +531,22 @@ distinct(A) ::= . {A = 0;} %destructor selcollist {sqlite3ExprListDelete(pParse->db, $$);} %type sclp {ExprList*} %destructor sclp {sqlite3ExprListDelete(pParse->db, $$);} -sclp(A) ::= selcollist(X) COMMA. {A = X;} +sclp(A) ::= selcollist(A) COMMA. sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) expr(X) as(Y). { - A = sqlite3ExprListAppend(pParse, P, X.pExpr); +selcollist(A) ::= sclp(A) expr(X) as(Y). { + A = sqlite3ExprListAppend(pParse, A, X.pExpr); if( Y.n>0 ) sqlite3ExprListSetName(pParse, A, &Y, 1); sqlite3ExprListSetSpan(pParse,A,&X); } -selcollist(A) ::= sclp(P) STAR. { +selcollist(A) ::= sclp(A) STAR. { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - A = sqlite3ExprListAppend(pParse, P, p); + A = sqlite3ExprListAppend(pParse, A, p); } -selcollist(A) ::= sclp(P) nm(X) DOT STAR(Y). { - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, &Y); +selcollist(A) ::= sclp(A) nm(X) DOT STAR. { + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, 0); Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - A = sqlite3ExprListAppend(pParse,P, pDot); + A = sqlite3ExprListAppend(pParse,A, pDot); } // An option "AS " phrase that can follow one of the expressions that @@ -584,8 +554,8 @@ selcollist(A) ::= sclp(P) nm(X) DOT STAR(Y). { // %type as {Token} as(X) ::= AS nm(Y). {X = Y;} -as(X) ::= ids(Y). {X = Y;} -as(X) ::= . {X.n = 0;} +as(X) ::= ids(X). +as(X) ::= . {X.n = 0; X.z = 0;} %type seltablist {SrcList*} @@ -606,32 +576,31 @@ from(A) ::= FROM seltablist(X). { // "seltablist" is a "Select Table List" - the content of the FROM clause // in a SELECT statement. "stl_prefix" is a prefix of this list. // -stl_prefix(A) ::= seltablist(X) joinop(Y). { - A = X; +stl_prefix(A) ::= seltablist(A) joinop(Y). { if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].fg.jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} -seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) +seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) as(Z) indexed_opt(I) on_opt(N) using_opt(U). { - A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); + A = sqlite3SrcListAppendFromTerm(pParse,A,&Y,&D,&Z,0,N,U); sqlite3SrcListIndexedBy(pParse, A, &I); } -seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) +seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_opt(N) using_opt(U). { - A = sqlite3SrcListAppendFromTerm(pParse,X,&Y,&D,&Z,0,N,U); + A = sqlite3SrcListAppendFromTerm(pParse,A,&Y,&D,&Z,0,N,U); sqlite3SrcListFuncArgs(pParse, A, E); } %ifndef SQLITE_OMIT_SUBQUERY - seltablist(A) ::= stl_prefix(X) LP select(S) RP + seltablist(A) ::= stl_prefix(A) LP select(S) RP as(Z) on_opt(N) using_opt(U). { - A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,S,N,U); + A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,S,N,U); } - seltablist(A) ::= stl_prefix(X) LP seltablist(F) RP + seltablist(A) ::= stl_prefix(A) LP seltablist(F) RP as(Z) on_opt(N) using_opt(U). { - if( X==0 && Z.n==0 && N==0 && U==0 ){ + if( A==0 && Z.n==0 && N==0 && U==0 ){ A = F; }else if( F->nSrc==1 ){ - A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,0,N,U); + A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,N,U); if( A ){ struct SrcList_item *pNew = &A->a[A->nSrc-1]; struct SrcList_item *pOld = F->a; @@ -646,7 +615,7 @@ seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) Select *pSubquery; sqlite3SrcListShiftJoinType(F); pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0,0); - A = sqlite3SrcListAppendFromTerm(pParse,X,0,0,&Z,pSubquery,N,U); + A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,pSubquery,N,U); } } %endif SQLITE_OMIT_SUBQUERY @@ -657,14 +626,17 @@ dbnm(A) ::= DOT nm(X). {A = X;} %type fullname {SrcList*} %destructor fullname {sqlite3SrcListDelete(pParse->db, $$);} -fullname(A) ::= nm(X) dbnm(Y). {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y);} +fullname(A) ::= nm(X) dbnm(Y). + {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y); /*A-overwrites-X*/} %type joinop {int} joinop(X) ::= COMMA|JOIN. { X = JT_INNER; } -joinop(X) ::= JOIN_KW(A) JOIN. { X = sqlite3JoinType(pParse,&A,0,0); } -joinop(X) ::= JOIN_KW(A) nm(B) JOIN. { X = sqlite3JoinType(pParse,&A,&B,0); } +joinop(X) ::= JOIN_KW(A) JOIN. + {X = sqlite3JoinType(pParse,&A,0,0); /*X-overwrites-A*/} +joinop(X) ::= JOIN_KW(A) nm(B) JOIN. + {X = sqlite3JoinType(pParse,&A,&B,0); /*X-overwrites-A*/} joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN. - { X = sqlite3JoinType(pParse,&A,&B,&C); } + {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/} %type on_opt {Expr*} %destructor on_opt {sqlite3ExprDelete(pParse->db, $$);} @@ -704,12 +676,12 @@ using_opt(U) ::= . {U = 0;} orderby_opt(A) ::= . {A = 0;} orderby_opt(A) ::= ORDER BY sortlist(X). {A = X;} -sortlist(A) ::= sortlist(X) COMMA expr(Y) sortorder(Z). { - A = sqlite3ExprListAppend(pParse,X,Y.pExpr); +sortlist(A) ::= sortlist(A) COMMA expr(Y) sortorder(Z). { + A = sqlite3ExprListAppend(pParse,A,Y.pExpr); sqlite3ExprListSetSortOrder(A,Z); } sortlist(A) ::= expr(Y) sortorder(Z). { - A = sqlite3ExprListAppend(pParse,0,Y.pExpr); + A = sqlite3ExprListAppend(pParse,0,Y.pExpr); /*A-overwrites-Y*/ sqlite3ExprListSetSortOrder(A,Z); } @@ -799,14 +771,20 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) %type setlist {ExprList*} %destructor setlist {sqlite3ExprListDelete(pParse->db, $$);} -setlist(A) ::= setlist(Z) COMMA nm(X) EQ expr(Y). { - A = sqlite3ExprListAppend(pParse, Z, Y.pExpr); +setlist(A) ::= setlist(A) COMMA nm(X) EQ expr(Y). { + A = sqlite3ExprListAppend(pParse, A, Y.pExpr); sqlite3ExprListSetName(pParse, A, &X, 1); } +setlist(A) ::= setlist(A) COMMA LP idlist(X) RP EQ expr(Y). { + A = sqlite3ExprListAppendVector(pParse, A, X, Y.pExpr); +} setlist(A) ::= nm(X) EQ expr(Y). { A = sqlite3ExprListAppend(pParse, 0, Y.pExpr); sqlite3ExprListSetName(pParse, A, &X, 1); } +setlist(A) ::= LP idlist(X) RP EQ expr(Y). { + A = sqlite3ExprListAppendVector(pParse, 0, X, Y.pExpr); +} ////////////////////////// The INSERT command ///////////////////////////////// // @@ -831,10 +809,10 @@ insert_cmd(A) ::= REPLACE. {A = OE_Replace;} idlist_opt(A) ::= . {A = 0;} idlist_opt(A) ::= LP idlist(X) RP. {A = X;} -idlist(A) ::= idlist(X) COMMA nm(Y). - {A = sqlite3IdListAppend(pParse->db,X,&Y);} +idlist(A) ::= idlist(A) COMMA nm(Y). + {A = sqlite3IdListAppend(pParse->db,A,&Y);} idlist(A) ::= nm(Y). - {A = sqlite3IdListAppend(pParse->db,0,&Y);} + {A = sqlite3IdListAppend(pParse->db,0,&Y); /*A-overwrites-Y*/} /////////////////////////// Expression Processing ///////////////////////////// // @@ -858,61 +836,87 @@ idlist(A) ::= nm(Y). ** new Expr to populate pOut. Set the span of pOut to be the identifier ** that created the expression. */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token *pValue){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, pValue); - pOut->zStart = pValue->z; - pOut->zEnd = &pValue->z[pValue->n]; + static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){ + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = (u8)op; + p->flags = EP_Leaf; + p->iAgg = -1; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + if( sqlite3Isquote(p->u.zToken[0]) ){ + if( p->u.zToken[0]=='"' ) p->flags |= EP_DblQuoted; + sqlite3Dequote(p->u.zToken); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + } + pOut->pExpr = p; + pOut->zStart = t.z; + pOut->zEnd = &t.z[t.n]; } } -expr(A) ::= term(X). {A = X;} -expr(A) ::= LP(B) expr(X) RP(E). {A.pExpr = X.pExpr; spanSet(&A,&B,&E);} -term(A) ::= NULL(X). {spanExpr(&A, pParse, @X, &X);} -expr(A) ::= id(X). {spanExpr(&A, pParse, TK_ID, &X);} -expr(A) ::= JOIN_KW(X). {spanExpr(&A, pParse, TK_ID, &X);} +expr(A) ::= term(A). +expr(A) ::= LP(B) expr(X) RP(E). + {spanSet(&A,&B,&E); /*A-overwrites-B*/ A.pExpr = X.pExpr;} +term(A) ::= NULL(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} +expr(A) ::= id(X). {spanExpr(&A,pParse,TK_ID,X); /*A-overwrites-X*/} +expr(A) ::= JOIN_KW(X). {spanExpr(&A,pParse,TK_ID,X); /*A-overwrites-X*/} expr(A) ::= nm(X) DOT nm(Y). { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Y); + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1); + spanSet(&A,&X,&Y); /*A-overwrites-X*/ A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&A,&X,&Y); } expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Y); - Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Z); + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1); + Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &Z, 1); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); + spanSet(&A,&X,&Z); /*A-overwrites-X*/ A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&A,&X,&Z); } -term(A) ::= INTEGER|FLOAT|BLOB(X). {spanExpr(&A, pParse, @X, &X);} -term(A) ::= STRING(X). {spanExpr(&A, pParse, @X, &X);} +term(A) ::= FLOAT|BLOB(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} +term(A) ::= STRING(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} +term(A) ::= INTEGER(X). { + A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); + A.zStart = X.z; + A.zEnd = X.z + X.n; + if( A.pExpr ) A.pExpr->flags |= EP_Leaf; +} expr(A) ::= VARIABLE(X). { - if( X.n>=2 && X.z[0]=='#' && sqlite3Isdigit(X.z[1]) ){ + if( !(X.z[0]=='#' && sqlite3Isdigit(X.z[1])) ){ + u32 n = X.n; + spanExpr(&A, pParse, TK_VARIABLE, X); + sqlite3ExprAssignVarNumber(pParse, A.pExpr, n); + }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ + Token t = X; /*A-overwrites-X*/ + assert( t.n>=2 ); + spanSet(&A, &t, &t); if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); A.pExpr = 0; }else{ - A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); - if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable); + A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, 0); + if( A.pExpr ) sqlite3GetInt32(&t.z[1], &A.pExpr->iTable); } - }else{ - spanExpr(&A, pParse, TK_VARIABLE, &X); - sqlite3ExprAssignVarNumber(pParse, A.pExpr); } - spanSet(&A, &X, &X); } -expr(A) ::= expr(E) COLLATE ids(C). { - A.pExpr = sqlite3ExprAddCollateToken(pParse, E.pExpr, &C, 1); - A.zStart = E.zStart; +expr(A) ::= expr(A) COLLATE ids(C). { + A.pExpr = sqlite3ExprAddCollateToken(pParse, A.pExpr, &C, 1); A.zEnd = &C.z[C.n]; } %ifndef SQLITE_OMIT_CAST expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). { + spanSet(&A,&X,&Y); /*A-overwrites-X*/ A.pExpr = sqlite3PExpr(pParse, TK_CAST, E.pExpr, 0, &T); - spanSet(&A,&X,&Y); } %endif SQLITE_OMIT_CAST expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP(E). { @@ -939,58 +943,71 @@ term(A) ::= CTIME_KW(OP). { ** objects and uses the result to populate a new ExprSpan object. */ static void spanBinaryExpr( - ExprSpan *pOut, /* Write the result here */ Parse *pParse, /* The parsing context. Errors accumulate here */ int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand */ + ExprSpan *pLeft, /* The left operand, and output */ ExprSpan *pRight /* The right operand */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pOut->zStart = pLeft->zStart; - pOut->zEnd = pRight->zEnd; + pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pLeft->zEnd = pRight->zEnd; } /* If doNot is true, then add a TK_NOT Expr-node wrapper around the ** outside of *ppExpr. */ - static void exprNot(Parse *pParse, int doNot, Expr **ppExpr){ - if( doNot ) *ppExpr = sqlite3PExpr(pParse, TK_NOT, *ppExpr, 0, 0); + static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){ + if( doNot ){ + pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0, 0); + } } } -expr(A) ::= expr(X) AND(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) OR(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) LT|GT|GE|LE(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) EQ|NE(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) BITAND|BITOR|LSHIFT|RSHIFT(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) PLUS|MINUS(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). - {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} -%type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} -expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { +expr(A) ::= LP(L) nexprlist(X) COMMA expr(Y) RP(R). { + ExprList *pList = sqlite3ExprListAppend(pParse, X, Y.pExpr); + A.pExpr = sqlite3PExpr(pParse, TK_VECTOR, 0, 0, 0); + if( A.pExpr ){ + A.pExpr->x.pList = pList; + spanSet(&A, &L, &R); + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + +expr(A) ::= expr(A) AND(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) OR(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) LT|GT|GE|LE(OP) expr(Y). + {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) EQ|NE(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) BITAND|BITOR|LSHIFT|RSHIFT(OP) expr(Y). + {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) PLUS|MINUS(OP) expr(Y). + {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y). + {spanBinaryExpr(pParse,@OP,&A,&Y);} +expr(A) ::= expr(A) CONCAT(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} +%type likeop {Token} +likeop(A) ::= LIKE_KW|MATCH(X). {A=X;/*A-overwrites-X*/} +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A=X; A.n|=0x80000000; /*A-overwrite-X*/} +expr(A) ::= expr(A) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; + int bNot = OP.n & 0x80000000; + OP.n &= 0x7fffffff; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); - A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - exprNot(pParse, OP.bNot, &A.pExpr); - A.zStart = X.zStart; + pList = sqlite3ExprListAppend(pParse,pList, A.pExpr); + A.pExpr = sqlite3ExprFunction(pParse, pList, &OP); + exprNot(pParse, bNot, &A); A.zEnd = Y.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; } -expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { +expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { ExprList *pList; + int bNot = OP.n & 0x80000000; + OP.n &= 0x7fffffff; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, A.pExpr); pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); - A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - exprNot(pParse, OP.bNot, &A.pExpr); - A.zStart = X.zStart; + A.pExpr = sqlite3ExprFunction(pParse, pList, &OP); + exprNot(pParse, bNot, &A); A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; } @@ -999,20 +1016,18 @@ expr(A) ::= expr(X) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { /* Construct an expression node for a unary postfix operator */ static void spanUnaryPostfix( - ExprSpan *pOut, /* Write the new expression node here */ Parse *pParse, /* Parsing context to record errors */ int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ + ExprSpan *pOperand, /* The operand, and output */ Token *pPostOp /* The operand token for setting the span */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zStart = pOperand->zStart; - pOut->zEnd = &pPostOp->z[pPostOp->n]; + pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOperand->zEnd = &pPostOp->z[pPostOp->n]; } } -expr(A) ::= expr(X) ISNULL|NOTNULL(E). {spanUnaryPostfix(&A,pParse,@E,&X,&E);} -expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} +expr(A) ::= expr(A) ISNULL|NOTNULL(E). {spanUnaryPostfix(pParse,@E,&A,&E);} +expr(A) ::= expr(A) NOT NULL(E). {spanUnaryPostfix(pParse,TK_NOTNULL,&A,&E);} %include { /* A routine to convert a binary TK_IS or TK_ISNOT expression into a @@ -1033,12 +1048,12 @@ expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} // If expr2 is NULL then code as TK_ISNULL or TK_NOTNULL. If expr2 // is any other expression, code as TK_IS or TK_ISNOT. // -expr(A) ::= expr(X) IS expr(Y). { - spanBinaryExpr(&A,pParse,TK_IS,&X,&Y); +expr(A) ::= expr(A) IS expr(Y). { + spanBinaryExpr(pParse,TK_IS,&A,&Y); binaryToUnaryIfNull(pParse, Y.pExpr, A.pExpr, TK_ISNULL); } -expr(A) ::= expr(X) IS NOT expr(Y). { - spanBinaryExpr(&A,pParse,TK_ISNOT,&X,&Y); +expr(A) ::= expr(A) IS NOT expr(Y). { + spanBinaryExpr(pParse,TK_ISNOT,&A,&Y); binaryToUnaryIfNull(pParse, Y.pExpr, A.pExpr, TK_NOTNULL); } @@ -1052,42 +1067,43 @@ expr(A) ::= expr(X) IS NOT expr(Y). { ExprSpan *pOperand, /* The operand */ Token *pPreOp /* The operand token for setting the span */ ){ - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); pOut->zStart = pPreOp->z; + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); pOut->zEnd = pOperand->zEnd; } } -expr(A) ::= NOT(B) expr(X). {spanUnaryPrefix(&A,pParse,@B,&X,&B);} -expr(A) ::= BITNOT(B) expr(X). {spanUnaryPrefix(&A,pParse,@B,&X,&B);} +expr(A) ::= NOT(B) expr(X). + {spanUnaryPrefix(&A,pParse,@B,&X,&B);/*A-overwrites-B*/} +expr(A) ::= BITNOT(B) expr(X). + {spanUnaryPrefix(&A,pParse,@B,&X,&B);/*A-overwrites-B*/} expr(A) ::= MINUS(B) expr(X). [BITNOT] - {spanUnaryPrefix(&A,pParse,TK_UMINUS,&X,&B);} + {spanUnaryPrefix(&A,pParse,TK_UMINUS,&X,&B);/*A-overwrites-B*/} expr(A) ::= PLUS(B) expr(X). [BITNOT] - {spanUnaryPrefix(&A,pParse,TK_UPLUS,&X,&B);} + {spanUnaryPrefix(&A,pParse,TK_UPLUS,&X,&B);/*A-overwrites-B*/} %type between_op {int} between_op(A) ::= BETWEEN. {A = 0;} between_op(A) ::= NOT BETWEEN. {A = 1;} -expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { +expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ExprList *pList = sqlite3ExprListAppend(pParse,0, X.pExpr); pList = sqlite3ExprListAppend(pParse,pList, Y.pExpr); - A.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, W.pExpr, 0, 0); + A.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, A.pExpr, 0, 0); if( A.pExpr ){ A.pExpr->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - exprNot(pParse, N, &A.pExpr); - A.zStart = W.zStart; + exprNot(pParse, N, &A); A.zEnd = Y.zEnd; } %ifndef SQLITE_OMIT_SUBQUERY %type in_op {int} in_op(A) ::= IN. {A = 0;} in_op(A) ::= NOT IN. {A = 1;} - expr(A) ::= expr(X) in_op(N) LP exprlist(Y) RP(E). [IN] { + expr(A) ::= expr(A) in_op(N) LP exprlist(Y) RP(E). [IN] { if( Y==0 ){ /* Expressions of the form ** @@ -1097,8 +1113,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ + sqlite3ExprDelete(pParse->db, A.pExpr); A.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[N]); - sqlite3ExprDelete(pParse->db, X.pExpr); }else if( Y->nExpr==1 ){ /* Expressions of the form: ** @@ -1125,75 +1141,50 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } - A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); + A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, A.pExpr, pRHS, 0); }else{ - A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); + A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0); if( A.pExpr ){ A.pExpr->x.pList = Y; sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); }else{ sqlite3ExprListDelete(pParse->db, Y); } - exprNot(pParse, N, &A.pExpr); + exprNot(pParse, N, &A); } - A.zStart = X.zStart; A.zEnd = &E.z[E.n]; } expr(A) ::= LP(B) select(X) RP(E). { + spanSet(&A,&B,&E); /*A-overwrites-B*/ A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = X; - ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, X); - } - A.zStart = B.z; + sqlite3PExprAddSelect(pParse, A.pExpr, X); + } + expr(A) ::= expr(A) in_op(N) LP select(Y) RP(E). [IN] { + A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0); + sqlite3PExprAddSelect(pParse, A.pExpr, Y); + exprNot(pParse, N, &A); A.zEnd = &E.z[E.n]; } - expr(A) ::= expr(X) in_op(N) LP select(Y) RP(E). [IN] { - A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = Y; - ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, Y); - } - exprNot(pParse, N, &A.pExpr); - A.zStart = X.zStart; - A.zEnd = &E.z[E.n]; - } - expr(A) ::= expr(X) in_op(N) nm(Y) dbnm(Z). [IN] { + expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z) paren_exprlist(E). [IN] { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z); - A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); - }else{ - sqlite3SrcListDelete(pParse->db, pSrc); - } - exprNot(pParse, N, &A.pExpr); - A.zStart = X.zStart; + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + if( E ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E); + A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0); + sqlite3PExprAddSelect(pParse, A.pExpr, pSelect); + exprNot(pParse, N, &A); A.zEnd = Z.z ? &Z.z[Z.n] : &Y.z[Y.n]; } expr(A) ::= EXISTS(B) LP select(Y) RP(E). { - Expr *p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - if( p ){ - p->x.pSelect = Y; - ExprSetProperty(p, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, p); - }else{ - sqlite3SelectDelete(pParse->db, Y); - } - A.zStart = B.z; - A.zEnd = &E.z[E.n]; + Expr *p; + spanSet(&A,&B,&E); /*A-overwrites-B*/ + p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + sqlite3PExprAddSelect(pParse, p, Y); } %endif SQLITE_OMIT_SUBQUERY /* CASE expressions */ expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). { + spanSet(&A,&C,&E); /*A-overwrites-C*/ A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0, 0); if( A.pExpr ){ A.pExpr->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y; @@ -1202,13 +1193,11 @@ expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). { sqlite3ExprListDelete(pParse->db, Y); sqlite3ExprDelete(pParse->db, Z); } - A.zStart = C.z; - A.zEnd = &E.z[E.n]; } %type case_exprlist {ExprList*} %destructor case_exprlist {sqlite3ExprListDelete(pParse->db, $$);} -case_exprlist(A) ::= case_exprlist(X) WHEN expr(Y) THEN expr(Z). { - A = sqlite3ExprListAppend(pParse,X, Y.pExpr); +case_exprlist(A) ::= case_exprlist(A) WHEN expr(Y) THEN expr(Z). { + A = sqlite3ExprListAppend(pParse,A, Y.pExpr); A = sqlite3ExprListAppend(pParse,A, Z.pExpr); } case_exprlist(A) ::= WHEN expr(Y) THEN expr(Z). { @@ -1221,7 +1210,7 @@ case_else(A) ::= ELSE expr(X). {A = X.pExpr;} case_else(A) ::= . {A = 0;} %type case_operand {Expr*} %destructor case_operand {sqlite3ExprDelete(pParse->db, $$);} -case_operand(A) ::= expr(X). {A = X.pExpr;} +case_operand(A) ::= expr(X). {A = X.pExpr; /*A-overwrites-X*/} case_operand(A) ::= . {A = 0;} %type exprlist {ExprList*} @@ -1229,12 +1218,21 @@ case_operand(A) ::= . {A = 0;} %type nexprlist {ExprList*} %destructor nexprlist {sqlite3ExprListDelete(pParse->db, $$);} -exprlist(A) ::= nexprlist(X). {A = X;} +exprlist(A) ::= nexprlist(A). exprlist(A) ::= . {A = 0;} -nexprlist(A) ::= nexprlist(X) COMMA expr(Y). - {A = sqlite3ExprListAppend(pParse,X,Y.pExpr);} +nexprlist(A) ::= nexprlist(A) COMMA expr(Y). + {A = sqlite3ExprListAppend(pParse,A,Y.pExpr);} nexprlist(A) ::= expr(Y). - {A = sqlite3ExprListAppend(pParse,0,Y.pExpr);} + {A = sqlite3ExprListAppend(pParse,0,Y.pExpr); /*A-overwrites-Y*/} + +%ifndef SQLITE_OMIT_SUBQUERY +/* A paren_exprlist is an optional expression list contained inside +** of parenthesis */ +%type paren_exprlist {ExprList*} +%destructor paren_exprlist {sqlite3ExprListDelete(pParse->db, $$);} +paren_exprlist(A) ::= . {A = 0;} +paren_exprlist(A) ::= LP exprlist(X) RP. {A = X;} +%endif SQLITE_OMIT_SUBQUERY ///////////////////////////// The CREATE INDEX command /////////////////////// @@ -1243,7 +1241,7 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) ON nm(Y) LP sortlist(Z) RP where_opt(W). { sqlite3CreateIndex(pParse, &X, &D, sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U, - &S, W, SQLITE_SO_ASC, NE); + &S, W, SQLITE_SO_ASC, NE, SQLITE_IDXTYPE_APPDEF); } %type uniqueflag {int} @@ -1298,11 +1296,11 @@ uniqueflag(A) ::= . {A = OE_None;} eidlist_opt(A) ::= . {A = 0;} eidlist_opt(A) ::= LP eidlist(X) RP. {A = X;} -eidlist(A) ::= eidlist(X) COMMA nm(Y) collate(C) sortorder(Z). { - A = parserAddExprIdListTerm(pParse, X, &Y, C, Z); +eidlist(A) ::= eidlist(A) COMMA nm(Y) collate(C) sortorder(Z). { + A = parserAddExprIdListTerm(pParse, A, &Y, C, Z); } eidlist(A) ::= nm(Y) collate(C) sortorder(Z). { - A = parserAddExprIdListTerm(pParse, 0, &Y, C, Z); + A = parserAddExprIdListTerm(pParse, 0, &Y, C, Z); /*A-overwrites-Y*/ } %type collate {int} @@ -1318,8 +1316,8 @@ cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite3DropIndex(pParse, X, E);} // %ifndef SQLITE_OMIT_VACUUM %ifndef SQLITE_OMIT_ATTACH -cmd ::= VACUUM. {sqlite3Vacuum(pParse);} -cmd ::= VACUUM nm. {sqlite3Vacuum(pParse);} +cmd ::= VACUUM. {sqlite3Vacuum(pParse,0);} +cmd ::= VACUUM nm(X). {sqlite3Vacuum(pParse,&X);} %endif SQLITE_OMIT_ATTACH %endif SQLITE_OMIT_VACUUM @@ -1334,15 +1332,15 @@ cmd ::= PRAGMA nm(X) dbnm(Z) EQ minus_num(Y). cmd ::= PRAGMA nm(X) dbnm(Z) LP minus_num(Y) RP. {sqlite3Pragma(pParse,&X,&Z,&Y,1);} -nmnum(A) ::= plus_num(X). {A = X;} -nmnum(A) ::= nm(X). {A = X;} -nmnum(A) ::= ON(X). {A = X;} -nmnum(A) ::= DELETE(X). {A = X;} -nmnum(A) ::= DEFAULT(X). {A = X;} +nmnum(A) ::= plus_num(A). +nmnum(A) ::= nm(A). +nmnum(A) ::= ON(A). +nmnum(A) ::= DELETE(A). +nmnum(A) ::= DEFAULT(A). %endif SQLITE_OMIT_PRAGMA %token_class number INTEGER|FLOAT. plus_num(A) ::= PLUS number(X). {A = X;} -plus_num(A) ::= number(X). {A = X;} +plus_num(A) ::= number(A). minus_num(A) ::= MINUS number(X). {A = X;} //////////////////////////// The CREATE TRIGGER command ///////////////////// @@ -1359,7 +1357,7 @@ trigger_decl(A) ::= temp(T) TRIGGER ifnotexists(NOERR) nm(B) dbnm(Z) trigger_time(C) trigger_event(D) ON fullname(E) foreach_clause when_clause(G). { sqlite3BeginTrigger(pParse, &B, &Z, C, D.a, D.b, E, G, T, NOERR); - A = (Z.n==0?B:Z); + A = (Z.n==0?B:Z); /*A-overwrites-T*/ } %type trigger_time {int} @@ -1370,9 +1368,9 @@ trigger_time(A) ::= . { A = TK_BEFORE; } %type trigger_event {struct TrigEvent} %destructor trigger_event {sqlite3IdListDelete(pParse->db, $$.b);} -trigger_event(A) ::= DELETE|INSERT(OP). {A.a = @OP; A.b = 0;} -trigger_event(A) ::= UPDATE(OP). {A.a = @OP; A.b = 0;} -trigger_event(A) ::= UPDATE OF idlist(X). {A.a = TK_UPDATE; A.b = X;} +trigger_event(A) ::= DELETE|INSERT(X). {A.a = @X; /*A-overwrites-X*/ A.b = 0;} +trigger_event(A) ::= UPDATE(X). {A.a = @X; /*A-overwrites-X*/ A.b = 0;} +trigger_event(A) ::= UPDATE OF idlist(X).{A.a = TK_UPDATE; A.b = X;} foreach_clause ::= . foreach_clause ::= FOR EACH ROW. @@ -1384,16 +1382,14 @@ when_clause(A) ::= WHEN expr(X). { A = X.pExpr; } %type trigger_cmd_list {TriggerStep*} %destructor trigger_cmd_list {sqlite3DeleteTriggerStep(pParse->db, $$);} -trigger_cmd_list(A) ::= trigger_cmd_list(Y) trigger_cmd(X) SEMI. { - assert( Y!=0 ); - Y->pLast->pNext = X; - Y->pLast = X; - A = Y; +trigger_cmd_list(A) ::= trigger_cmd_list(A) trigger_cmd(X) SEMI. { + assert( A!=0 ); + A->pLast->pNext = X; + A->pLast = X; } -trigger_cmd_list(A) ::= trigger_cmd(X) SEMI. { - assert( X!=0 ); - X->pLast = X; - A = X; +trigger_cmd_list(A) ::= trigger_cmd(A) SEMI. { + assert( A!=0 ); + A->pLast = A; } // Disallow qualified table names on INSERT, UPDATE, and DELETE statements @@ -1401,7 +1397,7 @@ trigger_cmd_list(A) ::= trigger_cmd(X) SEMI. { // the same database as the table that the trigger fires on. // %type trnm {Token} -trnm(A) ::= nm(X). {A = X;} +trnm(A) ::= nm(A). trnm(A) ::= nm DOT nm(X). { A = X; sqlite3ErrorMsg(pParse, @@ -1432,35 +1428,34 @@ tridxby ::= NOT INDEXED. { // UPDATE trigger_cmd(A) ::= UPDATE orconf(R) trnm(X) tridxby SET setlist(Y) where_opt(Z). - { A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R); } + {A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R);} // INSERT trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) idlist_opt(F) select(S). - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);} + {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);/*A-overwrites-R*/} // DELETE trigger_cmd(A) ::= DELETE FROM trnm(X) tridxby where_opt(Y). - {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y);} + {A = sqlite3TriggerDeleteStep(pParse->db, &X, Y);} // SELECT -trigger_cmd(A) ::= select(X). {A = sqlite3TriggerSelectStep(pParse->db, X); } +trigger_cmd(A) ::= select(X). + {A = sqlite3TriggerSelectStep(pParse->db, X); /*A-overwrites-X*/} // The special RAISE expression that may occur in trigger programs expr(A) ::= RAISE(X) LP IGNORE RP(Y). { + spanSet(&A,&X,&Y); /*A-overwrites-X*/ A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); if( A.pExpr ){ A.pExpr->affinity = OE_Ignore; } - A.zStart = X.z; - A.zEnd = &Y.z[Y.n]; } expr(A) ::= RAISE(X) LP raisetype(T) COMMA nm(Z) RP(Y). { + spanSet(&A,&X,&Y); /*A-overwrites-X*/ A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &Z); if( A.pExpr ) { A.pExpr->affinity = (char)T; } - A.zStart = X.z; - A.zEnd = &Y.z[Y.n]; } %endif !SQLITE_OMIT_TRIGGER @@ -1512,7 +1507,9 @@ cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). { sqlite3AlterRenameTable(pParse,X,&Z); } -cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column(Y). { +cmd ::= ALTER TABLE add_column_fullname + ADD kwcolumn_opt columnname(Y) carglist. { + Y.n = (int)(pParse->sLastToken.z-Y.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &Y); } add_column_fullname ::= fullname(X). { @@ -1556,9 +1553,9 @@ with(A) ::= WITH wqlist(W). { A = W; } with(A) ::= WITH RECURSIVE wqlist(W). { A = W; } wqlist(A) ::= nm(X) eidlist_opt(Y) AS LP select(Z) RP. { - A = sqlite3WithAdd(pParse, 0, &X, Y, Z); + A = sqlite3WithAdd(pParse, 0, &X, Y, Z); /*A-overwrites-X*/ } -wqlist(A) ::= wqlist(W) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. { - A = sqlite3WithAdd(pParse, W, &X, Y, Z); +wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, A, &X, Y, Z); } %endif SQLITE_OMIT_CTE diff --git a/src/pcache.c b/src/pcache.c index 5ac9d34a..68f4b51d 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -14,7 +14,29 @@ #include "sqliteInt.h" /* -** A complete page cache is an instance of this structure. +** A complete page cache is an instance of this structure. Every +** entry in the cache holds a single page of the database file. The +** btree layer only operates on the cached copy of the database pages. +** +** A page cache entry is "clean" if it exactly matches what is currently +** on disk. A page is "dirty" if it has been modified and needs to be +** persisted to disk. +** +** pDirty, pDirtyTail, pSynced: +** All dirty pages are linked into the doubly linked list using +** PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order +** such that p was added to the list more recently than p->pDirtyNext. +** PCache.pDirty points to the first (newest) element in the list and +** pDirtyTail to the last (oldest). +** +** The PCache.pSynced variable is used to optimize searching for a dirty +** page to eject from the cache mid-transaction. It is better to eject +** a page that does not require a journal sync than one that does. +** Therefore, pSynced is maintained to that it *almost* always points +** to either the oldest page in the pDirty/pDirtyTail list that has a +** clear PGHDR_NEED_SYNC flag or to a page that is older than this one +** (so that the right page to eject can be found by following pDirtyPrev +** pointers). */ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ @@ -31,6 +53,95 @@ struct PCache { sqlite3_pcache *pCache; /* Pluggable cache module */ }; +/********************************** Test and Debug Logic **********************/ +/* +** Debug tracing macros. Enable by by changing the "0" to "1" and +** recompiling. +** +** When sqlite3PcacheTrace is 1, single line trace messages are issued. +** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries +** is displayed for many operations, resulting in a lot of output. +*/ +#if defined(SQLITE_DEBUG) && 0 + int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ + int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ +# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} + void pcacheDump(PCache *pCache){ + int N; + int i, j; + sqlite3_pcache_page *pLower; + PgHdr *pPg; + unsigned char *a; + + if( sqlite3PcacheTrace<2 ) return; + if( pCache->pCache==0 ) return; + N = sqlite3PcachePagecount(pCache); + if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; + for(i=1; i<=N; i++){ + pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); + if( pLower==0 ) continue; + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf("\n"); + if( pPg->pPage==0 ){ + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); + } + } + } + #else +# define pcacheTrace(X) +# define pcacheDump(X) +#endif + +/* +** Check invariants on a PgHdr entry. Return true if everything is OK. +** Return false if any invariant is violated. +** +** This routine is for use inside of assert() statements only. For +** example: +** +** assert( sqlite3PcachePageSanity(pPg) ); +*/ +#if SQLITE_DEBUG +int sqlite3PcachePageSanity(PgHdr *pPg){ + PCache *pCache; + assert( pPg!=0 ); + assert( pPg->pgno>0 ); /* Page number is 1 or more */ + pCache = pPg->pCache; + assert( pCache!=0 ); /* Every page has an associated PCache */ + if( pPg->flags & PGHDR_CLEAN ){ + assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ + assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ + assert( pCache->pDirtyTail!=pPg ); + } + /* WRITEABLE pages must also be DIRTY */ + if( pPg->flags & PGHDR_WRITEABLE ){ + assert( pPg->flags & PGHDR_DIRTY ); /* WRITEABLE implies DIRTY */ + } + /* NEED_SYNC can be set independently of WRITEABLE. This can happen, + ** for example, when using the sqlite3PagerDontWrite() optimization: + ** (1) Page X is journalled, and gets WRITEABLE and NEED_SEEK. + ** (2) Page X moved to freelist, WRITEABLE is cleared + ** (3) Page X reused, WRITEABLE is set again + ** If NEED_SYNC had been cleared in step 2, then it would not be reset + ** in step 3, and page might be written into the database without first + ** syncing the rollback journal, which might cause corruption on a power + ** loss. + ** + ** Another example is when the database page size is smaller than the + ** disk sector size. When any page of a sector is journalled, all pages + ** in that sector are marked NEED_SYNC even if they are still CLEAN, just + ** in case they are later modified, since all pages in the same sector + ** must be journalled and synced before any of those pages can be safely + ** written. + */ + return 1; +} +#endif /* SQLITE_DEBUG */ + + /********************************** Linked List Management ********************/ /* Allowed values for second argument to pcacheManageDirtyList() */ @@ -47,17 +158,16 @@ struct PCache { static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ PCache *p = pPage->pCache; + pcacheTrace(("%p.DIRTYLIST.%s %d\n", p, + addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT", + pPage->pgno)); if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); assert( pPage->pDirtyPrev || pPage==p->pDirty ); /* Update the PCache1.pSynced variable if necessary. */ if( p->pSynced==pPage ){ - PgHdr *pSynced = pPage->pDirtyPrev; - while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ - pSynced = pSynced->pDirtyPrev; - } - p->pSynced = pSynced; + p->pSynced = pPage->pDirtyPrev; } if( pPage->pDirtyNext ){ @@ -69,10 +179,15 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( pPage->pDirtyPrev ){ pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; }else{ + /* If there are now no dirty pages in the cache, set eCreate to 2. + ** This is an optimization that allows sqlite3PcacheFetch() to skip + ** searching for a dirty page to eject from the cache when it might + ** otherwise have to. */ assert( pPage==p->pDirty ); p->pDirty = pPage->pDirtyNext; - if( p->pDirty==0 && p->bPurgeable ){ - assert( p->eCreate==1 ); + assert( p->bPurgeable || p->eCreate==2 ); + if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ + assert( p->bPurgeable==0 || p->eCreate==1 ); p->eCreate = 2; } } @@ -94,10 +209,19 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ } } p->pDirty = pPage; - if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + + /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set + ** pSynced to point to it. Checking the NEED_SYNC flag is an + ** optimization, as if pSynced points to a page with the NEED_SYNC + ** flag set sqlite3PcacheFetchStress() searches through all newer + ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ + if( !p->pSynced + && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ + ){ p->pSynced = pPage; } } + pcacheDump(p); } /* @@ -106,7 +230,9 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ */ static void pcacheUnpin(PgHdr *p){ if( p->pCache->bPurgeable ){ + pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno)); sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); + pcacheDump(p->pCache); } } @@ -176,6 +302,7 @@ int sqlite3PcacheOpen( p->pStress = pStress; p->szCache = 100; p->szSpill = 1; + pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable)); return sqlite3PcacheSetPageSize(p, szPage); } @@ -191,13 +318,14 @@ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), pCache->bPurgeable ); - if( pNew==0 ) return SQLITE_NOMEM; + if( pNew==0 ) return SQLITE_NOMEM_BKPT; sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); if( pCache->pCache ){ sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } pCache->pCache = pNew; pCache->szPage = szPage; + pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage)); } return SQLITE_OK; } @@ -232,11 +360,13 @@ sqlite3_pcache_page *sqlite3PcacheFetch( int createFlag /* If true, create page if it does not exist already */ ){ int eCreate; + sqlite3_pcache_page *pRes; assert( pCache!=0 ); assert( pCache->pCache!=0 ); assert( createFlag==3 || createFlag==0 ); assert( pgno>0 ); + assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); /* eCreate defines what to do if the page does not exist. ** 0 Do not allocate a new page. (createFlag==0) @@ -249,12 +379,15 @@ sqlite3_pcache_page *sqlite3PcacheFetch( assert( eCreate==0 || eCreate==1 || eCreate==2 ); assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); - return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + createFlag?" create":"",pRes)); + return pRes; } /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new -** page because new clean pages are available for reuse and the cache +** page because no clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to @@ -276,7 +409,11 @@ int sqlite3PcacheFetchStress( ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC ** cleared), but if that is not possible settle for any other ** unreferenced dirty page. - */ + ** + ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC + ** flag is currently referenced, then the following may leave pSynced + ** set incorrectly (pointing to other than the LRU page with NEED_SYNC + ** cleared). This is Ok, as pSynced is just an optimization. */ for(pPg=pCache->pSynced; pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); pPg=pPg->pDirtyPrev @@ -294,14 +431,16 @@ int sqlite3PcacheFetchStress( sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), numberOfCachePages(pCache)); #endif + pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); rc = pCache->xStress(pCache->pStress, pPg); + pcacheDump(pCache); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ return rc; } } } *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); - return *ppPage==0 ? SQLITE_NOMEM : SQLITE_OK; + return *ppPage==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; } /* @@ -322,7 +461,7 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( assert( pPage!=0 ); pPgHdr = (PgHdr*)pPage->pExtra; assert( pPgHdr->pPage==0 ); - memset(pPgHdr, 0, sizeof(PgHdr)); + memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); pPgHdr->pPage = pPage; pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; @@ -354,6 +493,7 @@ PgHdr *sqlite3PcacheFetchFinish( } pCache->nRefSum++; pPgHdr->nRef++; + assert( sqlite3PcachePageSanity(pPgHdr) ); return pPgHdr; } @@ -367,8 +507,11 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); - }else if( p->pDirtyPrev!=0 ){ - /* Move the page to the head of the dirty list. */ + }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Move the page to the head of the dirty list. If p->pDirtyPrev==0, + ** then page p is already at the head of the dirty list and the + ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE + ** tag above. */ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } @@ -379,6 +522,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ */ void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); + assert( sqlite3PcachePageSanity(p) ); p->nRef++; p->pCache->nRefSum++; } @@ -390,6 +534,7 @@ void sqlite3PcacheRef(PgHdr *p){ */ void sqlite3PcacheDrop(PgHdr *p){ assert( p->nRef==1 ); + assert( sqlite3PcachePageSanity(p) ); if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } @@ -403,13 +548,16 @@ void sqlite3PcacheDrop(PgHdr *p){ */ void sqlite3PcacheMakeDirty(PgHdr *p){ assert( p->nRef>0 ); - if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ + assert( sqlite3PcachePageSanity(p) ); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ p->flags &= ~PGHDR_DONT_WRITE; if( p->flags & PGHDR_CLEAN ){ p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); } + assert( sqlite3PcachePageSanity(p) ); } } @@ -418,11 +566,14 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ ** make it so. */ void sqlite3PcacheMakeClean(PgHdr *p){ - if( (p->flags & PGHDR_DIRTY) ){ + assert( sqlite3PcachePageSanity(p) ); + if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){ assert( (p->flags & PGHDR_CLEAN)==0 ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); if( p->nRef==0 ){ pcacheUnpin(p); } @@ -434,11 +585,24 @@ void sqlite3PcacheMakeClean(PgHdr *p){ */ void sqlite3PcacheCleanAll(PCache *pCache){ PgHdr *p; + pcacheTrace(("%p.CLEAN-ALL\n",pCache)); while( (p = pCache->pDirty)!=0 ){ sqlite3PcacheMakeClean(p); } } +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache)); + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -457,6 +621,8 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; assert( p->nRef>0 ); assert( newPgno>0 ); + assert( sqlite3PcachePageSanity(p) ); + pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ @@ -477,6 +643,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ if( pCache->pCache ){ PgHdr *p; PgHdr *pNext; + pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno)); for(p=pCache->pDirty; p; p=pNext){ pNext = p->pDirtyNext; /* This routine never gets call with a positive pgno except right @@ -484,7 +651,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ + if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } @@ -507,6 +674,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ */ void sqlite3PcacheClose(PCache *pCache){ assert( pCache->pCache!=0 ); + pcacheTrace(("%p.CLOSE\n",pCache)); sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } @@ -519,29 +687,31 @@ void sqlite3PcacheClear(PCache *pCache){ /* ** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pDirtyPrev pointers. +** Do not bother fixing the pDirtyPrev pointers. */ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ PgHdr result, *pTail; pTail = &result; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ if( pA->pgnopgno ){ pTail->pDirty = pA; pTail = pA; pA = pA->pDirty; + if( pA==0 ){ + pTail->pDirty = pB; + break; + } }else{ pTail->pDirty = pB; pTail = pB; pB = pB->pDirty; + if( pB==0 ){ + pTail->pDirty = pA; + break; + } } } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } return result.pDirty; } @@ -582,7 +752,8 @@ static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ } p = a[0]; for(i=1; ipDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; +} #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* diff --git a/src/pcache.h b/src/pcache.h index 475c04c0..9012edf2 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -26,7 +26,7 @@ struct PgHdr { sqlite3_pcache_page *pPage; /* Pcache object page handle */ void *pData; /* Page data */ void *pExtra; /* Extra content */ - PgHdr *pDirty; /* Transient list of dirty pages */ + PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES @@ -51,11 +51,10 @@ struct PgHdr { #define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ #define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before ** writing this page to the database */ -#define PGHDR_NEED_READ 0x010 /* Content is unread */ -#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ -#define PGHDR_MMAP 0x040 /* This is an mmap page object */ +#define PGHDR_DONT_WRITE 0x010 /* Do not write content to disk */ +#define PGHDR_MMAP 0x020 /* This is an mmap page object */ -#define PGHDR_WAL_APPEND 0x080 /* Appended to wal file */ +#define PGHDR_WAL_APPEND 0x040 /* Appended to wal file */ /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); @@ -99,6 +98,7 @@ void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ void sqlite3PcacheMove(PgHdr*, Pgno); @@ -137,6 +137,11 @@ int sqlite3PcachePagecount(PCache*); void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); #endif +#if defined(SQLITE_DEBUG) +/* Check invariants on a PgHdr object */ +int sqlite3PcachePageSanity(PgHdr*); +#endif + /* Set and get the suggested cache-size for the specified pager-cache. ** ** If no global maximum is configured, then the system attempts to limit @@ -173,4 +178,7 @@ void sqlite3PCacheSetDefault(void); int sqlite3HeaderSizePcache(void); int sqlite3HeaderSizePcache1(void); +/* Number of dirty pages as a percentage of the configured cache size */ +int sqlite3PCachePercentDirty(PCache*); + #endif /* _PCACHE_H_ */ diff --git a/src/pcache1.c b/src/pcache1.c index 7147f6a7..110d7ec6 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -279,7 +279,7 @@ static int pcache1InitBulk(PCache1 *pCache){ szBulk = -1024 * (i64)pcache1.nInitPage; } if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ - szBulk = pCache->szAlloc*pCache->nMax; + szBulk = pCache->szAlloc*(i64)pCache->nMax; } zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); sqlite3EndBenignMalloc(); @@ -348,7 +348,6 @@ static void *pcache1Alloc(int nByte){ ** Free an allocated buffer obtained from pcache1Alloc(). */ static void pcache1Free(void *p){ - int nFreed = 0; if( p==0 ) return; if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ PgFreeslot *pSlot; @@ -365,10 +364,13 @@ static void pcache1Free(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - nFreed = sqlite3MallocSize(p); - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); - sqlite3_mutex_leave(pcache1.mutex); + { + int nFreed = 0; + nFreed = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); + sqlite3_mutex_leave(pcache1.mutex); + } #endif sqlite3_free(p); } @@ -630,12 +632,30 @@ static void pcache1TruncateUnsafe( PCache1 *pCache, /* The cache to truncate */ unsigned int iLimit /* Drop pages with this pgno or larger */ ){ - TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ - unsigned int h; + TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h, iStop; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - for(h=0; hnHash; h++){ - PgHdr1 **pp = &pCache->apHash[h]; + assert( pCache->iMaxKey >= iLimit ); + assert( pCache->nHash > 0 ); + if( pCache->iMaxKey - iLimit < pCache->nHash ){ + /* If we are just shaving the last few pages off the end of the + ** cache, then there is no point in scanning the entire hash table. + ** Only scan those hash slots that might contain pages that need to + ** be removed. */ + h = iLimit % pCache->nHash; + iStop = pCache->iMaxKey % pCache->nHash; + TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ + }else{ + /* This is the general case where many pages are being removed. + ** It is necessary to scan the entire hash table */ + h = pCache->nHash/2; + iStop = h - 1; + } + for(;;){ + PgHdr1 **pp; PgHdr1 *pPage; + assert( hnHash ); + pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; @@ -644,11 +664,13 @@ static void pcache1TruncateUnsafe( pcache1FreePage(pPage); }else{ pp = &pPage->pNext; - TESTONLY( nPage++; ) + TESTONLY( if( nPage>=0 ) nPage++; ) } } + if( h==iStop ) break; + h = (h+1) % pCache->nHash; } - assert( pCache->nPage==nPage ); + assert( nPage<0 || pCache->nPage==(unsigned)nPage ); } /******************************************************************************/ @@ -688,8 +710,8 @@ static int pcache1Init(void *NotUsed){ #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); - pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); } #endif if( pcache1.separateCache @@ -1125,7 +1147,7 @@ static void pcache1Destroy(sqlite3_pcache *p){ PGroup *pGroup = pCache->pGroup; assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(pGroup); - pcache1TruncateUnsafe(pCache, 0); + if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); diff --git a/src/pragma.c b/src/pragma.c index c34d5421..e774de34 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -338,7 +338,7 @@ void sqlite3Pragma( } assert( pId2 ); - zDb = pId2->n>0 ? pDb->zName : 0; + zDb = pId2->n>0 ? pDb->zDbSName : 0; if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } @@ -993,6 +993,7 @@ void sqlite3Pragma( int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; + pDb->bSyncSet = 1; setAllPagerFlags(db); } } @@ -1029,7 +1030,7 @@ void sqlite3Pragma( ** compiler (eg. count_changes). So add an opcode to expire all ** compiled SQL statements after modifying a pragma value. */ - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + sqlite3VdbeAddOp0(v, OP_Expire); setAllPagerFlags(db); } break; @@ -1051,7 +1052,7 @@ void sqlite3Pragma( */ case PragTyp_TABLE_INFO: if( zRight ){ Table *pTab; - pTab = sqlite3FindTable(db, zRight, zDb); + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); if( pTab ){ static const char *azCol[] = { "cid", "name", "type", "notnull", "dflt_value", "pk" @@ -1076,12 +1077,13 @@ void sqlite3Pragma( }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } + assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); sqlite3VdbeMultiLoad(v, 1, "issisi", i-nHidden, pCol->zName, - pCol->zType ? pCol->zType : "", + sqlite3ColumnType(pCol,""), pCol->notNull ? 1 : 0, - pCol->zDflt, + pCol->pDflt ? pCol->pDflt->u.zToken : 0, k); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); } @@ -1102,14 +1104,14 @@ void sqlite3Pragma( sqlite3VdbeMultiLoad(v, 1, "ssii", pTab->zName, 0, - (int)sqlite3LogEstToInt(pTab->szTabRow), - (int)sqlite3LogEstToInt(pTab->nRowLogEst)); + pTab->szTabRow, + pTab->nRowLogEst); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeMultiLoad(v, 2, "sii", pIdx->zName, - (int)sqlite3LogEstToInt(pIdx->szIdxRow), - (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0])); + pIdx->szIdxRow, + pIdx->aiRowLogEst[0]); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } @@ -1189,10 +1191,10 @@ void sqlite3Pragma( setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; - assert( db->aDb[i].zName!=0 ); + assert( db->aDb[i].zDbSName!=0 ); sqlite3VdbeMultiLoad(v, 1, "iss", i, - db->aDb[i].zName, + db->aDb[i].zDbSName, sqlite3BtreeGetFilename(db->aDb[i].pBt)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -1332,12 +1334,10 @@ void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); sqlite3ColumnDefault(v, pTab, iKey, regRow); sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, - sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } - sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v); sqlite3VdbeGoto(v, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ @@ -1435,7 +1435,10 @@ void sqlite3Pragma( for(i=0; inDb; i++){ HashElem *x; Hash *pTbls; + int *aRoot; int cnt = 0; + int mxIdx = 0; + int nIdx; if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -1448,35 +1451,39 @@ void sqlite3Pragma( /* Do an integrity check of the B-Tree ** - ** Begin by filling registers 2, 3, ... with the root pages numbers + ** Begin by finding the root pages numbers ** for all tables and indices in the database. */ assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; - for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); - VdbeComment((v, "%s", pTab->zName)); - cnt++; - } + if( HasRowid(pTab) ) cnt++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; + } + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); + if( aRoot==0 ) break; + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); - VdbeComment((v, "%s", pIdx->zName)); - cnt++; + aRoot[cnt++] = pIdx->tnum; } } + aRoot[cnt] = 0; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, cnt+8 ); + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), P4_DYNAMIC); sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); @@ -1506,7 +1513,8 @@ void sqlite3Pragma( for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } - pParse->nMem = MAX(pParse->nMem, 8+j); + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); /* Verify that all NOT NULL columns really are NOT NULL */ @@ -1698,7 +1706,9 @@ void sqlite3Pragma( ** PRAGMA [schema.]user_version ** PRAGMA [schema.]user_version = ** - ** PRAGMA [schema.]freelist_count = + ** PRAGMA [schema.]freelist_count + ** + ** PRAGMA [schema.]data_version ** ** PRAGMA [schema.]application_id ** PRAGMA [schema.]application_id = @@ -1754,6 +1764,7 @@ void sqlite3Pragma( aOp[1].p3 = iCookie; sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + sqlite3VdbeReusable(v); } } break; @@ -1775,6 +1786,7 @@ void sqlite3Pragma( sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } + sqlite3VdbeReusable(v); } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ @@ -1910,15 +1922,15 @@ void sqlite3Pragma( Btree *pBt; const char *zState = "unknown"; int j; - if( db->aDb[i].zName==0 ) continue; + if( db->aDb[i].zDbSName==0 ) continue; pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zName, zState); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; diff --git a/src/prepare.c b/src/prepare.c index f74aa52e..74127bc7 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -33,7 +33,7 @@ static void corruptSchema( sqlite3DbFree(db, *pData->pzErrMsg); *pData->pzErrMsg = z; } - pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; + pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; } /* @@ -73,6 +73,7 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ ** structures that describe the table, index, or view. */ int rc; + u8 saved_iDb = db->init.iDb; sqlite3_stmt *pStmt; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ @@ -83,7 +84,8 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); - db->init.iDb = 0; + db->init.iDb = saved_iDb; + assert( saved_iDb==0 || (db->flags & SQLITE_Vacuum)!=0 ); if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); @@ -107,7 +109,7 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); + pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zDbSName); if( pIndex==0 ){ /* This can occur if there exists an index on a TEMP table which ** has the same name as another index on a permanent index. Since @@ -286,7 +288,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ char *zSql; zSql = sqlite3MPrintf(db, "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", - db->aDb[iDb].zName, zMasterName); + db->aDb[iDb].zDbSName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { sqlite3_xauth xAuth; @@ -307,7 +309,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ #endif } if( db->mallocFailed ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; sqlite3ResetAllSchemasOfConnection(db); } if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ @@ -516,18 +518,14 @@ static int sqlite3Prepare( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - Parse *pParse; /* Parsing context */ char *zErrMsg = 0; /* Error message */ int rc = SQLITE_OK; /* Result code */ int i; /* Loop counter */ + Parse sParse; /* Parsing context */ - /* Allocate the parsing context */ - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; - goto end_prepare; - } - pParse->pReprepare = pReprepare; + memset(&sParse, 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); + sParse.pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ assert( sqlite3_mutex_held(db->mutex) ); @@ -561,7 +559,7 @@ static int sqlite3Prepare( assert( sqlite3BtreeHoldsMutex(pBt) ); rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ - const char *zDb = db->aDb[i].zName; + const char *zDb = db->aDb[i].zDbSName; sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; @@ -571,8 +569,7 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); - pParse->db = db; - pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ + sParse.db = db; if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -585,61 +582,61 @@ static int sqlite3Prepare( } zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ - sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); - pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; + sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; sqlite3DbFree(db, zSqlCopy); }else{ - pParse->zTail = &zSql[nBytes]; + sParse.zTail = &zSql[nBytes]; } }else{ - sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3RunParser(&sParse, zSql, &zErrMsg); } - assert( 0==pParse->nQueryLoop ); + assert( 0==sParse.nQueryLoop ); - if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; - if( pParse->checkSchema ){ - schemaIsValid(pParse); + if( sParse.rc==SQLITE_DONE ) sParse.rc = SQLITE_OK; + if( sParse.checkSchema ){ + schemaIsValid(&sParse); } if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; + sParse.rc = SQLITE_NOMEM_BKPT; } if( pzTail ){ - *pzTail = pParse->zTail; + *pzTail = sParse.zTail; } - rc = pParse->rc; + rc = sParse.rc; #ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ + if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", "selectid", "order", "from", "detail" }; int iFirst, mx; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 4); + if( sParse.explain==2 ){ + sqlite3VdbeSetNumCols(sParse.pVdbe, 4); iFirst = 8; mx = 12; }else{ - sqlite3VdbeSetNumCols(pParse->pVdbe, 8); + sqlite3VdbeSetNumCols(sParse.pVdbe, 8); iFirst = 0; mx = 8; } for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, + sqlite3VdbeSetColName(sParse.pVdbe, i-iFirst, COLNAME_NAME, azColName[i], SQLITE_STATIC); } } #endif if( db->init.busy==0 ){ - Vdbe *pVdbe = pParse->pVdbe; - sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); + Vdbe *pVdbe = sParse.pVdbe; + sqlite3VdbeSetSql(pVdbe, zSql, (int)(sParse.zTail-zSql), saveSqlFlag); } - if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(pParse->pVdbe); + if( sParse.pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ + sqlite3VdbeFinalize(sParse.pVdbe); assert(!(*ppStmt)); }else{ - *ppStmt = (sqlite3_stmt*)pParse->pVdbe; + *ppStmt = (sqlite3_stmt*)sParse.pVdbe; } if( zErrMsg ){ @@ -650,16 +647,15 @@ static int sqlite3Prepare( } /* Delete any TriggerPrg structures allocated while parsing this statement. */ - while( pParse->pTriggerPrg ){ - TriggerPrg *pT = pParse->pTriggerPrg; - pParse->pTriggerPrg = pT->pNext; + while( sParse.pTriggerPrg ){ + TriggerPrg *pT = sParse.pTriggerPrg; + sParse.pTriggerPrg = pT->pNext; sqlite3DbFree(db, pT); } end_prepare: - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); + sqlite3ParserReset(&sParse); rc = sqlite3ApiExit(db, rc); assert( (rc&db->errMask)==rc ); return rc; diff --git a/src/printf.c b/src/printf.c index 238997f3..a888d50b 100644 --- a/src/printf.c +++ b/src/printf.c @@ -15,26 +15,26 @@ ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 1 /* Integer types. %d, %x, %o, and so forth */ -#define etFLOAT 2 /* Floating point. %f */ -#define etEXP 3 /* Exponentional notation. %e and %E */ -#define etGENERIC 4 /* Floating or exponential, depending on exponent. %g */ -#define etSIZE 5 /* Return number of characters processed so far. %n */ -#define etSTRING 6 /* Strings. %s */ -#define etDYNSTRING 7 /* Dynamically allocated strings. %z */ -#define etPERCENT 8 /* Percent symbol. %% */ -#define etCHARX 9 /* Characters. %c */ +#define etRADIX 0 /* Integer types. %d, %x, %o, and so forth */ +#define etFLOAT 1 /* Floating point. %f */ +#define etEXP 2 /* Exponentional notation. %e and %E */ +#define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 4 /* Return number of characters processed so far. %n */ +#define etSTRING 5 /* Strings. %s */ +#define etDYNSTRING 6 /* Dynamically allocated strings. %z */ +#define etPERCENT 7 /* Percent symbol. %% */ +#define etCHARX 8 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 10 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 11 /* Strings with '\'' doubled and enclosed in '', +#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 12 /* a pointer to a Token structure */ -#define etSRCLIST 13 /* a pointer to a SrcList */ -#define etPOINTER 14 /* The %p conversion */ -#define etSQLESCAPE3 15 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 16 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCLIST 12 /* a pointer to a SrcList */ +#define etPOINTER 13 /* The %p conversion */ +#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ -#define etINVALID 0 /* Any unrecognized conversion type */ +#define etINVALID 16 /* Any unrecognized conversion type */ /* @@ -189,7 +189,7 @@ void sqlite3VXPrintf( etByte flag_long; /* True if "l" flag is present */ etByte flag_longlong; /* True if the "ll" flag is present */ etByte done; /* Loop termination flag */ - etByte xtype = 0; /* Conversion paradigm */ + etByte xtype = etINVALID; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ u8 useIntern; /* Ok to use internal conversions (ex: %T) */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ diff --git a/src/resolve.c b/src/resolve.c index 81bb712a..dac73e5f 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -221,8 +221,8 @@ static int lookupName( zDb = 0; }else{ for(i=0; inDb; i++){ - assert( db->aDb[i].zName ); - if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + assert( db->aDb[i].zDbSName ); + if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ pSchema = db->aDb[i].pSchema; break; } @@ -400,6 +400,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } + if( sqlite3ExprVectorSize(pOrig)!=1 ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return WRC_Abort; + } resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); cnt = 1; pMatch = 0; @@ -623,7 +627,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* if( pSrcList==0 ) break; */ notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); - /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -646,19 +649,17 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ - int auth; /* Authorization to use the function */ int nId; /* Number of characters in function name */ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); - pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ - pDef = sqlite3FindFunction(pParse->db, zId, nId, -2, enc, 0); + pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); if( pDef==0 ){ no_such_func = 1; }else{ @@ -690,15 +691,17 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } } #ifndef SQLITE_OMIT_AUTHORIZATION - auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; + { + int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; } - pExpr->op = TK_NULL; - return WRC_Prune; } #endif if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ @@ -711,14 +714,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* Date/time functions that use 'now', and other functions like ** sqlite_version() that might change over time cannot be used ** in an index. */ - notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + notValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx); } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 ){ + }else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -763,6 +771,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pNC->ncFlags |= NC_VarSelect; } } break; @@ -771,6 +780,42 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } + case TK_BETWEEN: + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_IS: + case TK_ISNOT: { + int nLeft, nRight; + if( pParse->db->mallocFailed ) break; + assert( pExpr->pLeft!=0 ); + nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->op==TK_BETWEEN ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); + if( nRight==nLeft ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); + } + }else{ + assert( pExpr->pRight!=0 ); + nRight = sqlite3ExprVectorSize(pExpr->pRight); + } + if( nLeft!=nRight ){ + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_BETWEEN ); + sqlite3ErrorMsg(pParse, "row value misused"); + } + break; + } } return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; } diff --git a/src/rowset.c b/src/rowset.c index c2e73ed7..aa81607b 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -57,8 +57,9 @@ ** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST ** primitives are constant time. The cost of DESTROY is O(N). ** -** There is an added cost of O(N) when switching between TEST and -** SMALLEST primitives. +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. */ #include "sqliteInt.h" @@ -179,7 +180,9 @@ void sqlite3RowSetClear(RowSet *p){ */ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); - if( p->nFresh==0 ){ + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ struct RowSetChunk *pNew; pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ @@ -213,7 +216,9 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; @@ -237,28 +242,26 @@ static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pTail; pTail = &head; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ assert( pA->pRight==0 || pA->v<=pA->pRight->v ); assert( pB->pRight==0 || pB->v<=pB->pRight->v ); - if( pA->vv ){ - pTail->pRight = pA; + if( pA->v<=pB->v ){ + if( pA->vv ) pTail = pTail->pRight = pA; pA = pA->pRight; - pTail = pTail->pRight; - }else if( pB->vv ){ - pTail->pRight = pB; - pB = pB->pRight; - pTail = pTail->pRight; + if( pA==0 ){ + pTail->pRight = pB; + break; + } }else{ - pA = pA->pRight; + pTail = pTail->pRight = pB; + pB = pB->pRight; + if( pB==0 ){ + pTail->pRight = pA; + break; + } } } - if( pA ){ - assert( pA->pRight==0 || pA->v<=pA->pRight->v ); - pTail->pRight = pA; - }else{ - assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); - pTail->pRight = pB; - } return head.pRight; } @@ -281,9 +284,10 @@ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ aBucket[i] = pIn; pIn = pNext; } - pIn = 0; - for(i=0; i1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch causes a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ p = *ppList; *ppList = p->pRight; p->pLeft = p->pRight = 0; - return p; } - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ - return pLeft; - } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); return p; } @@ -378,59 +388,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ return p; } -/* -** Take all the entries on p->pEntry and on the trees in p->pForest and -** sort them all together into one big ordered list on p->pEntry. -** -** This routine should only be called once in the life of a RowSet. -*/ -static void rowSetToList(RowSet *p){ - - /* This routine is called only once */ - assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); - - if( (p->rsFlags & ROWSET_SORTED)==0 ){ - p->pEntry = rowSetEntrySort(p->pEntry); - } - - /* While this module could theoretically support it, sqlite3RowSetNext() - ** is never called after sqlite3RowSetText() for the same RowSet. So - ** there is never a forest to deal with. Should this change, simply - ** remove the assert() and the #if 0. */ - assert( p->pForest==0 ); -#if 0 - while( p->pForest ){ - struct RowSetEntry *pTree = p->pForest->pLeft; - if( pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(pTree, &pHead, &pTail); - p->pEntry = rowSetEntryMerge(p->pEntry, pHead); - } - p->pForest = p->pForest->pRight; - } -#endif - p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ -} - /* ** Extract the smallest element from the RowSet. ** Write the element into *pRowid. Return 1 on success. Return ** 0 if the RowSet is already empty. ** ** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. */ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ /* Merge the forest into a single sorted list on first call */ - if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ sqlite3RowSetClear(p); } return 1; @@ -453,13 +441,15 @@ int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ /* This routine is never called after sqlite3RowSetNext() */ assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); - /* Sort entries into the forest on the first test of a new batch + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. */ - if( iBatch!=pRowSet->iBatch ){ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ p = pRowSet->pEntry; if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; - if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entiries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ diff --git a/src/select.c b/src/select.c index c3132c23..199e13f1 100644 --- a/src/select.c +++ b/src/select.c @@ -56,6 +56,7 @@ struct SortCtx { int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ + u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -74,7 +75,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); - sqlite3WithDelete(db, p->pWith); + if( p->pWith ) sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFree(db, p); p = pPrior; bFree = 1; @@ -87,7 +88,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; pDest->iSDParm = iParm; - pDest->affSdst = 0; + pDest->zAffSdst = 0; pDest->iSdst = 0; pDest->nSdst = 0; } @@ -105,7 +106,7 @@ Select *sqlite3SelectNew( ExprList *pGroupBy, /* the GROUP BY clause */ Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ - u16 selFlags, /* Flag parameters, such as SF_Distinct */ + u32 selFlags, /* Flag parameters, such as SF_Distinct */ Expr *pLimit, /* LIMIT value. NULL means not used */ Expr *pOffset /* OFFSET value. NULL means no offset */ ){ @@ -169,7 +170,7 @@ void sqlite3SelectSetName(Select *p, const char *zName){ ** Delete the given Select structure and all of its substructures. */ void sqlite3SelectDelete(sqlite3 *db, Select *p){ - clearSelect(db, p, 1); + if( p ) clearSelect(db, p, 1); } /* @@ -589,9 +590,30 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( iLimit ){ int addr; + int r1 = 0; + /* Fill the sorter until it contains LIMIT+OFFSET entries. (The iLimit + ** register is initialized with value of LIMIT+OFFSET.) After the sorter + ** fills up, delete the least entry in the sorter after each insert. + ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */ addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + r1 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1); + VdbeComment((v, "seq")); + } sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + /* If the inner loop is driven by an index such that values from + ** the same iteration of the inner loop are in sorted order, then + ** immediately jump to the next iteration of an inner loop if the + ** entry from the current iteration does not fit into the top + ** LIMIT+OFFSET entries of the sorter. */ + int iBrk = sqlite3VdbeCurrentAddr(v) + 2; + sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addr); } } @@ -637,30 +659,6 @@ static void codeDistinct( sqlite3ReleaseTempReg(pParse, r1); } -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Generate an error message when a SELECT is used within a subexpression -** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error used to occur -** in multiple places. (The error only occurs in one place now, but we -** retain the subroutine to minimize code disruption.) -*/ -static int checkForMultiColumnSelectError( - Parse *pParse, /* Parse context. */ - SelectDest *pDest, /* Destination of SELECT results */ - int nExpr /* Number of result columns returned by SELECT */ -){ - int eDest = pDest->eDest; - if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ - sqlite3ErrorMsg(pParse, "only a single result allowed for " - "a SELECT that is part of an expression"); - return 1; - }else{ - return 0; - } -} -#endif - /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -870,19 +868,19 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - assert( nResultCol==1 ); - pDest->affSdst = - sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); + pushOntoSorter( + pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regResult, 1); + assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + r1, pDest->zAffSdst, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); } @@ -898,13 +896,14 @@ static void selectInnerLoop( } /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** store the results in the appropriate memory cell or array of + ** memory cells and break out of the scan loop. */ case SRT_Mem: { - assert( nResultCol==1 ); + assert( nResultCol==pDest->nSdst ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); + pushOntoSorter( + pParse, pSort, p, regResult, regResult, nResultCol, nPrefixReg); }else{ assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ @@ -1006,7 +1005,7 @@ static void selectInnerLoop( */ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ int nExtra = (N+X)*(sizeof(CollSeq*)+1); - KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra); + KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; p->nField = (u16)N; @@ -1028,7 +1027,7 @@ void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFree(0, p); + if( p->nRef==0 ) sqlite3DbFree(p->db, p); } } @@ -1219,21 +1218,21 @@ static void generateSortTail( sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ regRowid = 0; regRow = pDest->iSdst; nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); - regRow = sqlite3GetTempReg(pParse); - nSortData = 1; + regRow = sqlite3GetTempRange(pParse, nColumn); + nSortData = nColumn; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; if( pSort->labelBkOut ){ - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); @@ -1261,16 +1260,14 @@ static void generateSortTail( } #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { - assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, - &pDest->affSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regRow, 1); + assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, + pDest->zAffSdst, nColumn); + sqlite3ExprCacheAffinityChange(pParse, regRow, nColumn); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); break; } case SRT_Mem: { - assert( nColumn==1 ); - sqlite3ExprCodeMove(pParse, regRow, iParm, 1); /* The LIMIT clause will terminate the loop for us */ break; } @@ -1289,7 +1286,11 @@ static void generateSortTail( } } if( regRowid ){ - sqlite3ReleaseTempReg(pParse, regRow); + if( eDest==SRT_Set ){ + sqlite3ReleaseTempRange(pParse, regRow, nColumn); + }else{ + sqlite3ReleaseTempReg(pParse, regRow); + } sqlite3ReleaseTempReg(pParse, regRowid); } /* The bottom of the loop @@ -1429,20 +1430,20 @@ static const char *columnTypeImpl( zType = "INTEGER"; zOrigCol = "rowid"; }else{ - zType = pTab->aCol[iCol].zType; zOrigCol = pTab->aCol[iCol].zName; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } zOrigTab = pTab->zName; if( pNC->pParse ){ int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOrigDb = pNC->pParse->db->aDb[iDb].zName; + zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; } #else if( iCol<0 ){ zType = "INTEGER"; }else{ - zType = pTab->aCol[iCol].zType; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } #endif @@ -1688,7 +1689,7 @@ int sqlite3ColumnsFromExprList( sqlite3DbFree(db, aCol); *paCol = 0; *pnCol = 0; - return SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; } return SQLITE_OK; } @@ -1704,7 +1705,7 @@ int sqlite3ColumnsFromExprList( ** This routine requires that all identifiers in the SELECT ** statement be resolved. */ -static void selectAddColumnTypeAndCollation( +void sqlite3SelectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ Table *pTab, /* Add column type information to this table */ Select *pSelect /* SELECT used to determine types and collations */ @@ -1726,13 +1727,20 @@ static void selectAddColumnTypeAndCollation( sNC.pSrcList = pSelect->pSrc; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + const char *zType; + int n, m; p = a[i].pExpr; - if( pCol->zType==0 ){ - pCol->zType = sqlite3DbStrDup(db, - columnType(&sNC, p,0,0,0, &pCol->szEst)); - } + zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst); szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); + if( zType && (m = sqlite3Strlen30(zType))>0 ){ + n = sqlite3Strlen30(pCol->zName); + pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); + if( pCol->zName ){ + memcpy(&pCol->zName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + } + } if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ @@ -1769,7 +1777,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -1782,20 +1790,20 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ ** Get a VDBE for the given parser context. Create a new one if necessary. ** If an error occurs, return NULL and leave a message in pParse. */ -Vdbe *sqlite3GetVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe; - if( v==0 ){ - v = pParse->pVdbe = sqlite3VdbeCreate(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Init); - if( pParse->pToplevel==0 - && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) - ){ - pParse->okConstFactor = 1; - } - +static SQLITE_NOINLINE Vdbe *allocVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(pParse); + if( v ) sqlite3VdbeAddOp2(v, OP_Init, 0, 1); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; } return v; } +Vdbe *sqlite3GetVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe; + return v ? v : allocVdbe(pParse); +} /* @@ -1845,8 +1853,9 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeGoto(v, iBreak); - }else if( n>=0 && p->nSelectRow>(u64)n ){ - p->nSelectRow = n; + }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ + p->nSelectRow = sqlite3LogEst((u64)n); + p->selFlags |= SF_FixedLimit; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); @@ -2224,7 +2233,6 @@ static int multiSelect( if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); - sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -2287,12 +2295,12 @@ static int multiSelect( testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; - p->nSelectRow += pPrior->nSelectRow; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); if( pPrior->pLimit && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && nLimit>0 && p->nSelectRow > (u64)nLimit + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) ){ - p->nSelectRow = nLimit; + p->nSelectRow = sqlite3LogEst((u64)nLimit); } if( addr ){ sqlite3VdbeJumpHere(v, addr); @@ -2364,7 +2372,9 @@ static int multiSelect( pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; - if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; @@ -2499,7 +2509,7 @@ static int multiSelect( nCol = p->pEList->nExpr; pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM_BKPT; goto multi_select_end; } for(i=0, apColl=pKeyInfo->aColl; inSdst==1 || pParse->nErr>0 ); - pDest->affSdst = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); + testcase( pIn->nSdst>1 ); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + r1, pDest->zAffSdst, pIn->nSdst); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; @@ -2854,7 +2861,7 @@ static int multiSelectOrderBy( } if( j==nOrderBy ){ Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return SQLITE_NOMEM; + if( pNew==0 ) return SQLITE_NOMEM_BKPT; pNew->flags |= EP_IntValue; pNew->u.iValue = i; pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); @@ -3001,7 +3008,7 @@ static int multiSelectOrderBy( addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); sqlite3VdbeGoto(v, addrEofA); - p->nSelectRow += pPrior->nSelectRow; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } /* Generate a subroutine to run when the results from select B @@ -3688,12 +3695,13 @@ static int flattenSubquery( assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, - sqlite3ExprDup(db, pSub->pHaving, 0)); + pParent->pHaving = sqlite3ExprAnd(db, + sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving + ); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); } substSelect(db, pParent, iParent, pSub->pEList, 0); @@ -3776,12 +3784,18 @@ static int pushDownWhereTerms( ){ Expr *pNew; int nChng = 0; + Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; - if( (pSubq->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - return 0; /* restrictions (1) and (2) */ + for(pX=pSubq; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + testcase( pX->selFlags & SF_Aggregate ); + testcase( pX->selFlags & SF_Recursive ); + testcase( pX!=pSubq ); + return 0; /* restrictions (1) and (2) */ + } } if( pSubq->pLimit!=0 ){ - return 0; /* restriction (3) */ + return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); @@ -4091,7 +4105,7 @@ static int withExpand( pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); - if( db->mallocFailed ) return SQLITE_NOMEM; + if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; assert( pFrom->pSelect ); /* Check if this is a recursive CTE. */ @@ -4377,7 +4391,7 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; + zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; } for(j=0; jnCol; j++){ char *zName = pTab->aCol[j].zName; @@ -4551,7 +4565,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Select *pSel = pFrom->pSelect; if( pSel ){ while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } @@ -4860,16 +4874,6 @@ int sqlite3Select( } #endif - - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, p->pEList->nExpr) ){ - goto select_end; - } -#endif - /* Try to flatten subqueries in the FROM clause up into the main query */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) @@ -4971,10 +4975,24 @@ int sqlite3Select( } /* Generate code to implement the subquery + ** + ** The subquery is implemented as a co-routine if all of these are true: + ** (1) The subquery is guaranteed to be the outer loop (so that it + ** does not need to be computed more than once) + ** (2) The ALL keyword after SELECT is omitted. (Applications are + ** allowed to say "SELECT ALL" instead of just "SELECT" to disable + ** the use of co-routines.) + ** (3) Co-routines are not disabled using sqlite3_test_control() + ** with SQLITE_TESTCTRL_OPTIMIZATIONS. + ** + ** TODO: Are there other reasons beside (1) to use a co-routine + ** implementation? */ - if( pTabList->nSrc==1 - && (p->selFlags & SF_All)==0 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) + if( i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ + && (p->selFlags & SF_All)==0 /* (2) */ + && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -4987,7 +5005,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->pTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeEndCoroutine(v, pItem->regReturn); @@ -5010,7 +5028,7 @@ int sqlite3Select( /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); @@ -5018,7 +5036,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); @@ -5069,6 +5087,13 @@ int sqlite3Select( ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif } /* If there is an ORDER BY clause, then create an ephemeral index to @@ -5101,7 +5126,7 @@ int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = LARGEST_INT64; + p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); @@ -5125,10 +5150,12 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + assert( WHERE_USE_LIMIT==SF_FixedLimit ); + wctrlFlags |= p->selFlags & SF_FixedLimit; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, 0); + p->pEList, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -5138,6 +5165,7 @@ int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } @@ -5188,9 +5216,11 @@ int sqlite3Select( for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->u.x.iAlias = 0; } - if( p->nSelectRow>100 ) p->nSelectRow = 100; + assert( 66==sqlite3LogEst(100) ); + if( p->nSelectRow>66 ) p->nSelectRow = 66; }else{ - p->nSelectRow = 1; + assert( 0==sqlite3LogEst(1) ); + p->nSelectRow = 0; } /* If there is both a GROUP BY and an ORDER BY clause and they are diff --git a/src/shell.c b/src/shell.c index 85e36a88..732ef628 100644 --- a/src/shell.c +++ b/src/shell.c @@ -90,7 +90,7 @@ #else -# define shell_read_history(X) +# define shell_read_history(X) # define shell_write_history(X) # define shell_stifle_history(X) @@ -136,6 +136,16 @@ #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) +#if defined(_WIN32) || defined(WIN32) +#include + +/* string conversion routines only needed on Win32 */ +extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); +extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); +extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); +extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); +#endif + /* On Windows, we normally run with output mode of TEXT so that \n characters ** are automatically translated into \r\n. However, this behavior needs ** to be disabled in some cases (ex: when generating CSV output and when @@ -143,17 +153,17 @@ ** routines take care of that. */ #if defined(_WIN32) || defined(WIN32) -static void setBinaryMode(FILE *out){ - fflush(out); - _setmode(_fileno(out), _O_BINARY); +static void setBinaryMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_BINARY); } -static void setTextMode(FILE *out){ - fflush(out); - _setmode(_fileno(out), _O_TEXT); +static void setTextMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_TEXT); } #else -# define setBinaryMode(X) -# define setTextMode(X) +# define setBinaryMode(X,Y) +# define setTextMode(X,Y) #endif @@ -204,7 +214,7 @@ static void beginTimer(void){ /* Return the difference of two time_structs in seconds */ static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + (double)(pEnd->tv_sec - pStart->tv_sec); } @@ -229,8 +239,6 @@ static void endTimer(void){ #elif (defined(_WIN32) || defined(WIN32)) -#include - /* Saved resource information for the beginning of an operation */ static HANDLE hProcess; static FILETIME ftKernelBegin; @@ -261,7 +269,7 @@ static int hasTimer(void){ if( NULL != getProcessTimesAddr ){ return 1; } - FreeLibrary(hinstLib); + FreeLibrary(hinstLib); } } } @@ -307,7 +315,7 @@ static void endTimer(void){ #define HAS_TIMER hasTimer() #else -#define BEGIN_TIMER +#define BEGIN_TIMER #define END_TIMER #define HAS_TIMER 0 #endif @@ -361,6 +369,38 @@ static char *Argv0; static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ +/* +** Render output like fprintf(). Except, if the output is going to the +** console and if this is running on a Windows machine, translate the +** output from UTF-8 into MBCS. +*/ +#if defined(_WIN32) || defined(WIN32) +void utf8_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + if( stdout_is_console && (out==stdout || out==stderr) ){ + char *z1 = sqlite3_vmprintf(zFormat, ap); + char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); + sqlite3_free(z1); + fputs(z2, out); + sqlite3_free(z2); + }else{ + vfprintf(out, zFormat, ap); + } + va_end(ap); +} +#elif !defined(utf8_printf) +# define utf8_printf fprintf +#endif + +/* +** Render output like fprintf(). This should not be used on anything that +** includes string formatting (e.g. "%s"). +*/ +#if !defined(raw_printf) +# define raw_printf fprintf +#endif + /* ** Write I/O traces to the following stream. */ @@ -382,7 +422,7 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - fprintf(iotrace, "%s", z); + utf8_printf(iotrace, "%s", z); sqlite3_free(z); } #endif @@ -416,8 +456,8 @@ static int isNumber(const char *z, int *realnum){ } /* -** A global char* and an SQL function to access its current value -** from within an SQL statement. This program used to use the +** A global char* and an SQL function to access its current value +** from within an SQL statement. This program used to use the ** sqlite_exec_printf() API to substitue a string into an SQL statement. ** The correct way to do this with sqlite3 is to use the bind API, but ** since the shell is built around the callback paradigm it would be a lot @@ -483,11 +523,10 @@ static char *local_getline(char *zLine, FILE *in){ } } #if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, translate the + /* For interactive input on Windows systems, translate the ** multi-byte characterset characters into UTF-8. */ - if( stdin_is_interactive ){ - extern char *sqlite3_win32_mbcs_to_utf8(const char*); - char *zTrans = sqlite3_win32_mbcs_to_utf8(zLine); + if( stdin_is_interactive && in==stdin ){ + char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ int nTrans = strlen30(zTrans)+1; if( nTrans>nLine ){ @@ -539,41 +578,21 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ return zResult; } +#if defined(SQLITE_ENABLE_SESSION) /* -** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, translate the -** output from UTF-8 into MBCS. +** State information for a single open session */ -#if defined(_WIN32) || defined(WIN32) -void utf8_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) ){ - extern char *sqlite3_win32_utf8_to_mbcs(const char*); - char *z1 = sqlite3_vmprintf(zFormat, ap); - char *z2 = sqlite3_win32_utf8_to_mbcs(z1); - sqlite3_free(z1); - fputs(z2, out); - sqlite3_free(z2); - }else{ - vfprintf(out, zFormat, ap); - } - va_end(ap); -} -#elif !defined(utf8_printf) -# define utf8_printf fprintf +typedef struct OpenSession OpenSession; +struct OpenSession { + char *zName; /* Symbolic name for this session */ + int nFilter; /* Number of xFilter rejection GLOB patterns */ + char **azFilter; /* Array of xFilter rejection GLOB patterns */ + sqlite3_session *p; /* The open session */ +}; #endif /* -** Render output like fprintf(). This should not be used on anything that -** includes string formatting (e.g. "%s"). -*/ -#if !defined(raw_printf) -# define raw_printf fprintf -#endif - -/* -** Shell output mode information from before ".explain on", +** Shell output mode information from before ".explain on", ** saved so that it can be restored by ".explain off" */ typedef struct SavedModeInfo SavedModeInfo; @@ -608,8 +627,10 @@ struct ShellState { int normalMode; /* Output mode before ".explain on" */ int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ + int nCheck; /* Number of ".check" commands run */ unsigned shellFlgs; /* Various flags */ char *zDestTable; /* Name of destination table when MODE_Insert */ + char zTestcase[30]; /* Name of current test case */ char colSeparator[20]; /* Column separator character for several modes */ char rowSeparator[20]; /* Row separator character for MODE_Ascii */ int colWidth[100]; /* Requested width of each column when in column mode*/ @@ -625,6 +646,10 @@ struct ShellState { int *aiIndent; /* Array of indents used in MODE_Explain */ int nIndent; /* Size of array aiIndent[] */ int iIndent; /* Index of current op in aiIndent[] */ +#if defined(SQLITE_ENABLE_SESSION) + int nSession; /* Number of active sessions */ + OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +#endif }; /* @@ -647,6 +672,7 @@ struct ShellState { #define MODE_Csv 7 /* Quote strings, numbers are plain */ #define MODE_Explain 8 /* Like MODE_Column, but do not truncate data */ #define MODE_Ascii 9 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Pretty 10 /* Pretty-print schemas */ static const char *modeDescr[] = { "line", @@ -659,6 +685,7 @@ static const char *modeDescr[] = { "csv", "explain", "ascii", + "prettyprint", }; /* @@ -706,7 +733,7 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ static void output_quoted_string(FILE *out, const char *z){ int i; int nSingle = 0; - setBinaryMode(out); + setBinaryMode(out, 1); for(i=0; z[i]; i++){ if( z[i]=='\'' ) nSingle++; } @@ -729,7 +756,7 @@ static void output_quoted_string(FILE *out, const char *z){ } raw_printf(out,"'"); } - setTextMode(out); + setTextMode(out, 1); } /* @@ -771,11 +798,11 @@ static void output_html_string(FILE *out, const char *z){ int i; if( z==0 ) z = ""; while( *z ){ - for(i=0; z[i] - && z[i]!='<' - && z[i]!='&' - && z[i]!='>' - && z[i]!='\"' + for(i=0; z[i] + && z[i]!='<' + && z[i]!='&' + && z[i]!='>' + && z[i]!='\"' && z[i]!='\''; i++){} if( i>0 ){ @@ -803,22 +830,22 @@ static void output_html_string(FILE *out, const char *z){ ** array, then the string must be quoted for CSV. */ static const char needCsvQuote[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; /* @@ -835,8 +862,8 @@ static void output_csv(ShellState *p, const char *z, int bSep){ int i; int nSep = strlen30(p->colSeparator); for(i=0; z[i]; i++){ - if( needCsvQuote[((unsigned char*)z)[i]] - || (z[i]==p->colSeparator[0] && + if( needCsvQuote[((unsigned char*)z)[i]] + || (z[i]==p->colSeparator[0] && (nSep==1 || memcmp(z, p->colSeparator, nSep)==0)) ){ i = 0; break; @@ -870,6 +897,54 @@ static void interrupt_handler(int NotUsed){ } #endif +#ifndef SQLITE_OMIT_AUTHORIZATION +/* +** When the ".auth ON" is set, the following authorizer callback is +** invoked. It always returns SQLITE_OK. +*/ +static int shellAuth( + void *pClientData, + int op, + const char *zA1, + const char *zA2, + const char *zA3, + const char *zA4 +){ + ShellState *p = (ShellState*)pClientData; + static const char *azAction[] = { 0, + "CREATE_INDEX", "CREATE_TABLE", "CREATE_TEMP_INDEX", + "CREATE_TEMP_TABLE", "CREATE_TEMP_TRIGGER", "CREATE_TEMP_VIEW", + "CREATE_TRIGGER", "CREATE_VIEW", "DELETE", + "DROP_INDEX", "DROP_TABLE", "DROP_TEMP_INDEX", + "DROP_TEMP_TABLE", "DROP_TEMP_TRIGGER", "DROP_TEMP_VIEW", + "DROP_TRIGGER", "DROP_VIEW", "INSERT", + "PRAGMA", "READ", "SELECT", + "TRANSACTION", "UPDATE", "ATTACH", + "DETACH", "ALTER_TABLE", "REINDEX", + "ANALYZE", "CREATE_VTABLE", "DROP_VTABLE", + "FUNCTION", "SAVEPOINT", "RECURSIVE" + }; + int i; + const char *az[4]; + az[0] = zA1; + az[1] = zA2; + az[2] = zA3; + az[3] = zA4; + utf8_printf(p->out, "authorizer: %s", azAction[op]); + for(i=0; i<4; i++){ + raw_printf(p->out, " "); + if( az[i] ){ + output_c_string(p->out, az[i]); + }else{ + raw_printf(p->out, "NULL"); + } + } + raw_printf(p->out, "\n"); + return SQLITE_OK; +} +#endif + + /* ** This is the callback routine that the shell ** invokes for each row of a query result. @@ -986,7 +1061,70 @@ static int shell_callback( } break; } - case MODE_Semi: + case MODE_Semi: { /* .schema and .fullschema output */ + utf8_printf(p->out, "%s;\n", azArg[0]); + break; + } + case MODE_Pretty: { /* .schema and .fullschema with --indent */ + char *z; + int j; + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + assert( nArg==1 ); + if( azArg[0]==0 ) break; + if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 + ){ + utf8_printf(p->out, "%s;\n", azArg[0]); + break; + } + z = sqlite3_mprintf("%s", azArg[0]); + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ + if( c==cEnd ){ + cEnd = 0; + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 ){ + utf8_printf(p->out, "%.*s\n", j, z); + j = 0; + } + } + z[j++] = c; + if( nParen==1 && (c=='(' || c==',' || c=='\n') ){ + if( c=='\n' ) j--; + utf8_printf(p->out, "%.*s\n ", j, z); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } + } + } + z[j] = 0; + } + utf8_printf(p->out, "%s;\n", z); + sqlite3_free(z); + break; + } case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout, "%s", z); if( iout, "%s", p->colSeparator); - }else if( p->cMode==MODE_Semi ){ - utf8_printf(p->out, ";%s", p->rowSeparator); }else{ utf8_printf(p->out, "%s", p->rowSeparator); } @@ -1046,7 +1182,7 @@ static int shell_callback( break; } case MODE_Csv: { - setBinaryMode(p->out); + setBinaryMode(p->out, 1); if( p->cnt++==0 && p->showHeader ){ for(i=0; iout, "%s", p->rowSeparator); } - setTextMode(p->out); + setTextMode(p->out, 1); break; } case MODE_Insert: { @@ -1172,7 +1308,7 @@ static void set_table_name(ShellState *p, const char *zName){ ** added to zIn, and the result returned in memory obtained from malloc(). ** zIn, if it was not NULL, is freed. ** -** If the third argument, quote, is not '\0', then it is used as a +** If the third argument, quote, is not '\0', then it is used as a ** quote character for zAppend. */ static char *appendText(char *zIn, char const *zAppend, char quote){ @@ -1219,7 +1355,7 @@ static char *appendText(char *zIn, char const *zAppend, char quote){ ** semicolon terminator to the end of that line. ** ** If the number of columns is 1 and that column contains text "--" -** then write the semicolon on a separate line. That way, if a +** then write the semicolon on a separate line. That way, if a ** "--" comment occurs at the end of the statement, the comment ** won't consume the semicolon terminator. */ @@ -1249,7 +1385,7 @@ static int run_table_dump_query( } z = (const char*)sqlite3_column_text(pSelect, 0); utf8_printf(p->out, "%s", z); - for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; @@ -1258,7 +1394,7 @@ static int run_table_dump_query( raw_printf(p->out, "\n;\n"); }else{ raw_printf(p->out, ";\n"); - } + } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); @@ -1284,6 +1420,43 @@ static char *save_err_msg( return zErrMsg; } +#ifdef __linux__ +/* +** Attempt to display I/O stats on Linux using /proc/PID/io +*/ +static void displayLinuxIoStats(FILE *out){ + FILE *in; + char z[200]; + sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid()); + in = fopen(z, "rb"); + if( in==0 ) return; + while( fgets(z, sizeof(z), in)!=0 ){ + static const struct { + const char *zPattern; + const char *zDesc; + } aTrans[] = { + { "rchar: ", "Bytes received by read():" }, + { "wchar: ", "Bytes sent to write():" }, + { "syscr: ", "Read() system calls:" }, + { "syscw: ", "Write() system calls:" }, + { "read_bytes: ", "Bytes read from storage:" }, + { "write_bytes: ", "Bytes written to storage:" }, + { "cancelled_write_bytes: ", "Cancelled write bytes:" }, + }; + int i; + for(i=0; iout ){ - + iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); raw_printf(pArg->out, @@ -1380,18 +1553,18 @@ static int display_stats( raw_printf(pArg->out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + raw_printf(pArg->out, "Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + raw_printf(pArg->out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", - iCur); + iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", - iCur); + iCur); } if( pArg && pArg->out && db && pArg->pStmt ){ @@ -1406,6 +1579,10 @@ static int display_stats( raw_printf(pArg->out, "Virtual Machine Steps: %d\n", iCur); } +#ifdef __linux__ + displayLinuxIoStats(pArg->out); +#endif + /* Do not remove this machine readable comment: extra-stats-output-here */ return 0; @@ -1449,7 +1626,7 @@ static void display_scanstats( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); rEstLoop *= rEst; - raw_printf(pArg->out, + raw_printf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst ); @@ -1476,7 +1653,7 @@ static int str_in_array(const char *zStr, const char **azArray){ /* ** If compiled statement pSql appears to be an EXPLAIN statement, allocate ** and populate the ShellState.aiIndent[] array with the number of -** spaces each opcode should be indented before it is output. +** spaces each opcode should be indented before it is output. ** ** The indenting rules are: ** @@ -1561,7 +1738,7 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ if( str_in_array(zOp, azGoto) && p2opnIndent && (abYield[p2op] || sqlite3_column_int(pSql, 2)) ){ - for(i=p2op+1; iaiIndent[i] += 2; + for(i=p2op; iaiIndent[i] += 2; } } @@ -1581,12 +1758,110 @@ static void explain_data_delete(ShellState *p){ } /* -** Execute a statement or set of statements. Print -** any result rows/columns depending on the current mode +** Disable and restore .wheretrace and .selecttrace settings. +*/ +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) +extern int sqlite3SelectTrace; +static int savedSelectTrace; +#endif +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) +extern int sqlite3WhereTrace; +static int savedWhereTrace; +#endif +static void disable_debug_trace_modes(void){ +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) + savedSelectTrace = sqlite3SelectTrace; + sqlite3SelectTrace = 0; +#endif +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) + savedWhereTrace = sqlite3WhereTrace; + sqlite3WhereTrace = 0; +#endif +} +static void restore_debug_trace_modes(void){ +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) + sqlite3SelectTrace = savedSelectTrace; +#endif +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) + sqlite3WhereTrace = savedWhereTrace; +#endif +} + +/* +** Run a prepared statement +*/ +static void exec_prepared_stmt( + ShellState *pArg, /* Pointer to ShellState */ + sqlite3_stmt *pStmt, /* Statment to run */ + int (*xCallback)(void*,int,char**,char**,int*) /* Callback function */ +){ + int rc; + + /* perform the first step. this will tell us if we + ** have a result set or not and how wide it is. + */ + rc = sqlite3_step(pStmt); + /* if we have a result set... */ + if( SQLITE_ROW == rc ){ + /* if we have a callback... */ + if( xCallback ){ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + rc = SQLITE_NOMEM; + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; icMode==MODE_Insert ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ + + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( xCallback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); + } + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); + } + }else{ + do{ + rc = sqlite3_step(pStmt); + } while( rc == SQLITE_ROW ); + } + } +} + +/* +** Execute a statement or set of statements. Print +** any result rows/columns depending on the current mode ** set via the supplied callback. ** -** This is very similar to SQLite's built-in sqlite3_exec() -** function except it takes a slightly different callback +** This is very similar to SQLite's built-in sqlite3_exec() +** function except it takes a slightly different callback ** and callback data argument. */ static int shell_exec( @@ -1607,6 +1882,7 @@ static int shell_exec( } while( zSql[0] && (SQLITE_OK == rc) ){ + static const char *zStmtSql; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); if( SQLITE_OK != rc ){ if( pzErrMsg ){ @@ -1619,6 +1895,8 @@ static int shell_exec( while( IsSpace(zSql[0]) ) zSql++; continue; } + zStmtSql = sqlite3_sql(pStmt); + while( IsSpace(zStmtSql[0]) ) zStmtSql++; /* save off the prepared statment handle and reset row count */ if( pArg ){ @@ -1628,15 +1906,15 @@ static int shell_exec( /* echo the sql statement if echo on */ if( pArg && pArg->echoOn ){ - const char *zStmtSql = sqlite3_sql(pStmt); utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } /* Show the EXPLAIN QUERY PLAN if .eqp is on */ - if( pArg && pArg->autoEQP ){ + if( pArg && pArg->autoEQP && sqlite3_strlike("EXPLAIN%",zStmtSql,0)!=0 ){ sqlite3_stmt *pExplain; - char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", - sqlite3_sql(pStmt)); + char *zEQP; + disable_debug_trace_modes(); + zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ @@ -1648,17 +1926,31 @@ static int shell_exec( } sqlite3_finalize(pExplain); sqlite3_free(zEQP); + if( pArg->autoEQP>=2 ){ + /* Also do an EXPLAIN for ".eqp full" mode */ + zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + pArg->cMode = MODE_Explain; + explain_data_prepare(pArg, pExplain); + exec_prepared_stmt(pArg, pExplain, xCallback); + explain_data_delete(pArg); + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + } + restore_debug_trace_modes(); } if( pArg ){ pArg->cMode = pArg->mode; if( pArg->autoExplain && sqlite3_column_count(pStmt)==8 - && sqlite3_strlike("%EXPLAIN%", sqlite3_sql(pStmt),0)==0 + && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0 ){ pArg->cMode = MODE_Explain; } - + /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ if( pArg->cMode==MODE_Explain ){ @@ -1666,63 +1958,7 @@ static int shell_exec( } } - /* perform the first step. this will tell us if we - ** have a result set or not and how wide it is. - */ - rc = sqlite3_step(pStmt); - /* if we have a result set... */ - if( SQLITE_ROW == rc ){ - /* if we have a callback... */ - if( xCallback ){ - /* allocate space for col name ptr, value ptr, and type */ - int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); - if( !pData ){ - rc = SQLITE_NOMEM; - }else{ - char **azCols = (char **)pData; /* Names of result columns */ - char **azVals = &azCols[nCol]; /* Results */ - int *aiTypes = (int *)&azVals[nCol]; /* Result types */ - int i, x; - assert(sizeof(int) <= sizeof(char *)); - /* save off ptrs to column names */ - for(i=0; icMode==MODE_Insert ){ - azVals[i] = ""; - }else{ - azVals[i] = (char*)sqlite3_column_text(pStmt, i); - } - if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ - rc = SQLITE_NOMEM; - break; /* from for */ - } - } /* end for */ - - /* if data and types extracted successfully... */ - if( SQLITE_ROW == rc ){ - /* call the supplied callback with the result row data */ - if( xCallback(pArg, nCol, azVals, azCols, aiTypes) ){ - rc = SQLITE_ABORT; - }else{ - rc = sqlite3_step(pStmt); - } - } - } while( SQLITE_ROW == rc ); - sqlite3_free(pData); - } - }else{ - do{ - rc = sqlite3_step(pStmt); - } while( rc == SQLITE_ROW ); - } - } - + exec_prepared_stmt(pArg, pStmt, xCallback); explain_data_delete(pArg); /* print usage stats if stats on */ @@ -1735,7 +1971,7 @@ static int shell_exec( display_scanstats(db, pArg); } - /* Finalize the statement just executed. If this fails, save a + /* Finalize the statement just executed. If this fails, save a ** copy of the error message. Otherwise, set zSql to point to the ** next statement to execute. */ rc2 = sqlite3_finalize(pStmt); @@ -1777,7 +2013,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; - + if( strcmp(zTable, "sqlite_sequence")==0 ){ zPrepStmt = "DELETE FROM sqlite_sequence;\n"; }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ @@ -1807,7 +2043,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ char *zTableInfo = 0; char *zTmp = 0; int nRow = 0; - + zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); zTableInfo = appendText(zTableInfo, zTable, '"'); zTableInfo = appendText(zTableInfo, ");", 0); @@ -1866,7 +2102,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ ** "ORDER BY rowid DESC" to the end. */ static int run_schema_dump_query( - ShellState *p, + ShellState *p, const char *zQuery ){ int rc; @@ -1900,10 +2136,14 @@ static int run_schema_dump_query( ** Text of a help message */ static char zHelp[] = +#ifndef SQLITE_OMIT_AUTHORIZATION + ".auth ON|OFF Show authorizer callbacks\n" +#endif ".backup ?DB? FILE Backup DB (default \"main\") to FILE\n" ".bail on|off Stop after hitting an error. Default OFF\n" ".binary on|off Turn binary output on or off. Default OFF\n" ".changes on|off Show number of rows changed by SQL\n" + ".check GLOB Fail if output since .testcase does not match\n" ".clone NEWDB Clone data into NEWDB from the existing database\n" ".databases List names and files of attached databases\n" ".dbinfo ?DB? Show status information about the database\n" @@ -1911,10 +2151,10 @@ static char zHelp[] = " If TABLE specified, only dump tables matching\n" " LIKE pattern TABLE.\n" ".echo on|off Turn command echo on or off\n" - ".eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN\n" + ".eqp on|off|full Enable or disable automatic EXPLAIN QUERY PLAN\n" ".exit Exit this program\n" ".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic\n" - ".fullschema Show schema and the content of sqlite_stat tables\n" + ".fullschema ?--indent? Show schema and the content of sqlite_stat tables\n" ".headers on|off Turn display of headers on or off\n" ".help Show this message\n" ".import FILE TABLE Import data from FILE into TABLE\n" @@ -1941,7 +2181,8 @@ static char zHelp[] = " tcl TCL list elements\n" ".nullvalue STRING Use STRING in place of NULL values\n" ".once FILENAME Output for the next SQL command only to FILENAME\n" - ".open ?FILENAME? Close existing database and reopen FILENAME\n" + ".open ?--new? ?FILE? Close existing database and reopen FILE\n" + " The --new starts with an empty file\n" ".output ?FILENAME? Send output to FILENAME or stdout\n" ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" @@ -1950,18 +2191,21 @@ static char zHelp[] = ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE\n" ".save FILE Write in-memory database into FILE\n" ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" - ".schema ?TABLE? Show the CREATE statements\n" - " If TABLE specified, only show tables matching\n" - " LIKE pattern TABLE.\n" + ".schema ?PATTERN? Show the CREATE statements matching PATTERN\n" + " Add --indent for pretty-printing\n" ".separator COL ?ROW? Change the column separator and optionally the row\n" " separator for both the output mode and .import\n" +#if defined(SQLITE_ENABLE_SESSION) + ".session CMD ... Create or control sessions\n" +#endif ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" ".show Show the current values for various settings\n" - ".stats on|off Turn stats on or off\n" + ".stats ?on|off? Show stats or turn stats on or off\n" ".system CMD ARGS... Run CMD ARGS... in a system shell\n" ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" + ".testcase NAME Begin redirecting output to 'testcase-out.txt'\n" ".timeout MS Try opening locked tables for MS milliseconds\n" ".timer on|off Turn SQL timer on or off\n" ".trace FILE|off Output each SQL statement as it is run\n" @@ -1972,8 +2216,61 @@ static char zHelp[] = " Negative values right-justify\n" ; +#if defined(SQLITE_ENABLE_SESSION) +/* +** Print help information for the ".sessions" command +*/ +void session_help(ShellState *p){ + raw_printf(p->out, + ".session ?NAME? SUBCOMMAND ?ARGS...?\n" + "If ?NAME? is omitted, the first defined session is used.\n" + "Subcommands:\n" + " attach TABLE Attach TABLE\n" + " changeset FILE Write a changeset into FILE\n" + " close Close one session\n" + " enable ?BOOLEAN? Set or query the enable bit\n" + " filter GLOB... Reject tables matching GLOBs\n" + " indirect ?BOOLEAN? Mark or query the indirect status\n" + " isempty Query whether the session is empty\n" + " list List currently open session names\n" + " open DB NAME Open a new session on DB\n" + " patchset FILE Write a patchset into FILE\n" + ); +} +#endif + + /* Forward reference */ static int process_input(ShellState *p, FILE *in); + + +/* +** Read the content of a file into memory obtained from sqlite3_malloc64(). +** The caller is responsible for freeing the memory. +** +** NULL is returned if any error is encountered. +*/ +static char *readFile(const char *zName){ + FILE *in = fopen(zName, "rb"); + long nIn; + size_t nRead; + char *pBuf; + if( in==0 ) return 0; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc64( nIn+1 ); + if( pBuf==0 ) return 0; + nRead = fread(pBuf, nIn, 1, in); + fclose(in); + if( nRead!=1 ){ + sqlite3_free(pBuf); + return 0; + } + pBuf[nIn] = 0; + return pBuf; +} + /* ** Implementation of the "readfile(X)" SQL function. The entire content ** of the file named X is read and returned as a BLOB. NULL is returned @@ -1985,25 +2282,13 @@ static void readfileFunc( sqlite3_value **argv ){ const char *zName; - FILE *in; - long nIn; void *pBuf; UNUSED_PARAMETER(argc); zName = (const char*)sqlite3_value_text(argv[0]); if( zName==0 ) return; - in = fopen(zName, "rb"); - if( in==0 ) return; - fseek(in, 0, SEEK_END); - nIn = ftell(in); - rewind(in); - pBuf = sqlite3_malloc64( nIn ); - if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ - sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); - }else{ - sqlite3_free(pBuf); - } - fclose(in); + pBuf = readFile(zName); + if( pBuf ) sqlite3_result_blob(context, pBuf, -1, sqlite3_free); } /* @@ -2037,6 +2322,53 @@ static void writefileFunc( sqlite3_result_int64(context, rc); } +#if defined(SQLITE_ENABLE_SESSION) +/* +** Close a single OpenSession object and release all of its associated +** resources. +*/ +static void session_close(OpenSession *pSession){ + int i; + sqlite3session_delete(pSession->p); + sqlite3_free(pSession->zName); + for(i=0; inFilter; i++){ + sqlite3_free(pSession->azFilter[i]); + } + sqlite3_free(pSession->azFilter); + memset(pSession, 0, sizeof(OpenSession)); +} +#endif + +/* +** Close all OpenSession objects and release all associated resources. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static void session_close_all(ShellState *p){ + int i; + for(i=0; inSession; i++){ + session_close(&p->aSession[i]); + } + p->nSession = 0; +} +#else +# define session_close_all(X) +#endif + +/* +** Implementation of the xFilter function for an open session. Omit +** any tables named by ".session filter" but let all other table through. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static int session_filter(void *pCtx, const char *zTab){ + OpenSession *pSession = (OpenSession*)pCtx; + int i; + for(i=0; inFilter; i++){ + if( sqlite3_strglob(pSession->azFilter[i], zTab)==0 ) return 0; + } + return 1; +} +#endif + /* ** Make sure the database is open. If it is not, then open it. If ** the database fails to open, print an error message and exit. @@ -2051,7 +2383,7 @@ static void open_db(ShellState *p, int keepAlive){ shellstaticFunc, 0, 0); } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", + utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", p->zDbFilename, sqlite3_errmsg(p->db)); if( keepAlive ) return; exit(1); @@ -2215,7 +2547,7 @@ static void output_file_close(FILE *f){ /* ** Try to open an output file. The names "stdout" and "stderr" are -** recognized and do the right thing. NULL is returned if the output +** recognized and do the right thing. NULL is returned if the output ** filename is "off". */ static FILE *output_file_open(const char *zFile){ @@ -2238,13 +2570,22 @@ static FILE *output_file_open(const char *zFile){ /* ** A routine for handling output from sqlite3_trace(). */ -static void sql_trace_callback(void *pArg, const char *z){ +static int sql_trace_callback( + unsigned mType, + void *pArg, + void *pP, + void *pX +){ FILE *f = (FILE*)pArg; + UNUSED_PARAMETER(mType); + UNUSED_PARAMETER(pP); if( f ){ + const char *z = (const char*)pX; int i = (int)strlen(z); while( i>0 && z[i-1]==';' ){ i--; } utf8_printf(f, "%.*s;\n", i, z); } + return 0; } /* @@ -2404,7 +2745,7 @@ static void tryToCloneData( sqlite3 *newDb, const char *zTable ){ - sqlite3_stmt *pQuery = 0; + sqlite3_stmt *pQuery = 0; sqlite3_stmt *pInsert = 0; char *zQuery = 0; char *zInsert = 0; @@ -2641,10 +2982,10 @@ static int db_int(ShellState *p, const char *zSql){ /* ** Convert a 2-byte or 4-byte big-endian integer into a native integer */ -unsigned int get2byteInt(unsigned char *a){ +static unsigned int get2byteInt(unsigned char *a){ return (a[0]<<8) + a[1]; } -unsigned int get4byteInt(unsigned char *a){ +static unsigned int get4byteInt(unsigned char *a){ return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; } @@ -2708,9 +3049,9 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ utf8_printf(p->out, "%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) raw_printf(p->out, " (utf8)"); - if( val==2 ) raw_printf(p->out, " (utf16le)"); - if( val==3 ) raw_printf(p->out, " (utf16be)"); + if( val==1 ) raw_printf(p->out, " (utf8)"); + if( val==2 ) raw_printf(p->out, " (utf16le)"); + if( val==3 ) raw_printf(p->out, " (utf16be)"); } } raw_printf(p->out, "\n"); @@ -2749,6 +3090,130 @@ static int shellNomemError(void){ return 1; } +/* +** Compare the pattern in zGlob[] against the text in z[]. Return TRUE +** if they match and FALSE (0) if they do not match. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front +** +** ' ' Any span of whitespace matches any other span of +** whitespace. +** +** Extra whitespace at the end of z[] is ignored. +*/ +static int testcase_glob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( IsSpace(c) ){ + if( !IsSpace(*z) ) return 0; + while( IsSpace(*zGlob) ) zGlob++; + while( IsSpace(*z) ) z++; + }else if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && testcase_glob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( testcase_glob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( (z[0]=='-' || z[0]=='+') && IsDigit(z[1]) ) z++; + if( !IsDigit(z[0]) ) return 0; + z++; + while( IsDigit(z[0]) ){ z++; } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + while( IsSpace(*z) ){ z++; } + return *z==0; +} + + +/* +** Compare the string as a command-line option with either one or two +** initial "-" characters. +*/ +static int optionMatch(const char *zStr, const char *zOpt){ + if( zStr[0]!='-' ) return 0; + zStr++; + if( zStr[0]=='-' ) zStr++; + return strcmp(zStr, zOpt)==0; +} + +/* +** Delete a file. +*/ +int shellDeleteFile(const char *zFilename){ + int rc; +#ifdef _WIN32 + wchar_t *z = sqlite3_win32_utf8_to_unicode(zFilename); + rc = _wunlink(z); + sqlite3_free(z); +#else + rc = unlink(zFilename); +#endif + return rc; +} + /* ** If an input line begins with "." then invoke this routine to ** process that line. @@ -2770,9 +3235,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zLine[h]=='\'' || zLine[h]=='"' ){ int delim = zLine[h++]; azArg[nArg++] = &zLine[h]; - while( zLine[h] && zLine[h]!=delim ){ + while( zLine[h] && zLine[h]!=delim ){ if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++; - h++; + h++; } if( zLine[h]==delim ){ zLine[h++] = 0; @@ -2791,6 +3256,23 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==0 ) return 0; /* no tokens, no error */ n = strlen30(azArg[0]); c = azArg[0][0]; + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){ + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .auth ON|OFF\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( booleanValue(azArg[1]) ){ + sqlite3_set_authorizer(p->db, shellAuth, p); + }else{ + sqlite3_set_authorizer(p->db, 0, 0); + } + }else +#endif + if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) ){ @@ -2859,9 +3341,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ if( nArg==2 ){ if( booleanValue(azArg[1]) ){ - setBinaryMode(p->out); + setBinaryMode(p->out, 1); }else{ - setTextMode(p->out); + setTextMode(p->out, 1); } }else{ raw_printf(stderr, "Usage: .binary on|off\n"); @@ -2885,6 +3367,31 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + /* Cancel output redirection, if it is currently set (by .testcase) + ** Then read the content of the testcase-out.txt file and compare against + ** azArg[1]. If there are differences, report an error and exit. + */ + if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + char *zRes = 0; + output_reset(p); + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); + rc = 2; + }else if( (zRes = readFile("testcase-out.txt"))==0 ){ + raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); + rc = 2; + }else if( testcase_glob(azArg[1],zRes)==0 ){ + utf8_printf(stderr, + "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); + rc = 2; + }else{ + utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); + p->nCheck++; + } + sqlite3_free(zRes); + }else + if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ if( nArg==2 ){ tryToClone(p, azArg[1]); @@ -2933,11 +3440,11 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); p->nErr = 0; if( nArg==1 ){ - run_schema_dump_query(p, + run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" ); - run_schema_dump_query(p, + run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE name=='sqlite_sequence'" ); @@ -2982,11 +3489,15 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ - p->autoEQP = booleanValue(azArg[1]); + if( strcmp(azArg[1],"full")==0 ){ + p->autoEQP = 2; + }else{ + p->autoEQP = booleanValue(azArg[1]); + } }else{ - raw_printf(stderr, "Usage: .eqp on|off\n"); + raw_printf(stderr, "Usage: .eqp on|off|full\n"); rc = 1; - } + } }else if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ @@ -3020,15 +3531,19 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellState data; char *zErrMsg = 0; int doStats = 0; + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + if( nArg==2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; + nArg = 1; + } if( nArg!=1 ){ - raw_printf(stderr, "Usage: .fullschema\n"); + raw_printf(stderr, "Usage: .fullschema ?--indent?\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); - memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; rc = sqlite3_exec(p->db, "SELECT sql FROM" " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" @@ -3171,7 +3686,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable); char cSep = '('; while( xRead(&sCtx) ){ - zCreate = sqlite3_mprintf("%z%c\n \"%s\" TEXT", zCreate, cSep, sCtx.z); + zCreate = sqlite3_mprintf("%z%c\n \"%w\" TEXT", zCreate, cSep, sCtx.z); cSep = ','; if( sCtx.cTerm!=sCtx.cColSep ) break; } @@ -3368,7 +3883,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; idb, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ @@ -3485,21 +4000,43 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ - sqlite3 *savedDb = p->db; - const char *zSavedFilename = p->zDbFilename; - char *zNewFilename = 0; + char *zNewFilename; /* Name of the database file to open */ + int iName = 1; /* Index in azArg[] of the filename */ + int newFlag = 0; /* True to delete file before opening */ + /* Close the existing database */ + session_close_all(p); + sqlite3_close(p->db); p->db = 0; - if( nArg>=2 ) zNewFilename = sqlite3_mprintf("%s", azArg[1]); - p->zDbFilename = zNewFilename; - open_db(p, 1); - if( p->db!=0 ){ - sqlite3_close(savedDb); - sqlite3_free(p->zFreeOnClose); - p->zFreeOnClose = zNewFilename; - }else{ - sqlite3_free(zNewFilename); - p->db = savedDb; - p->zDbFilename = zSavedFilename; + sqlite3_free(p->zFreeOnClose); + p->zFreeOnClose = 0; + /* Check for command-line arguments */ + for(iName=1; iNameiName ? sqlite3_mprintf("%s", azArg[iName]) : 0; + if( zNewFilename ){ + if( newFlag ) shellDeleteFile(zNewFilename); + p->zDbFilename = zNewFilename; + open_db(p, 1); + if( p->db==0 ){ + utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); + sqlite3_free(zNewFilename); + }else{ + p->zFreeOnClose = zNewFilename; + } + } + if( p->db==0 ){ + /* As a fall-back open a TEMP database */ + p->zDbFilename = 0; + open_db(p, 0); } }else @@ -3662,7 +4199,12 @@ static int do_meta_command(char *zLine, ShellState *p){ memcpy(&data, p, sizeof(data)); data.showHeader = 0; data.cMode = data.mode = MODE_Semi; - if( nArg==2 ){ + if( nArg>=2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; + nArg--; + if( nArg==2 ) azArg[1] = azArg[2]; + } + if( nArg==2 && azArg[1][0]!='-' ){ int i; for(i=0; azArg[1][i]; i++) azArg[1][i] = ToLower(azArg[1][i]); if( strcmp(azArg[1],"sqlite_master")==0 ){ @@ -3717,7 +4259,7 @@ static int do_meta_command(char *zLine, ShellState *p){ callback, &data, &zErrMsg ); }else{ - raw_printf(stderr, "Usage: .schema ?LIKE-PATTERN?\n"); + raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; } @@ -3733,14 +4275,207 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ - extern int sqlite3SelectTrace; sqlite3SelectTrace = integerValue(azArg[1]); }else #endif +#if defined(SQLITE_ENABLE_SESSION) + if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + OpenSession *pSession = &p->aSession[0]; + char **azCmd = &azArg[1]; + int iSes = 0; + int nCmd = nArg - 1; + int i; + if( nArg<=1 ) goto session_syntax_error; + open_db(p, 0); + if( nArg>=3 ){ + for(iSes=0; iSesnSession; iSes++){ + if( strcmp(p->aSession[iSes].zName, azArg[1])==0 ) break; + } + if( iSesnSession ){ + pSession = &p->aSession[iSes]; + azCmd++; + nCmd--; + }else{ + pSession = &p->aSession[0]; + iSes = 0; + } + } + + /* .session attach TABLE + ** Invoke the sqlite3session_attach() interface to attach a particular + ** table so that it is never filtered. + */ + if( strcmp(azCmd[0],"attach")==0 ){ + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ){ + session_not_open: + raw_printf(stderr, "ERROR: No sessions are open\n"); + }else{ + rc = sqlite3session_attach(pSession->p, azCmd[1]); + if( rc ){ + raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc); + rc = 0; + } + } + }else + + /* .session changeset FILE + ** .session patchset FILE + ** Write a changeset or patchset into a file. The file is overwritten. + */ + if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + FILE *out = 0; + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ) goto session_not_open; + out = fopen(azCmd[1], "wb"); + if( out==0 ){ + utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", azCmd[1]); + }else{ + int szChng; + void *pChng; + if( azCmd[0][0]=='c' ){ + rc = sqlite3session_changeset(pSession->p, &szChng, &pChng); + }else{ + rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); + } + if( rc ){ + printf("Error: error code %d\n", rc); + rc = 0; + } + if( pChng + && fwrite(pChng, szChng, 1, out)!=1 ){ + raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", + szChng); + } + sqlite3_free(pChng); + fclose(out); + } + }else + + /* .session close + ** Close the identified session + */ + if( strcmp(azCmd[0], "close")==0 ){ + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + session_close(pSession); + p->aSession[iSes] = p->aSession[--p->nSession]; + } + }else + + /* .session enable ?BOOLEAN? + ** Query or set the enable flag + */ + if( strcmp(azCmd[0], "enable")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_enable(pSession->p, ii); + utf8_printf(p->out, "session %s enable flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session filter GLOB .... + ** Set a list of GLOB patterns of table names to be excluded. + */ + if( strcmp(azCmd[0], "filter")==0 ){ + int ii, nByte; + if( nCmd<2 ) goto session_syntax_error; + if( p->nSession ){ + for(ii=0; iinFilter; ii++){ + sqlite3_free(pSession->azFilter[ii]); + } + sqlite3_free(pSession->azFilter); + nByte = sizeof(pSession->azFilter[0])*(nCmd-1); + pSession->azFilter = sqlite3_malloc( nByte ); + if( pSession->azFilter==0 ){ + raw_printf(stderr, "Error: out or memory\n"); + exit(1); + } + for(ii=1; iiazFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); + } + pSession->nFilter = ii-1; + } + }else + + /* .session indirect ?BOOLEAN? + ** Query or set the indirect flag + */ + if( strcmp(azCmd[0], "indirect")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_indirect(pSession->p, ii); + utf8_printf(p->out, "session %s indirect flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session isempty + ** Determine if the session is empty + */ + if( strcmp(azCmd[0], "isempty")==0 ){ + int ii; + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + ii = sqlite3session_isempty(pSession->p); + utf8_printf(p->out, "session %s isempty flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session list + ** List all currently open sessions + */ + if( strcmp(azCmd[0],"list")==0 ){ + for(i=0; inSession; i++){ + utf8_printf(p->out, "%d %s\n", i, p->aSession[i].zName); + } + }else + + /* .session open DB NAME + ** Open a new session called NAME on the attached database DB. + ** DB is normally "main". + */ + if( strcmp(azCmd[0],"open")==0 ){ + char *zName; + if( nCmd!=3 ) goto session_syntax_error; + zName = azCmd[2]; + if( zName[0]==0 ) goto session_syntax_error; + for(i=0; inSession; i++){ + if( strcmp(p->aSession[i].zName,zName)==0 ){ + utf8_printf(stderr, "Session \"%s\" already exists\n", zName); + goto meta_command_exit; + } + } + if( p->nSession>=ArraySize(p->aSession) ){ + raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession)); + goto meta_command_exit; + } + pSession = &p->aSession[p->nSession]; + rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); + if( rc ){ + raw_printf(stderr, "Cannot open session: error code=%d\n", rc); + rc = 0; + goto meta_command_exit; + } + pSession->nFilter = 0; + sqlite3session_table_filter(pSession->p, session_filter, pSession); + p->nSession++; + pSession->zName = sqlite3_mprintf("%s", zName); + }else + /* If no command name matches, show a syntax error */ + session_syntax_error: + session_help(p); + }else +#endif #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change @@ -3801,17 +4536,18 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ + static const char *azBool[] = { "off", "on", "full", "unk" }; int i; if( nArg!=1 ){ raw_printf(stderr, "Usage: .show\n"); rc = 1; goto meta_command_exit; } - utf8_printf(p->out, "%12.12s: %s\n","echo", p->echoOn ? "on" : "off"); - utf8_printf(p->out, "%12.12s: %s\n","eqp", p->autoEQP ? "on" : "off"); + utf8_printf(p->out, "%12.12s: %s\n","echo", azBool[p->echoOn!=0]); + utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); utf8_printf(p->out, "%12.12s: %s\n","explain", p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - utf8_printf(p->out,"%12.12s: %s\n","headers", p->showHeader ? "on" : "off"); + utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]); utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); utf8_printf(p->out, "%12.12s: ", "nullvalue"); output_c_string(p->out, p->nullValue); @@ -3824,19 +4560,23 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(p->out,"%12.12s: ", "rowseparator"); output_c_string(p->out, p->rowSeparator); raw_printf(p->out, "\n"); - utf8_printf(p->out, "%12.12s: %s\n","stats", p->statsOn ? "on" : "off"); + utf8_printf(p->out, "%12.12s: %s\n","stats", azBool[p->statsOn!=0]); utf8_printf(p->out, "%12.12s: ", "width"); for (i=0;i<(int)ArraySize(p->colWidth) && p->colWidth[i] != 0;i++) { raw_printf(p->out, "%d ", p->colWidth[i]); } raw_printf(p->out, "\n"); + utf8_printf(p->out, "%12.12s: %s\n", "filename", + p->zDbFilename ? p->zDbFilename : ""); }else if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ if( nArg==2 ){ p->statsOn = booleanValue(azArg[1]); + }else if( nArg==1 ){ + display_stats(p->db, p, 0); }else{ - raw_printf(stderr, "Usage: .stats on|off\n"); + raw_printf(stderr, "Usage: .stats ?on|off?\n"); rc = 1; } }else @@ -3945,6 +4685,20 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azResult); }else + /* Begin redirecting output to the file "testcase-out.txt" */ + if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ + output_reset(p); + p->out = output_file_open("testcase-out.txt"); + if( p->out==0 ){ + utf8_printf(stderr, "Error: cannot open 'testcase-out.txt'\n"); + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); + } + }else + if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ @@ -3994,9 +4748,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: - case SQLITE_TESTCTRL_RESERVE: + case SQLITE_TESTCTRL_RESERVE: if( nArg==3 ){ - int opt = (int)strtol(azArg[2], 0, 0); + int opt = (int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); } else { @@ -4020,7 +4774,7 @@ static int do_meta_command(char *zLine, ShellState *p){ break; /* sqlite3_test_control(int, uint) */ - case SQLITE_TESTCTRL_PENDING_BYTE: + case SQLITE_TESTCTRL_PENDING_BYTE: if( nArg==3 ){ unsigned int opt = (unsigned int)integerValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); @@ -4030,13 +4784,13 @@ static int do_meta_command(char *zLine, ShellState *p){ " int option\n", azArg[1]); } break; - + /* sqlite3_test_control(int, int) */ - case SQLITE_TESTCTRL_ASSERT: - case SQLITE_TESTCTRL_ALWAYS: - case SQLITE_TESTCTRL_NEVER_CORRUPT: + case SQLITE_TESTCTRL_ASSERT: + case SQLITE_TESTCTRL_ALWAYS: + case SQLITE_TESTCTRL_NEVER_CORRUPT: if( nArg==3 ){ - int opt = booleanValue(azArg[2]); + int opt = booleanValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); } else { @@ -4047,9 +4801,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, char *) */ #ifdef SQLITE_N_KEYWORD - case SQLITE_TESTCTRL_ISKEYWORD: + case SQLITE_TESTCTRL_ISKEYWORD: if( nArg==3 ){ - const char *opt = azArg[2]; + const char *opt = azArg[2]; rc2 = sqlite3_test_control(testctrl, opt); raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); } else { @@ -4062,7 +4816,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_IMPOSTER: if( nArg==5 ){ - rc2 = sqlite3_test_control(testctrl, p->db, + rc2 = sqlite3_test_control(testctrl, p->db, azArg[2], integerValue(azArg[3]), integerValue(azArg[4])); @@ -4072,10 +4826,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } break; - case SQLITE_TESTCTRL_BITVEC_TEST: - case SQLITE_TESTCTRL_FAULT_INSTALL: - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: - case SQLITE_TESTCTRL_SCRATCHMALLOC: + case SQLITE_TESTCTRL_BITVEC_TEST: + case SQLITE_TESTCTRL_FAULT_INSTALL: + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: + case SQLITE_TESTCTRL_SCRATCHMALLOC: default: utf8_printf(stderr, "Error: CLI support for testctrl %s not implemented\n", @@ -4089,7 +4843,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); }else - + if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ enableTimer = booleanValue(azArg[1]); @@ -4102,7 +4856,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else - + if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ open_db(p, 0); if( nArg!=2 ){ @@ -4114,9 +4868,9 @@ static int do_meta_command(char *zLine, ShellState *p){ p->traceOut = output_file_open(azArg[1]); #if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) if( p->traceOut==0 ){ - sqlite3_trace(p->db, 0, 0); + sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace(p->db, sql_trace_callback, p->traceOut); + sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); } #endif }else @@ -4182,7 +4936,7 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n"); rc = 1; goto meta_command_exit; - } + } }else #endif /* SQLITE_USER_AUTHENTICATION */ @@ -4237,7 +4991,6 @@ static int do_meta_command(char *zLine, ShellState *p){ #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ - extern int sqlite3WhereTrace; sqlite3WhereTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff; }else #endif @@ -4356,7 +5109,7 @@ static int process_input(ShellState *p, FILE *in){ zLine = one_input_line(in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( stdin_is_interactive ) printf("\n"); + if( in==0 && stdin_is_interactive ) printf("\n"); break; } if( seenInterrupt ){ @@ -4414,7 +5167,7 @@ static int process_input(ShellState *p, FILE *in){ if( rc || zErrMsg ){ char zPrefix[100]; if( in!=0 || !stdin_is_interactive ){ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error: near line %d:", startline); }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:"); @@ -4456,8 +5209,13 @@ static int process_input(ShellState *p, FILE *in){ ** Return a pathname which is the user's home directory. A ** 0 return indicates an error of some kind. */ -static char *find_home_dir(void){ +static char *find_home_dir(int clearFlag){ static char *home_dir = NULL; + if( clearFlag ){ + free(home_dir); + home_dir = 0; + return 0; + } if( home_dir ) return home_dir; #if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ @@ -4532,7 +5290,7 @@ static void process_sqliterc( FILE *in = NULL; if (sqliterc == NULL) { - home_dir = find_home_dir(); + home_dir = find_home_dir(0); if( home_dir==0 ){ raw_printf(stderr, "-- warning: cannot find home directory;" " cannot read ~/.sqliterc\n"); @@ -4556,7 +5314,7 @@ static void process_sqliterc( /* ** Show available command line options */ -static const char zOptions[] = +static const char zOptions[] = " -ascii set output mode to 'ascii'\n" " -bail stop after hitting an error\n" " -batch force batch I/O\n" @@ -4593,7 +5351,7 @@ static const char zOptions[] = ; static void usage(int showDetail){ utf8_printf(stderr, - "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "Usage: %s [OPTIONS] FILENAME [SQL]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist.\n", Argv0); if( showDetail ){ @@ -4655,7 +5413,20 @@ static char *cmdline_option_value(int argc, char **argv, int i){ return argv[i]; } +#ifndef SQLITE_SHELL_IS_UTF8 +# if (defined(_WIN32) || defined(WIN32)) && defined(_MSC_VER) +# define SQLITE_SHELL_IS_UTF8 (0) +# else +# define SQLITE_SHELL_IS_UTF8 (1) +# endif +#endif + +#if SQLITE_SHELL_IS_UTF8 int SQLITE_CDECL main(int argc, char **argv){ +#else +int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char **argv; +#endif char *zErrMsg = 0; ShellState data; const char *zInitFile = 0; @@ -4666,6 +5437,11 @@ int SQLITE_CDECL main(int argc, char **argv){ int nCmd = 0; char **azCmd = 0; + setBinaryMode(stdin, 0); + setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ + stdin_is_interactive = isatty(0); + stdout_is_console = isatty(1); + #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", @@ -4673,12 +5449,24 @@ int SQLITE_CDECL main(int argc, char **argv){ exit(1); } #endif - setBinaryMode(stdin); - setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ - Argv0 = argv[0]; main_init(&data); - stdin_is_interactive = isatty(0); - stdout_is_console = isatty(1); +#if !SQLITE_SHELL_IS_UTF8 + sqlite3_initialize(); + argv = sqlite3_malloc64(sizeof(argv[0])*argc); + if( argv==0 ){ + raw_printf(stderr, "out of memory\n"); + exit(1); + } + for(i=0; i=1 && argv && argv[0] ); + Argv0 = argv[0]; /* Make sure we have a valid signal handler early, before anything ** else is done. @@ -4734,7 +5522,7 @@ int SQLITE_CDECL main(int argc, char **argv){ zInitFile = cmdline_option_value(argc, argv, ++i); }else if( strcmp(z,"-batch")==0 ){ /* Need to check for batch mode here to so we can avoid printing - ** informational messages (like from process_sqliterc) before + ** informational messages (like from process_sqliterc) before ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; @@ -4747,6 +5535,8 @@ int SQLITE_CDECL main(int argc, char **argv){ szHeap = integerValue(zSize); if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); +#else + (void)cmdline_option_value(argc, argv, ++i); #endif }else if( strcmp(z,"-scratch")==0 ){ int n, sz; @@ -4875,6 +5665,8 @@ int SQLITE_CDECL main(int argc, char **argv){ data.echoOn = 1; }else if( strcmp(z,"-eqp")==0 ){ data.autoEQP = 1; + }else if( strcmp(z,"-eqpfull")==0 ){ + data.autoEQP = 2; }else if( strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( strcmp(z,"-scanstats")==0 ){ @@ -4986,7 +5778,7 @@ int SQLITE_CDECL main(int argc, char **argv){ printf(".\nUse \".open FILENAME\" to reopen on a " "persistent database.\n"); } - zHome = find_home_dir(); + zHome = find_home_dir(0); if( zHome ){ nHistory = strlen30(zHome) + 20; if( (zHistory = malloc(nHistory))!=0 ){ @@ -5006,8 +5798,14 @@ int SQLITE_CDECL main(int argc, char **argv){ } set_table_name(&data, 0); if( data.db ){ + session_close_all(&data); sqlite3_close(data.db); } - sqlite3_free(data.zFreeOnClose); + sqlite3_free(data.zFreeOnClose); + find_home_dir(1); +#if !SQLITE_SHELL_IS_UTF8 + for(i=0; i /* Needed for the definition of va_list */ /* @@ -54,8 +54,17 @@ extern "C" { #ifndef SQLITE_CDECL # define SQLITE_CDECL #endif +#ifndef SQLITE_APICALL +# define SQLITE_APICALL +#endif #ifndef SQLITE_STDCALL -# define SQLITE_STDCALL +# define SQLITE_STDCALL SQLITE_APICALL +#endif +#ifndef SQLITE_CALLBACK +# define SQLITE_CALLBACK +#endif +#ifndef SQLITE_SYSAPI +# define SQLITE_SYSAPI #endif /* @@ -99,7 +108,8 @@ extern "C" { ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** -** Since version 3.6.18, SQLite source code has been stored in the +** Since [version 3.6.18] ([dateof:3.6.18]), +** SQLite source code has been stored in the ** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite @@ -443,7 +453,8 @@ int sqlite3_exec( ** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to -** address this, newer versions of SQLite (version 3.3.8 and later) include +** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] +** and later) include ** support for additional result codes that provide more detailed information ** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the @@ -506,6 +517,7 @@ int sqlite3_exec( #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) +#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -966,6 +978,12 @@ struct sqlite3_io_methods { ** on whether or not the file has been renamed, moved, or deleted since it ** was first opened. ** +**
  • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the +** underlying native file handle associated with a file handle. This file +** control interprets its argument as a pointer to a native file handle and +** writes the resulting value there. +** **
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] ** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This ** opcode causes the xFileControl method to swap the file handle with the one @@ -1016,6 +1034,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 #define SQLITE_FCNTL_JOURNAL_POINTER 28 +#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1035,6 +1054,16 @@ struct sqlite3_io_methods { */ typedef struct sqlite3_mutex sqlite3_mutex; +/* +** CAPI3REF: Loadable Extension Thunk +** +** A pointer to the opaque sqlite3_api_routines structure is passed as +** the third parameter to entry points of [loadable extensions]. This +** structure must be typedefed in order to work around compiler warnings +** on some platforms. +*/ +typedef struct sqlite3_api_routines sqlite3_api_routines; + /* ** CAPI3REF: OS Interface Object ** @@ -1228,7 +1257,7 @@ struct sqlite3_vfs { const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); /* ** The methods above are in versions 1 through 3 of the sqlite_vfs object. - ** New fields may be appended in figure versions. The iVersion + ** New fields may be appended in future versions. The iVersion ** value will increment whenever this happens. */ }; @@ -1820,6 +1849,20 @@ struct sqlite3_mem_methods { ** is enabled (using the [PRAGMA threads] command) and the amount of content ** to be sorted exceeds the page size times the minimum of the ** [PRAGMA cache_size] setting and this value. +** +** [[SQLITE_CONFIG_STMTJRNL_SPILL]] +**
    SQLITE_CONFIG_STMTJRNL_SPILL +**
    ^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which +** becomes the [statement journal] spill-to-disk threshold. +** [Statement journals] are held in memory until their size (in bytes) +** exceeds this threshold, at which point they are written to disk. +** Or if the threshold is -1, statement journals are always held +** exclusively in memory. +** Since many statement journals never become large, setting the spill +** threshold to a value such as 64KiB can greatly reduce the amount of +** I/O required to support statement rollback. +** The default value for this setting is controlled by the +** [SQLITE_STMTJRNL_SPILL] compile-time option. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1847,6 +1890,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ +#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1904,11 +1948,53 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back.
    ** +**
    SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
    +**
    ^This option is used to enable or disable the two-argument +** version of the [fts3_tokenizer()] function which is part of the +** [FTS3] full-text search engine extension. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable fts3_tokenizer() or +** positive to enable fts3_tokenizer() or negative to leave the setting +** unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the new setting is not reported back.
    +** +**
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    +**
    ^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argument to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
    +** +**
    SQLITE_DBCONFIG_MAINDBNAME
    +**
    ^This option is used to change the name of the "main" database +** schema. ^The sole argument is a pointer to a constant UTF8 string +** which will become the new schema name in place of "main". ^SQLite +** does not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into this DBCONFIG option is unchanged +** until after the database connection closes. +**
    +** ** */ -#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ -#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ -#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ /* @@ -2185,7 +2271,7 @@ int sqlite3_complete16(const void *sql); ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ -int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); +int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); /* ** CAPI3REF: Set A Busy Timeout @@ -2707,6 +2793,9 @@ int sqlite3_set_authorizer( ** CAPI3REF: Tracing And Profiling Functions ** METHOD: sqlite3 ** +** These routines are deprecated. Use the [sqlite3_trace_v2()] interface +** instead of the routines described here. +** ** These routines register callback functions that can be used for ** tracing and profiling the execution of SQL statements. ** @@ -2732,10 +2821,104 @@ int sqlite3_set_authorizer( ** sqlite3_profile() function is considered experimental and is ** subject to change in future versions of SQLite. */ -void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); -SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, +SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, + void(*xTrace)(void*,const char*), void*); +SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); +/* +** CAPI3REF: SQL Trace Event Codes +** KEYWORDS: SQLITE_TRACE +** +** These constants identify classes of events that can be monitored +** using the [sqlite3_trace_v2()] tracing logic. The third argument +** to [sqlite3_trace_v2()] is an OR-ed combination of one or more of +** the following constants. ^The first argument to the trace callback +** is one of the following constants. +** +** New tracing constants may be added in future releases. +** +** ^A trace callback has four arguments: xCallback(T,C,P,X). +** ^The T argument is one of the integer type codes above. +** ^The C argument is a copy of the context pointer passed in as the +** fourth argument to [sqlite3_trace_v2()]. +** The P and X arguments are pointers whose meanings depend on T. +** +**
    +** [[SQLITE_TRACE_STMT]]
    SQLITE_TRACE_STMT
    +**
    ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement +** first begins running and possibly at other times during the +** execution of the prepared statement, such as at the start of each +** trigger subprogram. ^The P argument is a pointer to the +** [prepared statement]. ^The X argument is a pointer to a string which +** is the unexpanded SQL text of the prepared statement or an SQL comment +** that indicates the invocation of a trigger. ^The callback can compute +** the same text that would have been returned by the legacy [sqlite3_trace()] +** interface by using the X argument when X begins with "--" and invoking +** [sqlite3_expanded_sql(P)] otherwise. +** +** [[SQLITE_TRACE_PROFILE]]
    SQLITE_TRACE_PROFILE
    +**
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same +** information as is provided by the [sqlite3_profile()] callback. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument points to a 64-bit integer which is the estimated of +** the number of nanosecond that the prepared statement took to run. +** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. +** +** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    +**
    ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared +** statement generates a single row of result. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument is unused. +** +** [[SQLITE_TRACE_CLOSE]]
    SQLITE_TRACE_CLOSE
    +**
    ^An SQLITE_TRACE_CLOSE callback is invoked when a database +** connection closes. +** ^The P argument is a pointer to the [database connection] object +** and the X argument is unused. +**
    +*/ +#define SQLITE_TRACE_STMT 0x01 +#define SQLITE_TRACE_PROFILE 0x02 +#define SQLITE_TRACE_ROW 0x04 +#define SQLITE_TRACE_CLOSE 0x08 + +/* +** CAPI3REF: SQL Trace Hook +** METHOD: sqlite3 +** +** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback +** function X against [database connection] D, using property mask M +** and context pointer P. ^If the X callback is +** NULL or if the M mask is zero, then tracing is disabled. The +** M argument should be the bitwise OR-ed combination of +** zero or more [SQLITE_TRACE] constants. +** +** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides +** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** +** ^The X callback is invoked whenever any of the events identified by +** mask M occur. ^The integer return value from the callback is currently +** ignored, though this may change in future releases. Callback +** implementations should return zero to ensure future compatibility. +** +** ^A trace callback is invoked with four arguments: callback(T,C,P,X). +** ^The T argument is one of the [SQLITE_TRACE] +** constants to indicate why the callback was invoked. +** ^The C argument is a copy of the context pointer. +** The P and X arguments are pointers whose meanings depend on T. +** +** The sqlite3_trace_v2() interface is intended to replace the legacy +** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which +** are deprecated. +*/ +int sqlite3_trace_v2( + sqlite3*, + unsigned uMask, + int(*xCallback)(unsigned,void*,void*,void*), + void *pCtx +); + /* ** CAPI3REF: Query Progress Callbacks ** METHOD: sqlite3 @@ -3354,11 +3537,35 @@ int sqlite3_prepare16_v2( ** CAPI3REF: Retrieving Statement SQL ** METHOD: sqlite3_stmt ** -** ^This interface can be used to retrieve a saved copy of the original -** SQL text used to create a [prepared statement] if that statement was -** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. +** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8 +** SQL text used to create [prepared statement] P if P was +** created by either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. +** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 +** string containing the SQL text of prepared statement P with +** [bound parameters] expanded. +** +** ^(For example, if a prepared statement is created using the SQL +** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 +** and parameter :xyz is unbound, then sqlite3_sql() will return +** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql() +** will return "SELECT 2345,NULL".)^ +** +** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory +** is available to hold the result, or if the result would exceed the +** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** +** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of +** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time +** option causes sqlite3_expanded_sql() to always return NULL. +** +** ^The string returned by sqlite3_sql(P) is managed by SQLite and is +** automatically freed when the prepared statement is finalized. +** ^The string returned by sqlite3_expanded_sql(P), on the other hand, +** is obtained from [sqlite3_malloc()] and must be free by the application +** by passing it to [sqlite3_free()]. */ const char *sqlite3_sql(sqlite3_stmt *pStmt); +char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database @@ -3853,7 +4060,8 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** other than [SQLITE_ROW] before any subsequent invocation of ** sqlite3_step(). Failure to reset the prepared statement using ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from -** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step() began ** calling [sqlite3_reset()] automatically in this circumstance rather ** than returning [SQLITE_MISUSE]. This is not considered a compatibility ** break because any application that ever receives an SQLITE_MISUSE error @@ -4516,12 +4724,13 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** SQLite will invoke the destructor function X with parameter P exactly ** once, when the metadata is discarded. ** SQLite is free to discard the metadata at any time, including:
      -**
    • when the corresponding function parameter changes, or -**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the -** SQL statement, or -**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or -**
    • during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.
    )^ +**
  • ^(when the corresponding function parameter changes)^, or +**
  • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
  • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
  • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^ ** ** Note the last bullet in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the @@ -5158,7 +5367,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument ** to be invoked whenever a row is updated, inserted or deleted in -** a rowid table. +** a [rowid table]. ** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** @@ -5197,8 +5406,8 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** on the same [database connection] D, or NULL for ** the first call on D. ** -** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] -** interfaces. +** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], +** and [sqlite3_preupdate_hook()] interfaces. */ void *sqlite3_update_hook( sqlite3*, @@ -5215,7 +5424,8 @@ void *sqlite3_update_hook( ** and disabled if the argument is false.)^ ** ** ^Cache sharing is enabled and disabled for an entire process. -** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, ** sharing was enabled or disabled for each thread separately. ** ** ^(The cache sharing mode set by this interface effects all subsequent @@ -5309,7 +5519,8 @@ int sqlite3_db_release_memory(sqlite3*); ** from the heap. ** )^ ** -** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** Beginning with SQLite [version 3.7.3] ([dateof:3.7.3]), +** the soft heap limit is enforced ** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] ** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], ** the soft heap limit is enforced on every memory allocation. Without @@ -5348,7 +5559,7 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** column exists. ^The sqlite3_table_column_metadata() interface returns ** SQLITE_ERROR and if the specified column does not exist. ** ^If the column-name parameter to sqlite3_table_column_metadata() is a -** NULL pointer, then this routine simply checks for the existance of the +** NULL pointer, then this routine simply checks for the existence of the ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it ** does not. ** @@ -5445,9 +5656,18 @@ int sqlite3_table_column_metadata( ** should free this memory by calling [sqlite3_free()]. ** ** ^Extension loading must be enabled using -** [sqlite3_enable_load_extension()] prior to calling this API, +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, ** otherwise an error will be returned. ** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** ** See also the [load_extension() SQL function]. */ int sqlite3_load_extension( @@ -5470,6 +5690,17 @@ int sqlite3_load_extension( ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ +** +** Security warning: It is recommended that extension loading +** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); @@ -5483,7 +5714,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three -** arguments and expects and integer result as if the signature of the +** arguments and expects an integer result as if the signature of the ** entry point where as follows: ** ** 
    @@ -5509,7 +5740,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
 ** See also: [sqlite3_reset_auto_extension()]
 ** and [sqlite3_cancel_auto_extension()]
 */
-int sqlite3_auto_extension(void (*xEntryPoint)(void));
+int sqlite3_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Cancel Automatic Extension Loading
@@ -5521,7 +5752,7 @@ int sqlite3_auto_extension(void (*xEntryPoint)(void));
 ** unregistered and it returns 0 if X was not on the list of initialization
 ** routines.
 */
-int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void));
+int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Reset Automatic Extension Loading
@@ -5683,13 +5914,15 @@ struct sqlite3_module {
 ** the xUpdate method are automatically rolled back by SQLite.
 **
 ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
-** structure for SQLite version 3.8.2. If a virtual table extension is
+** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). 
+** If a virtual table extension is
 ** used with an SQLite version earlier than 3.8.2, the results of attempting 
 ** to read or write the estimatedRows field are undefined (but are likely 
 ** to included crashing the application). The estimatedRows field should
 ** therefore only be used if [sqlite3_libversion_number()] returns a
 ** value greater than or equal to 3008002. Similarly, the idxFlags field
-** was added for version 3.9.0. It may therefore only be used if
+** was added for [version 3.9.0] ([dateof:3.9.0]). 
+** It may therefore only be used if
 ** sqlite3_libversion_number() returns a value greater than or equal to
 ** 3009000.
 */
@@ -6387,7 +6620,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*);
 #define SQLITE_MUTEX_STATIC_MEM       3  /* sqlite3_malloc() */
 #define SQLITE_MUTEX_STATIC_MEM2      4  /* NOT USED */
 #define SQLITE_MUTEX_STATIC_OPEN      4  /* sqlite3BtreeOpen() */
-#define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_random() */
+#define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_randomness() */
 #define SQLITE_MUTEX_STATIC_LRU       6  /* lru page list */
 #define SQLITE_MUTEX_STATIC_LRU2      7  /* NOT USED */
 #define SQLITE_MUTEX_STATIC_PMEM      7  /* sqlite3PageMalloc() */
@@ -6491,6 +6724,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
 #define SQLITE_TESTCTRL_EXPLAIN_STMT            19  /* NOT USED */
+#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD    19
 #define SQLITE_TESTCTRL_NEVER_CORRUPT           20
 #define SQLITE_TESTCTRL_VDBE_COVERAGE           21
 #define SQLITE_TESTCTRL_BYTEORDER               22
@@ -6697,6 +6931,18 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** memory used by all pager caches associated with the database connection.)^
 ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0.
 **
+** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] 
+** ^(
    SQLITE_DBSTATUS_CACHE_USED_SHARED
    
+** 
    This parameter is similar to DBSTATUS_CACHE_USED, except that if a
+** pager cache is shared between two or more connections the bytes of heap
+** memory used by that pager cache is divided evenly between the attached
+** connections.)^  In other words, if none of the pager caches associated
+** with the database connection are shared, this request returns the same
+** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are
+** shared, the value returned by this call will be smaller than that returned
+** by DBSTATUS_CACHE_USED. ^The highwater mark associated with
+** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0.
+**
 ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    
 ** 
    This parameter returns the approximate number of bytes of heap
 ** memory used to store the schema for all databases associated
@@ -6754,7 +7000,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 #define SQLITE_DBSTATUS_CACHE_MISS           8
 #define SQLITE_DBSTATUS_CACHE_WRITE          9
 #define SQLITE_DBSTATUS_DEFERRED_FKS        10
-#define SQLITE_DBSTATUS_MAX                 10   /* Largest defined DBSTATUS */
+#define SQLITE_DBSTATUS_CACHE_USED_SHARED   11
+#define SQLITE_DBSTATUS_MAX                 11   /* Largest defined DBSTATUS */
 
 
 /*
@@ -7108,7 +7355,7 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with
 ** an error.
 **
-** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if 
+** ^A call to sqlite3_backup_init() will fail, returning NULL, if 
 ** there is already a read or read-write transaction open on the 
 ** destination database.
 **
@@ -7489,7 +7736,7 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...);
 ** previously registered write-ahead log callback. ^Note that the
 ** [sqlite3_wal_autocheckpoint()] interface and the
 ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will
-** those overwrite any prior [sqlite3_wal_hook()] settings.
+** overwrite any prior [sqlite3_wal_hook()] settings.
 */
 void *sqlite3_wal_hook(
   sqlite3*, 
@@ -7886,6 +8133,114 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*);
 */
 int sqlite3_db_cacheflush(sqlite3*);
 
+/*
+** CAPI3REF: The pre-update hook.
+**
+** ^These interfaces are only available if SQLite is compiled using the
+** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option.
+**
+** ^The [sqlite3_preupdate_hook()] interface registers a callback function
+** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation
+** on a [rowid table].
+** ^At most one preupdate hook may be registered at a time on a single
+** [database connection]; each call to [sqlite3_preupdate_hook()] overrides
+** the previous setting.
+** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()]
+** with a NULL pointer as the second parameter.
+** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as
+** the first parameter to callbacks.
+**
+** ^The preupdate hook only fires for changes to [rowid tables]; the preupdate
+** hook is not invoked for changes to [virtual tables] or [WITHOUT ROWID]
+** tables.
+**
+** ^The second parameter to the preupdate callback is a pointer to
+** the [database connection] that registered the preupdate hook.
+** ^The third parameter to the preupdate callback is one of the constants
+** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the
+** kind of update operation that is about to occur.
+** ^(The fourth parameter to the preupdate callback is the name of the
+** database within the database connection that is being modified.  This
+** will be "main" for the main database or "temp" for TEMP tables or 
+** the name given after the AS keyword in the [ATTACH] statement for attached
+** databases.)^
+** ^The fifth parameter to the preupdate callback is the name of the
+** table that is being modified.
+** ^The sixth parameter to the preupdate callback is the initial [rowid] of the
+** row being changes for SQLITE_UPDATE and SQLITE_DELETE changes and is
+** undefined for SQLITE_INSERT changes.
+** ^The seventh parameter to the preupdate callback is the final [rowid] of
+** the row being changed for SQLITE_UPDATE and SQLITE_INSERT changes and is
+** undefined for SQLITE_DELETE changes.
+**
+** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()],
+** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces
+** provide additional information about a preupdate event. These routines
+** may only be called from within a preupdate callback.  Invoking any of
+** these routines from outside of a preupdate callback or with a
+** [database connection] pointer that is different from the one supplied
+** to the preupdate callback results in undefined and probably undesirable
+** behavior.
+**
+** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns
+** in the row that is being inserted, updated, or deleted.
+**
+** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to
+** a [protected sqlite3_value] that contains the value of the Nth column of
+** the table row before it is updated.  The N parameter must be between 0
+** and one less than the number of columns or the behavior will be
+** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE
+** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the
+** behavior is undefined.  The [sqlite3_value] that P points to
+** will be destroyed when the preupdate callback returns.
+**
+** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to
+** a [protected sqlite3_value] that contains the value of the Nth column of
+** the table row after it is updated.  The N parameter must be between 0
+** and one less than the number of columns or the behavior will be
+** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE
+** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the
+** behavior is undefined.  The [sqlite3_value] that P points to
+** will be destroyed when the preupdate callback returns.
+**
+** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate
+** callback was invoked as a result of a direct insert, update, or delete
+** operation; or 1 for inserts, updates, or deletes invoked by top-level 
+** triggers; or 2 for changes resulting from triggers called by top-level
+** triggers; and so forth.
+**
+** See also:  [sqlite3_update_hook()]
+*/
+SQLITE_EXPERIMENTAL void *sqlite3_preupdate_hook(
+  sqlite3 *db,
+  void(*xPreUpdate)(
+    void *pCtx,                   /* Copy of third arg to preupdate_hook() */
+    sqlite3 *db,                  /* Database handle */
+    int op,                       /* SQLITE_UPDATE, DELETE or INSERT */
+    char const *zDb,              /* Database name */
+    char const *zName,            /* Table name */
+    sqlite3_int64 iKey1,          /* Rowid of row about to be deleted/updated */
+    sqlite3_int64 iKey2           /* New rowid value (for a rowid UPDATE) */
+  ),
+  void*
+);
+SQLITE_EXPERIMENTAL int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **);
+SQLITE_EXPERIMENTAL int sqlite3_preupdate_count(sqlite3 *);
+SQLITE_EXPERIMENTAL int sqlite3_preupdate_depth(sqlite3 *);
+SQLITE_EXPERIMENTAL int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **);
+
+/*
+** CAPI3REF: Low-level system error code
+**
+** ^Attempt to return the underlying operating system error code or error
+** number that caused the most recent I/O error or failure to open a file.
+** The return value is OS-dependent.  For example, on unix systems, after
+** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be
+** called to get back the underlying "errno" that caused the problem, such
+** as ENOSPC, EAUTH, EISDIR, and so forth.  
+*/
+int sqlite3_system_errno(sqlite3*);
+
 /*
 ** CAPI3REF: Database Snapshot
 ** KEYWORDS: {snapshot}
@@ -7944,17 +8299,30 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
 ** CAPI3REF: Start a read transaction on an historical snapshot
 ** EXPERIMENTAL
 **
-** ^The [sqlite3_snapshot_open(D,S,P)] interface attempts to move the
-** read transaction that is currently open on schema S of
-** [database connection] D so that it refers to historical [snapshot] P.
+** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a
+** read transaction for schema S of
+** [database connection] D such that the read transaction
+** refers to historical [snapshot] P, rather than the most
+** recent change to the database.
 ** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success
 ** or an appropriate [error code] if it fails.
 **
 ** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be
-** the first operation, apart from other sqlite3_snapshot_open() calls,
-** following the [BEGIN] that starts a new read transaction.
-** ^A [snapshot] will fail to open if it has been overwritten by a 
-** [checkpoint].  
+** the first operation following the [BEGIN] that takes the schema S
+** out of [autocommit mode].
+** ^In other words, schema S must not currently be in
+** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the
+** database connection D must be out of [autocommit mode].
+** ^A [snapshot] will fail to open if it has been overwritten by a
+** [checkpoint].
+** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the
+** database connection D does not know that the database file for
+** schema S is in [WAL mode].  A database connection might not know
+** that the database file is in [WAL mode] if there has been no prior
+** I/O on that database connection, or if the database entered [WAL mode] 
+** after the most recent I/O on the database connection.)^
+** (Hint: Run "[PRAGMA application_id]" against a newly opened
+** database connection in order to make it ready to use snapshots.)
 **
 ** The [sqlite3_snapshot_open()] interface is only available when the
 ** SQLITE_ENABLE_SNAPSHOT compile-time option is used.
@@ -7978,6 +8346,33 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_open(
 */
 SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
 
+/*
+** CAPI3REF: Compare the ages of two snapshot handles.
+** EXPERIMENTAL
+**
+** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages
+** of two valid snapshot handles. 
+**
+** If the two snapshot handles are not associated with the same database 
+** file, the result of the comparison is undefined. 
+**
+** Additionally, the result of the comparison is only valid if both of the
+** snapshot handles were obtained by calling sqlite3_snapshot_get() since the
+** last time the wal file was deleted. The wal file is deleted when the
+** database is changed back to rollback mode or when the number of database
+** clients drops to zero. If either snapshot handle was obtained before the 
+** wal file was last deleted, the value returned by this function 
+** is undefined.
+**
+** Otherwise, this API returns a negative value if P1 refers to an older
+** snapshot than P2, zero if the two handles refer to the same database
+** snapshot, and a positive value if P1 is a newer snapshot than P2.
+*/
+SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
+  sqlite3_snapshot *p1,
+  sqlite3_snapshot *p2
+);
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
@@ -7989,4 +8384,4 @@ SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
 #ifdef __cplusplus
 }  /* End of the 'extern "C"' block */
 #endif
-#endif /* _SQLITE3_H_ */
+#endif /* SQLITE3_H */
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 2e1c764a..ce87e746 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -15,12 +15,10 @@
 ** as extensions by SQLite should #include this file instead of 
 ** sqlite3.h.
 */
-#ifndef _SQLITE3EXT_H_
-#define _SQLITE3EXT_H_
+#ifndef SQLITE3EXT_H
+#define SQLITE3EXT_H
 #include "sqlite3.h"
 
-typedef struct sqlite3_api_routines sqlite3_api_routines;
-
 /*
 ** The following structure holds pointers to all of the SQLite API
 ** routines.
@@ -279,8 +277,23 @@ struct sqlite3_api_routines {
   int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int);
   int (*strlike)(const char*,const char*,unsigned int);
   int (*db_cacheflush)(sqlite3*);
+  /* Version 3.12.0 and later */
+  int (*system_errno)(sqlite3*);
+  /* Version 3.14.0 and later */
+  int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
+  char *(*expanded_sql)(sqlite3_stmt*);
 };
 
+/*
+** This is the function signature used for all extension entry points.  It
+** is also defined in the file "loadext.c".
+*/
+typedef int (*sqlite3_loadext_entry)(
+  sqlite3 *db,                       /* Handle to the database. */
+  char **pzErrMsg,                   /* Used to set error string on failure. */
+  const sqlite3_api_routines *pThunk /* Extension API function pointers. */
+);
+
 /*
 ** The following macros redefine the API routines so that they are
 ** redirected through the global sqlite3_api structure.
@@ -522,6 +535,11 @@ struct sqlite3_api_routines {
 #define sqlite3_status64               sqlite3_api->status64
 #define sqlite3_strlike                sqlite3_api->strlike
 #define sqlite3_db_cacheflush          sqlite3_api->db_cacheflush
+/* Version 3.12.0 and later */
+#define sqlite3_system_errno           sqlite3_api->system_errno
+/* Version 3.14.0 and later */
+#define sqlite3_trace_v2               sqlite3_api->trace_v2
+#define sqlite3_expanded_sql           sqlite3_api->expanded_sql
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
@@ -539,4 +557,4 @@ struct sqlite3_api_routines {
 # define SQLITE_EXTENSION_INIT3     /*no-op*/
 #endif
 
-#endif /* _SQLITE3EXT_H_ */
+#endif /* SQLITE3EXT_H */
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f04aebe7..98cfcb4e 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -12,8 +12,51 @@
 ** Internal interface definitions for SQLite.
 **
 */
-#ifndef _SQLITEINT_H_
-#define _SQLITEINT_H_
+#ifndef SQLITEINT_H
+#define SQLITEINT_H
+
+/* Special Comments:
+**
+** Some comments have special meaning to the tools that measure test
+** coverage:
+**
+**    NO_TEST                     - The branches on this line are not
+**                                  measured by branch coverage.  This is
+**                                  used on lines of code that actually
+**                                  implement parts of coverage testing.
+**
+**    OPTIMIZATION-IF-TRUE        - This branch is allowed to alway be false
+**                                  and the correct answer is still obtained,
+**                                  though perhaps more slowly.
+**
+**    OPTIMIZATION-IF-FALSE       - This branch is allowed to alway be true
+**                                  and the correct answer is still obtained,
+**                                  though perhaps more slowly.
+**
+**    PREVENTS-HARMLESS-OVERREAD  - This branch prevents a buffer overread
+**                                  that would be harmless and undetectable
+**                                  if it did occur.  
+**
+** In all cases, the special comment must be enclosed in the usual
+** slash-asterisk...asterisk-slash comment marks, with no spaces between the 
+** asterisks and the comment text.
+*/
+
+/*
+** Make sure the Tcl calling convention macro is defined.  This macro is
+** only used by test code and Tcl integration code.
+*/
+#ifndef SQLITE_TCLAPI
+#  define SQLITE_TCLAPI
+#endif
+
+/*
+** Make sure that rand_s() is available on Windows systems with MSVC 2005
+** or higher.
+*/
+#if defined(_MSC_VER) && _MSC_VER>=1400
+#  define _CRT_RAND_S
+#endif
 
 /*
 ** Include the header file used to customize the compiler options for MSVC.
@@ -149,7 +192,7 @@
 ** to the next, so we have developed the following set of #if statements
 ** to generate appropriate macros for a wide range of compilers.
 **
-** The correct "ANSI" way to do this is to use the intptr_t type. 
+** The correct "ANSI" way to do this is to use the intptr_t type.
 ** Unfortunately, that typedef is not available on all compilers, or
 ** if it is available, it requires an #include of specific headers
 ** that vary from one machine to the next.
@@ -173,21 +216,6 @@
 # define SQLITE_PTR_TO_INT(X)  ((int)(X))
 #endif
 
-/*
-** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to
-** something between S (inclusive) and E (exclusive).
-**
-** In other words, S is a buffer and E is a pointer to the first byte after
-** the end of buffer S.  This macro returns true if P points to something
-** contained within the buffer S.
-*/
-#if defined(HAVE_STDINT_H)
-# define SQLITE_WITHIN(P,S,E) \
-    ((uintptr_t)(P)>=(uintptr_t)(S) && (uintptr_t)(P)<(uintptr_t)(E))
-#else
-# define SQLITE_WITHIN(P,S,E) ((P)>=(S) && (P)<(E))
-#endif
-
 /*
 ** A macro to hint to the compiler that a function should not be
 ** inlined.
@@ -206,7 +234,7 @@
 ** the SQLITE_DISABLE_INTRINSIC define.
 */
 #if !defined(SQLITE_DISABLE_INTRINSIC)
-#  if defined(_MSC_VER) && _MSC_VER>=1300
+#  if defined(_MSC_VER) && _MSC_VER>=1400
 #    if !defined(_WIN32_WCE)
 #      include 
 #      pragma intrinsic(_byteswap_ushort)
@@ -316,7 +344,7 @@
 ** is set.  Thus NDEBUG becomes an opt-in rather than an opt-out
 ** feature.
 */
-#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
 # define NDEBUG 1
 #endif
 #if defined(NDEBUG) && defined(SQLITE_DEBUG)
@@ -331,7 +359,7 @@
 #endif
 
 /*
-** The testcase() macro is used to aid in coverage testing.  When 
+** The testcase() macro is used to aid in coverage testing.  When
 ** doing coverage testing, the condition inside the argument to
 ** testcase() must be evaluated both true and false in order to
 ** get full branch coverage.  The testcase() macro is inserted
@@ -377,7 +405,7 @@
 #endif
 
 /*
-** The ALWAYS and NEVER macros surround boolean expressions which 
+** The ALWAYS and NEVER macros surround boolean expressions which
 ** are intended to always be true or false, respectively.  Such
 ** expressions could be omitted from the code completely.  But they
 ** are included in a few cases in order to enhance the resilience
@@ -391,7 +419,7 @@
 ** be true and false so that the unreachable code they specify will
 ** not be counted as untested code.
 */
-#if defined(SQLITE_COVERAGE_TEST)
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
 # define ALWAYS(X)      (1)
 # define NEVER(X)       (0)
 #elif !defined(NDEBUG)
@@ -443,6 +471,13 @@
 # undef  SQLITE_NEED_ERR_NAME
 #endif
 
+/*
+** SQLITE_ENABLE_EXPLAIN_COMMENTS is incompatible with SQLITE_OMIT_EXPLAIN
+*/
+#ifdef SQLITE_OMIT_EXPLAIN
+# undef SQLITE_ENABLE_EXPLAIN_COMMENTS
+#endif
+
 /*
 ** Return true (non-zero) if the input is an integer that is too large
 ** to fit in 32-bits.  This macro is used inside of various testcase()
@@ -490,7 +525,7 @@
 
 /*
 ** OMIT_TEMPDB is set to 1 if SQLITE_OMIT_TEMPDB is defined, or 0
-** afterward. Having this macro allows us to cause the C compiler 
+** afterward. Having this macro allows us to cause the C compiler
 ** to omit code used by TEMP tables without messy #ifndef statements.
 */
 #ifdef SQLITE_OMIT_TEMPDB
@@ -529,7 +564,7 @@
 
 /*
 ** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if
-** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it 
+** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it
 ** to zero.
 */
 #if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0
@@ -568,8 +603,12 @@
 /*
 ** Macros to compute minimum and maximum of two numbers.
 */
-#define MIN(A,B) ((A)<(B)?(A):(B))
-#define MAX(A,B) ((A)>(B)?(A):(B))
+#ifndef MIN
+# define MIN(A,B) ((A)<(B)?(A):(B))
+#endif
+#ifndef MAX
+# define MAX(A,B) ((A)>(B)?(A):(B))
+#endif
 
 /*
 ** Swap two objects of type TYPE.
@@ -677,7 +716,7 @@ typedef INT8_TYPE i8;              /* 1-byte signed integer */
 **      4 -> 20           1000 -> 99        1048576 -> 200
 **     10 -> 33           1024 -> 100    4294967296 -> 320
 **
-** The LogEst can be negative to indicate fractional values. 
+** The LogEst can be negative to indicate fractional values.
 ** Examples:
 **
 **    0.5 -> -10           0.1 -> -33        0.0625 -> -40
@@ -698,6 +737,27 @@ typedef INT16_TYPE LogEst;
 # endif
 #endif
 
+/* The uptr type is an unsigned integer large enough to hold a pointer
+*/
+#if defined(HAVE_STDINT_H)
+  typedef uintptr_t uptr;
+#elif SQLITE_PTRSIZE==4
+  typedef u32 uptr;
+#else
+  typedef u64 uptr;
+#endif
+
+/*
+** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to
+** something between S (inclusive) and E (exclusive).
+**
+** In other words, S is a buffer and E is a pointer to the first byte after
+** the end of buffer S.  This macro returns true if P points to something
+** contained within the buffer S.
+*/
+#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E)))
+
+
 /*
 ** Macros to determine whether the machine is big or little endian,
 ** and whether or not that determination is run-time or compile-time.
@@ -743,7 +803,7 @@ typedef INT16_TYPE LogEst;
 #define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
 #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
 
-/* 
+/*
 ** Round up a number to the next larger multiple of 8.  This is used
 ** to force 8-byte alignment on 64-bit architectures.
 */
@@ -837,7 +897,7 @@ typedef INT16_TYPE LogEst;
 
 /*
 ** An instance of the following structure is used to store the busy-handler
-** callback for a given sqlite handle. 
+** callback for a given sqlite handle.
 **
 ** The sqlite.busyHandler member of the sqlite struct contains the busy
 ** callback for the database handle. Each pager opened via the sqlite
@@ -882,9 +942,9 @@ struct BusyHandler {
 
 /*
 ** The following value as a destructor means to use sqlite3DbFree().
-** The sqlite3DbFree() routine requires two parameters instead of the 
-** one parameter that destructors normally want.  So we have to introduce 
-** this magic value that the code knows to handle differently.  Any 
+** The sqlite3DbFree() routine requires two parameters instead of the
+** one parameter that destructors normally want.  So we have to introduce
+** this magic value that the code knows to handle differently.  Any
 ** pointer will work here as long as it is distinct from SQLITE_STATIC
 ** and SQLITE_TRANSIENT.
 */
@@ -911,16 +971,16 @@ struct BusyHandler {
   int sqlite3_wsd_init(int N, int J);
   void *sqlite3_wsd_find(void *K, int L);
 #else
-  #define SQLITE_WSD 
+  #define SQLITE_WSD
   #define GLOBAL(t,v) v
   #define sqlite3GlobalConfig sqlite3Config
 #endif
 
 /*
 ** The following macros are used to suppress compiler warnings and to
-** make it clear to human readers when a function parameter is deliberately 
+** make it clear to human readers when a function parameter is deliberately
 ** left unused within the body of a function. This usually happens when
-** a function is called via a function pointer. For example the 
+** a function is called via a function pointer. For example the
 ** implementation of an SQL aggregate step callback may not use the
 ** parameter indicating the number of arguments passed to the aggregate,
 ** if it knows that this is enforced elsewhere.
@@ -963,6 +1023,7 @@ typedef struct LookasideSlot LookasideSlot;
 typedef struct Module Module;
 typedef struct NameContext NameContext;
 typedef struct Parse Parse;
+typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
 typedef struct RowSet RowSet;
 typedef struct Savepoint Savepoint;
@@ -986,7 +1047,7 @@ typedef struct WhereInfo WhereInfo;
 typedef struct With With;
 
 /*
-** Defer sourcing vdbe.h and btree.h until after the "u8" and 
+** Defer sourcing vdbe.h and btree.h until after the "u8" and
 ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
 ** pointer types (i.e. FuncDef) defined above.
 */
@@ -994,10 +1055,39 @@ typedef struct With With;
 #include "vdbe.h"
 #include "pager.h"
 #include "pcache.h"
-
 #include "os.h"
 #include "mutex.h"
 
+/* The SQLITE_EXTRA_DURABLE compile-time option used to set the default
+** synchronous setting to EXTRA.  It is no longer supported.
+*/
+#ifdef SQLITE_EXTRA_DURABLE
+# warning Use SQLITE_DEFAULT_SYNCHRONOUS=3 instead of SQLITE_EXTRA_DURABLE
+# define SQLITE_DEFAULT_SYNCHRONOUS 3
+#endif
+
+/*
+** Default synchronous levels.
+**
+** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ
+** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1.
+**
+**           PAGER_SYNCHRONOUS       DEFAULT_SYNCHRONOUS
+**   OFF           1                         0
+**   NORMAL        2                         1
+**   FULL          3                         2
+**   EXTRA         4                         3
+**
+** The "PRAGMA synchronous" statement also uses the zero-based numbers.
+** In other words, the zero-based numbers are used for all external interfaces
+** and the one-based values are used internally.
+*/
+#ifndef SQLITE_DEFAULT_SYNCHRONOUS
+# define SQLITE_DEFAULT_SYNCHRONOUS (PAGER_SYNCHRONOUS_FULL-1)
+#endif
+#ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS
+# define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS
+#endif
 
 /*
 ** Each database file to be accessed by the system is an instance
@@ -1007,9 +1097,10 @@ typedef struct With With;
 ** databases may be attached.
 */
 struct Db {
-  char *zName;         /* Name of this database */
+  char *zDbSName;      /* Name of this database. (schema name, not filename) */
   Btree *pBt;          /* The B*Tree structure for this database file */
   u8 safety_level;     /* How aggressive at syncing data to disk */
+  u8 bSyncSet;         /* True if "PRAGMA synchronous=N" has been run */
   Schema *pSchema;     /* Pointer to database schema (possibly shared) */
 };
 
@@ -1020,7 +1111,7 @@ struct Db {
 ** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing.
 ** In shared cache mode, a single Schema object can be shared by multiple
 ** Btrees that refer to the same underlying BtShared object.
-** 
+**
 ** Schema objects are automatically deallocated when the last Btree that
 ** references them is destroyed.   The TEMP Schema is manually freed by
 ** sqlite3_close().
@@ -1045,7 +1136,7 @@ struct Schema {
 };
 
 /*
-** These macros can be used to test, set, or clear bits in the 
+** These macros can be used to test, set, or clear bits in the
 ** Db.pSchema->flags field.
 */
 #define DbHasProperty(D,I,P)     (((D)->aDb[I].pSchema->schemaFlags&(P))==(P))
@@ -1109,13 +1200,15 @@ struct LookasideSlot {
 };
 
 /*
-** A hash table for function definitions.
+** A hash table for built-in function definitions.  (Application-defined
+** functions use a regular table table from hash.h.)
 **
 ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots.
-** Collisions are on the FuncDef.pHash chain.
+** Collisions are on the FuncDef.u.pHash chain.
 */
+#define SQLITE_FUNC_HASH_SZ 23
 struct FuncDefHash {
-  FuncDef *a[23];       /* Hash table for functions */
+  FuncDef *a[SQLITE_FUNC_HASH_SZ];       /* Hash table for functions */
 };
 
 #ifdef SQLITE_USER_AUTHENTICATION
@@ -1156,6 +1249,15 @@ void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**);
                                const char*);
 #endif
 
+#ifndef SQLITE_OMIT_DEPRECATED
+/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
+** in the style of sqlite3_trace()
+*/
+#define SQLITE_TRACE_LEGACY  0x80
+#else
+#define SQLITE_TRACE_LEGACY  0
+#endif /* SQLITE_OMIT_DEPRECATED */
+
 
 /*
 ** Each database connection is an instance of the following structure.
@@ -1173,6 +1275,7 @@ struct sqlite3 {
   unsigned int openFlags;       /* Flags passed to sqlite3_vfs.xOpen() */
   int errCode;                  /* Most recent error code (SQLITE_*) */
   int errMask;                  /* & result codes with this before returning */
+  int iSysErrno;                /* Errno value from last system error */
   u16 dbOptFlags;               /* Flags to enable/disable optimizations */
   u8 enc;                       /* Text encoding */
   u8 autoCommit;                /* The auto-commit flag. */
@@ -1184,6 +1287,7 @@ struct sqlite3 {
   u8 suppressErr;               /* Do not issue error messages if true */
   u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
   u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
+  u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1204,16 +1308,23 @@ struct sqlite3 {
   int nVDestroy;                /* Number of active OP_VDestroy operations */
   int nExtension;               /* Number of loaded extensions */
   void **aExtension;            /* Array of shared library handles */
-  void (*xTrace)(void*,const char*);        /* Trace function */
+  int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
   void *pProfileArg;                        /* Argument to profile function */
-  void *pCommitArg;                 /* Argument to xCommitCallback() */   
+  void *pCommitArg;                 /* Argument to xCommitCallback() */
   int (*xCommitCallback)(void*);    /* Invoked at every commit. */
-  void *pRollbackArg;               /* Argument to xRollbackCallback() */   
+  void *pRollbackArg;               /* Argument to xRollbackCallback() */
   void (*xRollbackCallback)(void*); /* Invoked at every commit. */
   void *pUpdateArg;
   void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  void *pPreUpdateArg;          /* First argument to xPreUpdateCallback */
+  void (*xPreUpdateCallback)(   /* Registered using sqlite3_preupdate_hook() */
+    void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64
+  );
+  PreUpdate *pPreUpdate;        /* Context for active pre-update callback */
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 #ifndef SQLITE_OMIT_WAL
   int (*xWalCallback)(void *, sqlite3 *, const char *, int);
   void *pWalArg;
@@ -1243,7 +1354,7 @@ struct sqlite3 {
   VTable **aVTrans;             /* Virtual tables with open transactions */
   VTable *pDisconnect;    /* Disconnect these in next sqlite3_prepare() */
 #endif
-  FuncDefHash aFunc;            /* Hash table of connection functions */
+  Hash aFunc;                   /* Hash table of connection functions */
   Hash aCollSeq;                /* All collating sequences */
   BusyHandler busyHandler;      /* Busy callback */
   Db aDbStatic[2];              /* Static space for the 2 default backends */
@@ -1255,8 +1366,8 @@ struct sqlite3 {
   i64 nDeferredImmCons;         /* Net deferred immediate constraints */
   int *pnBytesFreed;            /* If not NULL, increment this in DbFree() */
 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
-  /* The following variables are all protected by the STATIC_MASTER 
-  ** mutex, not by sqlite3.mutex. They are used by code in notify.c. 
+  /* The following variables are all protected by the STATIC_MASTER
+  ** mutex, not by sqlite3.mutex. They are used by code in notify.c.
   **
   ** When X.pUnlockConnection==Y, that means that X is waiting for Y to
   ** unlock so that it can proceed.
@@ -1284,6 +1395,11 @@ struct sqlite3 {
 
 /*
 ** Possible values for the sqlite3.flags.
+**
+** Value constraints (enforced via assert()):
+**      SQLITE_FullFSync     == PAGER_FULLFSYNC
+**      SQLITE_CkptFullFSync == PAGER_CKPT_FULLFSYNC
+**      SQLITE_CacheSpill    == PAGER_CACHE_SPILL
 */
 #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
 #define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
@@ -1311,12 +1427,14 @@ struct sqlite3 {
 #define SQLITE_AutoIndex      0x00100000  /* Enable automatic indexes */
 #define SQLITE_PreferBuiltin  0x00200000  /* Preference to built-in funcs */
 #define SQLITE_LoadExtension  0x00400000  /* Enable load_extension */
-#define SQLITE_EnableTrigger  0x00800000  /* True to enable triggers */
-#define SQLITE_DeferFKs       0x01000000  /* Defer all FK constraints */
-#define SQLITE_QueryOnly      0x02000000  /* Disable database changes */
-#define SQLITE_VdbeEQP        0x04000000  /* Debug EXPLAIN QUERY PLAN */
-#define SQLITE_Vacuum         0x08000000  /* Currently in a VACUUM */
-#define SQLITE_CellSizeCk     0x10000000  /* Check btree cell sizes on load */
+#define SQLITE_LoadExtFunc    0x00800000  /* Enable load_extension() SQL func */
+#define SQLITE_EnableTrigger  0x01000000  /* True to enable triggers */
+#define SQLITE_DeferFKs       0x02000000  /* Defer all FK constraints */
+#define SQLITE_QueryOnly      0x04000000  /* Disable database changes */
+#define SQLITE_VdbeEQP        0x08000000  /* Debug EXPLAIN QUERY PLAN */
+#define SQLITE_Vacuum         0x10000000  /* Currently in a VACUUM */
+#define SQLITE_CellSizeCk     0x20000000  /* Check btree cell sizes on load */
+#define SQLITE_Fts3Tokenizer  0x40000000  /* Enable fts3_tokenizer(2) */
 
 
 /*
@@ -1370,27 +1488,33 @@ struct sqlite3 {
 
 /*
 ** Each SQL function is defined by an instance of the following
-** structure.  A pointer to this structure is stored in the sqlite.aFunc
-** hash table.  When multiple functions have the same name, the hash table
-** points to a linked list of these structures.
+** structure.  For global built-in functions (ex: substr(), max(), count())
+** a pointer to this structure is held in the sqlite3BuiltinFunctions object.
+** For per-connection application-defined functions, a pointer to this
+** structure is held in the db->aHash hash table.
+**
+** The u.pHash field is used by the global built-ins.  The u.pDestructor
+** field is used by per-connection app-def functions.
 */
 struct FuncDef {
-  i16 nArg;            /* Number of arguments.  -1 means unlimited */
+  i8 nArg;             /* Number of arguments.  -1 means unlimited */
   u16 funcFlags;       /* Some combination of SQLITE_FUNC_* */
   void *pUserData;     /* User data parameter */
   FuncDef *pNext;      /* Next function with same name */
   void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */
   void (*xFinalize)(sqlite3_context*);                  /* Agg finalizer */
-  char *zName;         /* SQL name of the function. */
-  FuncDef *pHash;      /* Next with a different name but the same hash */
-  FuncDestructor *pDestructor;   /* Reference counted destructor function */
+  const char *zName;   /* SQL name of the function. */
+  union {
+    FuncDef *pHash;      /* Next with a different name but the same hash */
+    FuncDestructor *pDestructor;   /* Reference counted destructor function */
+  } u;
 };
 
 /*
 ** This structure encapsulates a user-function destructor callback (as
 ** configured using create_function_v2()) and a reference counter. When
 ** create_function_v2() is called to create a function with a destructor,
-** a single object of this type is allocated. FuncDestructor.nRef is set to 
+** a single object of this type is allocated. FuncDestructor.nRef is set to
 ** the number of FuncDef objects created (either 1 or 3, depending on whether
 ** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor
 ** member of each of the new FuncDef objects is set to point to the allocated
@@ -1411,6 +1535,13 @@ struct FuncDestructor {
 ** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG.  And
 ** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC.  There
 ** are assert() statements in the code to verify this.
+**
+** Value constraints (enforced via assert()):
+**     SQLITE_FUNC_MINMAX    ==  NC_MinMaxAgg      == SF_MinMaxAgg
+**     SQLITE_FUNC_LENGTH    ==  OPFLAG_LENGTHARG
+**     SQLITE_FUNC_TYPEOF    ==  OPFLAG_TYPEOFARG
+**     SQLITE_FUNC_CONSTANT  ==  SQLITE_DETERMINISTIC from the API
+**     SQLITE_FUNC_ENCMASK   depends on SQLITE_UTF* macros in the API
 */
 #define SQLITE_FUNC_ENCMASK  0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */
 #define SQLITE_FUNC_LIKE     0x0004 /* Candidate for the LIKE optimization */
@@ -1432,10 +1563,10 @@ struct FuncDestructor {
 ** used to create the initializers for the FuncDef structures.
 **
 **   FUNCTION(zName, nArg, iArg, bNC, xFunc)
-**     Used to create a scalar function definition of a function zName 
+**     Used to create a scalar function definition of a function zName
 **     implemented by C function xFunc that accepts nArg arguments. The
 **     value passed as iArg is cast to a (void*) and made available
-**     as the user-data (sqlite3_user_data()) for the function. If 
+**     as the user-data (sqlite3_user_data()) for the function. If
 **     argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set.
 **
 **   VFUNCTION(zName, nArg, iArg, bNC, xFunc)
@@ -1454,8 +1585,8 @@ struct FuncDestructor {
 **     FUNCTION().
 **
 **   LIKEFUNC(zName, nArg, pArg, flags)
-**     Used to create a scalar function definition of a function zName 
-**     that accepts nArg arguments and is implemented by a call to C 
+**     Used to create a scalar function definition of a function zName
+**     that accepts nArg arguments and is implemented by a call to C
 **     function likeFunc. Argument pArg is cast to a (void *) and made
 **     available as the function user-data (sqlite3_user_data()). The
 **     FuncDef.flags variable is set to the value passed as the flags
@@ -1463,28 +1594,28 @@ struct FuncDestructor {
 */
 #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} }
 #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} }
 #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} }
 #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \
   {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\
-   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, 0, 0}
+   SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} }
 #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \
   {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \
-   pArg, 0, xFunc, 0, #zName, 0, 0}
+   pArg, 0, xFunc, 0, #zName, }
 #define LIKEFUNC(zName, nArg, arg, flags) \
   {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \
-   (void *)arg, 0, likeFunc, 0, #zName, 0, 0}
+   (void *)arg, 0, likeFunc, 0, #zName, {0} }
 #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \
   {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \
-   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0}
+   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}}
 #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \
   {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \
-   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName,0,0}
+   SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}}
 
 /*
 ** All current savepoints are stored in a linked list starting at
@@ -1526,10 +1657,8 @@ struct Module {
 ** of this structure.
 */
 struct Column {
-  char *zName;     /* Name of this column */
+  char *zName;     /* Name of this column, \000, then the type */
   Expr *pDflt;     /* Default value of this column */
-  char *zDflt;     /* Original text of the default value */
-  char *zType;     /* Data type for this column */
   char *zColl;     /* Collating sequence.  If NULL, use the default */
   u8 notNull;      /* An OE_ code for handling a NOT NULL constraint */
   char affinity;   /* One of the SQLITE_AFF_... values */
@@ -1541,6 +1670,7 @@ struct Column {
 */
 #define COLFLAG_PRIMKEY  0x0001    /* Column is part of the primary key */
 #define COLFLAG_HIDDEN   0x0002    /* A hidden column in a virtual table */
+#define COLFLAG_HASTYPE  0x0004    /* Type name follows column name */
 
 /*
 ** A "Collating Sequence" is defined by an instance of the following
@@ -1571,7 +1701,7 @@ struct CollSeq {
 **
 ** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and
 ** 't' for SQLITE_AFF_TEXT.  But we can save a little space and improve
-** the speed a little by numbering the values consecutively.  
+** the speed a little by numbering the values consecutively.
 **
 ** But rather than start with 0 or 1, we begin with 'A'.  That way,
 ** when multiple affinity types are concatenated into a string and
@@ -1590,7 +1720,7 @@ struct CollSeq {
 
 /*
 ** The SQLITE_AFF_MASK values masks off the significant bits of an
-** affinity value. 
+** affinity value.
 */
 #define SQLITE_AFF_MASK     0x47
 
@@ -1603,6 +1733,7 @@ struct CollSeq {
 ** operator is NULL.  It is added to certain comparison operators to
 ** prove that the operands are always NOT NULL.
 */
+#define SQLITE_KEEPNULL     0x08  /* Used by vector == or <> */
 #define SQLITE_JUMPIFNULL   0x10  /* jumps if either operand is NULL */
 #define SQLITE_STOREP2      0x20  /* Store result in reg[P2] rather than jump */
 #define SQLITE_NULLEQ       0x80  /* NULL=NULL */
@@ -1610,20 +1741,20 @@ struct CollSeq {
 
 /*
 ** An object of this type is created for each virtual table present in
-** the database schema. 
+** the database schema.
 **
 ** If the database schema is shared, then there is one instance of this
 ** structure for each database connection (sqlite3*) that uses the shared
 ** schema. This is because each database connection requires its own unique
-** instance of the sqlite3_vtab* handle used to access the virtual table 
-** implementation. sqlite3_vtab* handles can not be shared between 
-** database connections, even when the rest of the in-memory database 
+** instance of the sqlite3_vtab* handle used to access the virtual table
+** implementation. sqlite3_vtab* handles can not be shared between
+** database connections, even when the rest of the in-memory database
 ** schema is shared, as the implementation often stores the database
 ** connection handle passed to it via the xConnect() or xCreate() method
 ** during initialization internally. This database connection handle may
-** then be used by the virtual table implementation to access real tables 
-** within the database. So that they appear as part of the callers 
-** transaction, these accesses need to be made via the same database 
+** then be used by the virtual table implementation to access real tables
+** within the database. So that they appear as part of the callers
+** transaction, these accesses need to be made via the same database
 ** connection as that used to execute SQL operations on the virtual table.
 **
 ** All VTable objects that correspond to a single table in a shared
@@ -1635,19 +1766,19 @@ struct CollSeq {
 ** sqlite3_vtab* handle in the compiled query.
 **
 ** When an in-memory Table object is deleted (for example when the
-** schema is being reloaded for some reason), the VTable objects are not 
-** deleted and the sqlite3_vtab* handles are not xDisconnect()ed 
+** schema is being reloaded for some reason), the VTable objects are not
+** deleted and the sqlite3_vtab* handles are not xDisconnect()ed
 ** immediately. Instead, they are moved from the Table.pVTable list to
 ** another linked list headed by the sqlite3.pDisconnect member of the
-** corresponding sqlite3 structure. They are then deleted/xDisconnected 
+** corresponding sqlite3 structure. They are then deleted/xDisconnected
 ** next time a statement is prepared using said sqlite3*. This is done
 ** to avoid deadlock issues involving multiple sqlite3.mutex mutexes.
 ** Refer to comments above function sqlite3VtabUnlockList() for an
 ** explanation as to why it is safe to add an entry to an sqlite3.pDisconnect
 ** list without holding the corresponding sqlite3.mutex mutex.
 **
-** The memory for objects of this type is always allocated by 
-** sqlite3DbMalloc(), using the connection handle stored in VTable.db as 
+** The memory for objects of this type is always allocated by
+** sqlite3DbMalloc(), using the connection handle stored in VTable.db as
 ** the first argument.
 */
 struct VTable {
@@ -1815,7 +1946,7 @@ struct FKey {
 ** key is set to NULL.  CASCADE means that a DELETE or UPDATE of the
 ** referenced table row is propagated into the row that holds the
 ** foreign key.
-** 
+**
 ** The following symbolic values are used to record which type
 ** of action to take.
 */
@@ -1836,7 +1967,7 @@ struct FKey {
 
 /*
 ** An instance of the following structure is passed as the first
-** argument to sqlite3VdbeKeyCompare and is used to control the 
+** argument to sqlite3VdbeKeyCompare and is used to control the
 ** comparison of the two index keys.
 **
 ** Note that aSortOrder[] and aColl[] have nField+1 slots.  There
@@ -1877,7 +2008,7 @@ struct KeyInfo {
 ** The key comparison functions actually return default_rc when they find
 ** an equals comparison.  default_rc can be -1, 0, or +1.  If there are
 ** multiple entries in the b-tree with the same key (when only looking
-** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to 
+** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to
 ** cause the search to find the last match, or +1 to cause the search to
 ** find the first match.
 **
@@ -1914,7 +2045,7 @@ struct UnpackedRecord {
 ** In the Table structure describing Ex1, nCol==3 because there are
 ** three columns in the table.  In the Index structure describing
 ** Ex2, nColumn==2 since 2 of the 3 columns of Ex1 are indexed.
-** The value of aiColumn is {2, 0}.  aiColumn[0]==2 because the 
+** The value of aiColumn is {2, 0}.  aiColumn[0]==2 because the
 ** first column to be indexed (c3) has an index of 2 in Ex1.aCol[].
 ** The second column to be indexed (c1) has an index of 0 in
 ** Ex1.aCol[], hence Ex2.aiColumn[1]==0.
@@ -1922,7 +2053,7 @@ struct UnpackedRecord {
 ** The Index.onError field determines whether or not the indexed columns
 ** must be unique and what to do if they are not.  When Index.onError=OE_None,
 ** it means this is not a unique index.  Otherwise it is a unique index
-** and the value of Index.onError indicate the which conflict resolution 
+** and the value of Index.onError indicate the which conflict resolution
 ** algorithm to employ whenever an attempt is made to insert a non-unique
 ** element.
 **
@@ -1987,7 +2118,7 @@ struct Index {
 #define XN_EXPR      (-2)     /* Indexed column is an expression */
 
 /*
-** Each sample stored in the sqlite_stat3 table is represented in memory 
+** Each sample stored in the sqlite_stat3 table is represented in memory
 ** using a structure of this type.  See documentation at the top of the
 ** analyze.c source file for additional information.
 */
@@ -2082,9 +2213,9 @@ typedef int ynVar;
 ** to represent the greater-than-or-equal-to operator in the expression
 ** tree.
 **
-** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, 
+** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB,
 ** or TK_STRING), then Expr.token contains the text of the SQL literal. If
-** the expression is a variable (TK_VARIABLE), then Expr.token contains the 
+** the expression is a variable (TK_VARIABLE), then Expr.token contains the
 ** variable name. Finally, if the expression is an SQL function (TK_FUNCTION),
 ** then Expr.token contains the name of the function.
 **
@@ -2095,7 +2226,7 @@ typedef int ynVar;
 ** a CASE expression or an IN expression of the form " IN (, ...)".
 ** Expr.x.pSelect is used if the expression is a sub-select or an expression of
 ** the form " IN (SELECT ...)". If the EP_xIsSelect bit is set in the
-** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is 
+** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is
 ** valid.
 **
 ** An expression of the form ID or ID.ID refers to a column in a table.
@@ -2106,8 +2237,8 @@ typedef int ynVar;
 ** value is also stored in the Expr.iAgg column in the aggregate so that
 ** it can be accessed after all aggregates are computed.
 **
-** If the expression is an unbound variable marker (a question mark 
-** character '?' in the original SQL) then the Expr.iTable holds the index 
+** If the expression is an unbound variable marker (a question mark
+** character '?' in the original SQL) then the Expr.iTable holds the index
 ** number for that variable.
 **
 ** If the expression is a subquery then Expr.iColumn holds an integer
@@ -2146,7 +2277,7 @@ struct Expr {
 
   /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no
   ** space is allocated for the fields below this point. An attempt to
-  ** access them will result in a segfault or malfunction. 
+  ** access them will result in a segfault or malfunction.
   *********************************************************************/
 
   Expr *pLeft;           /* Left subnode */
@@ -2167,9 +2298,11 @@ struct Expr {
   int iTable;            /* TK_COLUMN: cursor number of table holding column
                          ** TK_REGISTER: register number
                          ** TK_TRIGGER: 1 -> new, 0 -> old
-                         ** EP_Unlikely:  134217728 times likelihood */
+                         ** EP_Unlikely:  134217728 times likelihood
+                         ** TK_SELECT: 1st register of result vector */
   ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
-                         ** TK_VARIABLE: variable number (always >= 1). */
+                         ** TK_VARIABLE: variable number (always >= 1).
+                         ** TK_SELECT_COLUMN: column of the result vector */
   i16 iAgg;              /* Which entry in pAggInfo->aCol[] or ->aFunc[] */
   i16 iRightJoinTable;   /* If EP_FromJoin, the right table of the join */
   u8 op2;                /* TK_REGISTER: original value of Expr.op
@@ -2205,6 +2338,7 @@ struct Expr {
 #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */
 #define EP_Subquery  0x200000 /* Tree contains a TK_SELECT operator */
 #define EP_Alias     0x400000 /* Is an alias for a result set column */
+#define EP_Leaf      0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */
 
 /*
 ** Combinations of two or more EP_* flags
@@ -2212,7 +2346,7 @@ struct Expr {
 #define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */
 
 /*
-** These macros can be used to test, set, or clear bits in the 
+** These macros can be used to test, set, or clear bits in the
 ** Expr.flags field.
 */
 #define ExprHasProperty(E,P)     (((E)->flags&(P))!=0)
@@ -2231,8 +2365,8 @@ struct Expr {
 #endif
 
 /*
-** Macros to determine the number of bytes required by a normal Expr 
-** struct, an Expr struct with the EP_Reduced flag set in Expr.flags 
+** Macros to determine the number of bytes required by a normal Expr
+** struct, an Expr struct with the EP_Reduced flag set in Expr.flags
 ** and an Expr struct with the EP_TokenOnly flag set.
 */
 #define EXPR_FULLSIZE           sizeof(Expr)           /* Full size */
@@ -2240,7 +2374,7 @@ struct Expr {
 #define EXPR_TOKENONLYSIZE      offsetof(Expr,pLeft)   /* Fewer features */
 
 /*
-** Flags passed to the sqlite3ExprDup() function. See the header comment 
+** Flags passed to the sqlite3ExprDup() function. See the header comment
 ** above sqlite3ExprDup() for details.
 */
 #define EXPRDUP_REDUCE         0x0001  /* Used reduced-size Expr nodes */
@@ -2322,7 +2456,11 @@ struct IdList {
 ** tables in a join to 32 instead of 64.  But it also reduces the size
 ** of the library by 738 bytes on ix86.
 */
-typedef u64 Bitmask;
+#ifdef SQLITE_BITMASK_TYPE
+  typedef SQLITE_BITMASK_TYPE Bitmask;
+#else
+  typedef u64 Bitmask;
+#endif
 
 /*
 ** The number of bits in a Bitmask.  "BMS" means "BitMask Size".
@@ -2334,6 +2472,7 @@ typedef u64 Bitmask;
 */
 #define MASKBIT(n)   (((Bitmask)1)<<(n))
 #define MASKBIT32(n) (((unsigned int)1)<<(n))
+#define ALLBITS      ((Bitmask)-1)
 
 /*
 ** The following structure describes the FROM clause of a SELECT statement.
@@ -2368,7 +2507,7 @@ struct SrcList {
     int regReturn;    /* Register holding return address of addrFillSub */
     int regResult;    /* Registers holding results of a co-routine */
     struct {
-      u8 jointype;      /* Type of join between this able and the previous */
+      u8 jointype;      /* Type of join between this table and the previous */
       unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
       unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
       unsigned isTabFunc :1;     /* True if table-valued-function syntax */
@@ -2406,22 +2545,28 @@ struct SrcList {
 /*
 ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin()
 ** and the WhereInfo.wctrlFlags member.
+**
+** Value constraints (enforced via assert()):
+**     WHERE_USE_LIMIT  == SF_FixedLimit
 */
 #define WHERE_ORDERBY_NORMAL   0x0000 /* No-op */
 #define WHERE_ORDERBY_MIN      0x0001 /* ORDER BY processing for min() func */
 #define WHERE_ORDERBY_MAX      0x0002 /* ORDER BY processing for max() func */
 #define WHERE_ONEPASS_DESIRED  0x0004 /* Want to do one-pass UPDATE/DELETE */
-#define WHERE_DUPLICATES_OK    0x0008 /* Ok to return a row more than once */
-#define WHERE_OMIT_OPEN_CLOSE  0x0010 /* Table cursors are already open */
-#define WHERE_FORCE_TABLE      0x0020 /* Do not use an index-only search */
-#define WHERE_ONETABLE_ONLY    0x0040 /* Only code the 1st table in pTabList */
-#define WHERE_NO_AUTOINDEX     0x0080 /* Disallow automatic indexes */
-#define WHERE_GROUPBY          0x0100 /* pOrderBy is really a GROUP BY */
-#define WHERE_DISTINCTBY       0x0200 /* pOrderby is really a DISTINCT clause */
-#define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
-#define WHERE_SORTBYGROUP      0x0800 /* Support sqlite3WhereIsSorted() */
-#define WHERE_REOPEN_IDX       0x1000 /* Try to use OP_ReopenIdx */
-#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */
+#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */
+#define WHERE_DUPLICATES_OK    0x0010 /* Ok to return a row more than once */
+#define WHERE_OR_SUBCLAUSE     0x0020 /* Processing a sub-WHERE as part of
+                                      ** the OR optimization  */
+#define WHERE_GROUPBY          0x0040 /* pOrderBy is really a GROUP BY */
+#define WHERE_DISTINCTBY       0x0080 /* pOrderby is really a DISTINCT clause */
+#define WHERE_WANT_DISTINCT    0x0100 /* All output needs to be distinct */
+#define WHERE_SORTBYGROUP      0x0200 /* Support sqlite3WhereIsSorted() */
+#define WHERE_SEEK_TABLE       0x0400 /* Do not defer seeks on main table */
+#define WHERE_ORDERBY_LIMIT    0x0800 /* ORDERBY+LIMIT on the inner loop */
+                        /*     0x1000    not currently used */
+                        /*     0x2000    not currently used */
+#define WHERE_USE_LIMIT        0x4000 /* Use the LIMIT in cost estimates */
+                        /*     0x8000    not currently used */
 
 /* Allowed return values from sqlite3WhereIsDistinct()
 */
@@ -2439,12 +2584,12 @@ struct SrcList {
 ** pEList corresponds to the result set of a SELECT and is NULL for
 ** other statements.
 **
-** NameContexts can be nested.  When resolving names, the inner-most 
+** NameContexts can be nested.  When resolving names, the inner-most
 ** context is searched first.  If no match is found, the next outer
 ** context is checked.  If there is still no match, the next context
 ** is checked.  This process continues until either a match is found
 ** or all contexts are check.  When a match is found, the nRef member of
-** the context containing the match is incremented. 
+** the context containing the match is incremented.
 **
 ** Each subquery gets a new NameContext.  The pNext field points to the
 ** NameContext in the parent query.  Thus the process of scanning the
@@ -2465,16 +2610,18 @@ struct NameContext {
 /*
 ** Allowed values for the NameContext, ncFlags field.
 **
-** Note:  NC_MinMaxAgg must have the same value as SF_MinMaxAgg and
-** SQLITE_FUNC_MINMAX.
-** 
+** Value constraints (all checked via assert()):
+**    NC_HasAgg    == SF_HasAgg
+**    NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX
+**
 */
 #define NC_AllowAgg  0x0001  /* Aggregate functions are allowed here */
-#define NC_HasAgg    0x0002  /* One or more aggregate functions seen */
+#define NC_PartIdx   0x0002  /* True if resolving a partial index WHERE */
 #define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
 #define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
-#define NC_PartIdx   0x0010  /* True if resolving a partial index WHERE */
+#define NC_HasAgg    0x0010  /* One or more aggregate functions seen */
 #define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
+#define NC_VarSelect 0x0040  /* A correlated subquery has been seen */
 #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
 
 /*
@@ -2500,13 +2647,13 @@ struct NameContext {
 struct Select {
   ExprList *pEList;      /* The fields of the result */
   u8 op;                 /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */
-  u16 selFlags;          /* Various SF_* values */
+  LogEst nSelectRow;     /* Estimated number of result rows */
+  u32 selFlags;          /* Various SF_* values */
   int iLimit, iOffset;   /* Memory registers holding LIMIT & OFFSET counters */
 #if SELECTTRACE_ENABLED
   char zSelName[12];     /* Symbolic name of this SELECT use for debugging */
 #endif
   int addrOpenEphm[2];   /* OP_OpenEphem opcodes related to this select */
-  u64 nSelectRow;        /* Estimated number of result rows */
   SrcList *pSrc;         /* The FROM clause */
   Expr *pWhere;          /* The WHERE clause */
   ExprList *pGroupBy;    /* The GROUP BY clause */
@@ -2522,23 +2669,30 @@ struct Select {
 /*
 ** Allowed values for Select.selFlags.  The "SF" prefix stands for
 ** "Select Flag".
+**
+** Value constraints (all checked via assert())
+**     SF_HasAgg     == NC_HasAgg
+**     SF_MinMaxAgg  == NC_MinMaxAgg     == SQLITE_FUNC_MINMAX
+**     SF_FixedLimit == WHERE_USE_LIMIT
 */
-#define SF_Distinct        0x0001  /* Output should be DISTINCT */
-#define SF_All             0x0002  /* Includes the ALL keyword */
-#define SF_Resolved        0x0004  /* Identifiers have been resolved */
-#define SF_Aggregate       0x0008  /* Contains aggregate functions */
-#define SF_UsesEphemeral   0x0010  /* Uses the OpenEphemeral opcode */
-#define SF_Expanded        0x0020  /* sqlite3SelectExpand() called on this */
-#define SF_HasTypeInfo     0x0040  /* FROM subqueries have Table metadata */
-#define SF_Compound        0x0080  /* Part of a compound query */
-#define SF_Values          0x0100  /* Synthesized from VALUES clause */
-#define SF_MultiValue      0x0200  /* Single VALUES term with multiple rows */
-#define SF_NestedFrom      0x0400  /* Part of a parenthesized FROM clause */
-#define SF_MaybeConvert    0x0800  /* Need convertCompoundSelectToSubquery() */
-#define SF_MinMaxAgg       0x1000  /* Aggregate containing min() or max() */
-#define SF_Recursive       0x2000  /* The recursive part of a recursive CTE */
-#define SF_Converted       0x4000  /* By convertCompoundSelectToSubquery() */
-#define SF_IncludeHidden   0x8000  /* Include hidden columns in output */
+#define SF_Distinct       0x00001  /* Output should be DISTINCT */
+#define SF_All            0x00002  /* Includes the ALL keyword */
+#define SF_Resolved       0x00004  /* Identifiers have been resolved */
+#define SF_Aggregate      0x00008  /* Contains agg functions or a GROUP BY */
+#define SF_HasAgg         0x00010  /* Contains aggregate functions */
+#define SF_UsesEphemeral  0x00020  /* Uses the OpenEphemeral opcode */
+#define SF_Expanded       0x00040  /* sqlite3SelectExpand() called on this */
+#define SF_HasTypeInfo    0x00080  /* FROM subqueries have Table metadata */
+#define SF_Compound       0x00100  /* Part of a compound query */
+#define SF_Values         0x00200  /* Synthesized from VALUES clause */
+#define SF_MultiValue     0x00400  /* Single VALUES term with multiple rows */
+#define SF_NestedFrom     0x00800  /* Part of a parenthesized FROM clause */
+#define SF_MinMaxAgg      0x01000  /* Aggregate containing min() or max() */
+#define SF_Recursive      0x02000  /* The recursive part of a recursive CTE */
+#define SF_FixedLimit     0x04000  /* nSelectRow set by a constant LIMIT */
+#define SF_MaybeConvert   0x08000  /* Need convertCompoundSelectToSubquery() */
+#define SF_Converted      0x10000  /* By convertCompoundSelectToSubquery() */
+#define SF_IncludeHidden  0x20000  /* Include hidden columns in output */
 
 
 /*
@@ -2546,7 +2700,7 @@ struct Select {
 ** by one of the following macros.  The "SRT" prefix means "SELECT Result
 ** Type".
 **
-**     SRT_Union       Store results as a key in a temporary index 
+**     SRT_Union       Store results as a key in a temporary index
 **                     identified by pDest->iSDParm.
 **
 **     SRT_Except      Remove results from the temporary index pDest->iSDParm.
@@ -2570,7 +2724,7 @@ struct Select {
 **                     of the query.  This destination implies "LIMIT 1".
 **
 **     SRT_Set         The result must be a single column.  Store each
-**                     row of result as the key in table pDest->iSDParm. 
+**                     row of result as the key in table pDest->iSDParm.
 **                     Apply the affinity pDest->affSdst before storing
 **                     results.  Used to implement "IN (SELECT ...)".
 **
@@ -2630,7 +2784,7 @@ struct Select {
 */
 struct SelectDest {
   u8 eDest;            /* How to dispose of the results.  On of SRT_* above. */
-  char affSdst;        /* Affinity used when eDest==SRT_Set */
+  char *zAffSdst;      /* Affinity used when eDest==SRT_Set */
   int iSDParm;         /* A parameter used by the eDest disposal method */
   int iSdst;           /* Base register where results are written */
   int nSdst;           /* Number of registers allocated */
@@ -2638,7 +2792,7 @@ struct SelectDest {
 };
 
 /*
-** During code generation of statements that do inserts into AUTOINCREMENT 
+** During code generation of statements that do inserts into AUTOINCREMENT
 ** tables, the following information is attached to the Table.u.autoInc.p
 ** pointer of each autoincrement table to record some side information that
 ** the code generator needs.  We have to keep per-table autoincrement
@@ -2661,7 +2815,7 @@ struct AutoincInfo {
 #endif
 
 /*
-** At least one instance of the following structure is created for each 
+** At least one instance of the following structure is created for each
 ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE
 ** statement. All such objects are stored in the linked list headed at
 ** Parse.pTriggerPrg and deleted once statement compilation has been
@@ -2674,7 +2828,7 @@ struct AutoincInfo {
 ** values for both pTrigger and orconf.
 **
 ** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns
-** accessed (or set to 0 for triggers fired as a result of INSERT 
+** accessed (or set to 0 for triggers fired as a result of INSERT
 ** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to
 ** a mask of new.* columns used by the program.
 */
@@ -2715,7 +2869,7 @@ struct TriggerPrg {
 ** is constant but the second part is reset at the beginning and end of
 ** each recursion.
 **
-** The nTableLock and aTableLock variables are only used if the shared-cache 
+** The nTableLock and aTableLock variables are only used if the shared-cache
 ** feature is enabled (if sqlite3Tsd()->useSharedData is true). They are
 ** used to store the set of table-locks required by the statement being
 ** compiled. Function sqlite3TableLock() is used to add entries to the
@@ -2735,36 +2889,24 @@ struct Parse {
   u8 hasCompound;      /* Need to invoke convertCompoundSelectToSubquery() */
   u8 okConstFactor;    /* OK to factor out constants */
   u8 disableLookaside; /* Number of times lookaside has been disabled */
-  int aTempReg[8];     /* Holding area for temporary registers */
+  u8 nColCache;        /* Number of entries in aColCache[] */
   int nRangeReg;       /* Size of the temporary register block */
   int iRangeReg;       /* First register in temporary register block */
   int nErr;            /* Number of errors seen */
   int nTab;            /* Number of previously allocated VDBE cursors */
   int nMem;            /* Number of memory cells used so far */
-  int nSet;            /* Number of sets used so far */
-  int nOnce;           /* Number of OP_Once instructions so far */
   int nOpAlloc;        /* Number of slots allocated for Vdbe.aOp[] */
   int szOpAlloc;       /* Bytes of memory space allocated for Vdbe.aOp[] */
-  int iFixedOp;        /* Never back out opcodes iFixedOp-1 or earlier */
   int ckBase;          /* Base register of data during check constraints */
   int iSelfTab;        /* Table of an index whose exprs are being coded */
   int iCacheLevel;     /* ColCache valid when aColCache[].iLevel<=iCacheLevel */
   int iCacheCnt;       /* Counter used to generate aColCache[].lru values */
   int nLabel;          /* Number of labels used */
   int *aLabel;         /* Space to hold the labels */
-  struct yColCache {
-    int iTable;           /* Table cursor number */
-    i16 iColumn;          /* Table column number */
-    u8 tempReg;           /* iReg is a temp register that needs to be freed */
-    int iLevel;           /* Nesting level */
-    int iReg;             /* Reg with value of this column. 0 means none. */
-    int lru;              /* Least recently used entry has the smallest value */
-  } aColCache[SQLITE_N_COLCACHE];  /* One for each column cache entry */
   ExprList *pConstExpr;/* Constant expressions */
   Token constraintName;/* Name of the constraint currently being parsed */
   yDbMask writeMask;   /* Start a write transaction on these databases */
   yDbMask cookieMask;  /* Bitmask of schema verified databases */
-  int cookieValue[SQLITE_MAX_ATTACHED+2];  /* Values of cookies to verify */
   int regRowid;        /* Register holding rowid of CREATE TABLE entry */
   int regRoot;         /* Register holding root page number for new objects */
   int nMaxArg;         /* Max args passed to user function by sub-program */
@@ -2777,8 +2919,6 @@ struct Parse {
   TableLock *aTableLock; /* Required table locks for shared-cache mode */
 #endif
   AutoincInfo *pAinc;  /* Information about AUTOINCREMENT counters */
-
-  /* Information used while coding trigger programs. */
   Parse *pToplevel;    /* Parse structure for main program (or NULL) */
   Table *pTriggerTab;  /* Table triggers are being coded for */
   int addrCrTab;       /* Address of OP_CreateTable opcode on CREATE TABLE */
@@ -2789,13 +2929,32 @@ struct Parse {
   u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
   u8 disableTriggers;  /* True to disable triggers */
 
+  /**************************************************************************
+  ** Fields above must be initialized to zero.  The fields that follow,
+  ** down to the beginning of the recursive section, do not need to be
+  ** initialized as they will be set before being used.  The boundary is
+  ** determined by offsetof(Parse,aColCache).
+  **************************************************************************/
+
+  struct yColCache {
+    int iTable;           /* Table cursor number */
+    i16 iColumn;          /* Table column number */
+    u8 tempReg;           /* iReg is a temp register that needs to be freed */
+    int iLevel;           /* Nesting level */
+    int iReg;             /* Reg with value of this column. 0 means none. */
+    int lru;              /* Least recently used entry has the smallest value */
+  } aColCache[SQLITE_N_COLCACHE];  /* One for each column cache entry */
+  int aTempReg[8];        /* Holding area for temporary registers */
+  Token sNameToken;       /* Token with unqualified schema object name */
+
   /************************************************************************
   ** Above is constant between recursions.  Below is reset before and after
   ** each recursion.  The boundary between these two regions is determined
-  ** using offsetof(Parse,nVar) so the nVar field must be the first field
-  ** in the recursive region.
+  ** using offsetof(Parse,sLastToken) so the sLastToken field must be the
+  ** first field in the recursive region.
   ************************************************************************/
 
+  Token sLastToken;       /* The last token parsed */
   ynVar nVar;               /* Number of '?' variables seen in the SQL so far */
   int nzVar;                /* Number of available slots in azVar[] */
   u8 iPkSortOrder;          /* ASC or DESC for INTEGER PRIMARY KEY */
@@ -2804,7 +2963,6 @@ struct Parse {
   u8 declareVtab;           /* True if inside sqlite3_declare_vtab() */
   int nVtabLock;            /* Number of virtual tables to lock */
 #endif
-  int nAlias;               /* Number of aliased result set columns */
   int nHeight;              /* Expression tree height of current sub-select */
 #ifndef SQLITE_OMIT_EXPLAIN
   int iSelectId;            /* ID of current select for EXPLAIN output */
@@ -2816,8 +2974,6 @@ struct Parse {
   Table *pNewTable;         /* A table being constructed by CREATE TABLE */
   Trigger *pNewTrigger;     /* Trigger under construct by a CREATE TRIGGER */
   const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */
-  Token sNameToken;         /* Token with unqualified schema object name */
-  Token sLastToken;         /* The last token parsed */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   Token sArg;               /* Complete text of a module argument */
   Table **apVtabLock;       /* Pointer to virtual tables needing locking */
@@ -2828,6 +2984,14 @@ struct Parse {
   With *pWithToFree;        /* Free this WITH object at the end of the parse */
 };
 
+/*
+** Sizes and pointers of various parts of the Parse object.
+*/
+#define PARSE_HDR_SZ offsetof(Parse,aColCache) /* Recursive part w/o aColCache*/
+#define PARSE_RECURSE_SZ offsetof(Parse,sLastToken)    /* Recursive part */
+#define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */
+#define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ)  /* Pointer to tail */
+
 /*
 ** Return true if currently inside an sqlite3_declare_vtab() call.
 */
@@ -2848,6 +3012,15 @@ struct AuthContext {
 
 /*
 ** Bitfield flags for P5 value in various opcodes.
+**
+** Value constraints (enforced via assert()):
+**    OPFLAG_LENGTHARG    == SQLITE_FUNC_LENGTH
+**    OPFLAG_TYPEOFARG    == SQLITE_FUNC_TYPEOF
+**    OPFLAG_BULKCSR      == BTREE_BULKLOAD
+**    OPFLAG_SEEKEQ       == BTREE_SEEK_EQ
+**    OPFLAG_FORDELETE    == BTREE_FORDELETE
+**    OPFLAG_SAVEPOSITION == BTREE_SAVEPOSITION
+**    OPFLAG_AUXDELETE    == BTREE_AUXDELETE
 */
 #define OPFLAG_NCHANGE       0x01    /* OP_Insert: Set to update db->nChange */
                                      /* Also used in P2 (not P5) of OP_Delete */
@@ -2856,6 +3029,9 @@ struct AuthContext {
 #define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
 #define OPFLAG_APPEND        0x08    /* This is likely to be an append */
 #define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+#define OPFLAG_ISNOOP        0x40    /* OP_Delete does pre-update-hook only */
+#endif
 #define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
 #define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
 #define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
@@ -2868,10 +3044,10 @@ struct AuthContext {
 
 /*
  * Each trigger present in the database schema is stored as an instance of
- * struct Trigger. 
+ * struct Trigger.
  *
  * Pointers to instances of struct Trigger are stored in two ways.
- * 1. In the "trigHash" hash table (part of the sqlite3* that represents the 
+ * 1. In the "trigHash" hash table (part of the sqlite3* that represents the
  *    database). This allows Trigger structures to be retrieved by name.
  * 2. All triggers associated with a single table form a linked list, using the
  *    pNext member of struct Trigger. A pointer to the first element of the
@@ -2897,7 +3073,7 @@ struct Trigger {
 
 /*
 ** A trigger is either a BEFORE or an AFTER trigger.  The following constants
-** determine which. 
+** determine which.
 **
 ** If there are multiple triggers, you might of some BEFORE and some AFTER.
 ** In that cases, the constants below can be ORed together.
@@ -2907,15 +3083,15 @@ struct Trigger {
 
 /*
  * An instance of struct TriggerStep is used to store a single SQL statement
- * that is a part of a trigger-program. 
+ * that is a part of a trigger-program.
  *
  * Instances of struct TriggerStep are stored in a singly linked list (linked
- * using the "pNext" member) referenced by the "step_list" member of the 
+ * using the "pNext" member) referenced by the "step_list" member of the
  * associated struct Trigger instance. The first element of the linked list is
  * the first step of the trigger-program.
- * 
+ *
  * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or
- * "SELECT" statement. The meanings of the other members is determined by the 
+ * "SELECT" statement. The meanings of the other members is determined by the
  * value of "op" as follows:
  *
  * (op == TK_INSERT)
@@ -2925,7 +3101,7 @@ struct Trigger {
  * zTarget   -> Dequoted name of the table to insert into.
  * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then
  *              this stores values to be inserted. Otherwise NULL.
- * pIdList   -> If this is an INSERT INTO ... () VALUES ... 
+ * pIdList   -> If this is an INSERT INTO ... () VALUES ...
  *              statement, then this stores the column-names to be
  *              inserted into.
  *
@@ -2933,7 +3109,7 @@ struct Trigger {
  * zTarget   -> Dequoted name of the table to delete from.
  * pWhere    -> The WHERE clause of the DELETE statement if one is specified.
  *              Otherwise NULL.
- * 
+ *
  * (op == TK_UPDATE)
  * zTarget   -> Dequoted name of the table to update.
  * pWhere    -> The WHERE clause of the UPDATE statement if one is specified.
@@ -2941,7 +3117,7 @@ struct Trigger {
  * pExprList -> A list of the columns to update and the expressions to update
  *              them to. See sqlite3Update() documentation of "pChanges"
  *              argument.
- * 
+ *
  */
 struct TriggerStep {
   u8 op;               /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */
@@ -2959,7 +3135,7 @@ struct TriggerStep {
 /*
 ** The following structure contains information used by the sqliteFix...
 ** routines as they walk the parse tree to make database references
-** explicit.  
+** explicit.
 */
 typedef struct DbFixer DbFixer;
 struct DbFixer {
@@ -3020,6 +3196,7 @@ struct Sqlite3Config {
   int neverCorrupt;                 /* Database is always well-formed */
   int szLookaside;                  /* Default lookaside buffer size */
   int nLookaside;                   /* Default lookaside buffer count */
+  int nStmtSpill;                   /* Stmt-journal spill-to-disk threshold */
   sqlite3_mem_methods m;            /* Low-level memory allocation interface */
   sqlite3_mutex_methods mutex;      /* Low-level mutex interface */
   sqlite3_pcache_methods2 pcache2;  /* Low-level page-cache interface */
@@ -3063,6 +3240,7 @@ struct Sqlite3Config {
   int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
 #endif
   int bLocaltimeFault;              /* True to fail localtime() calls */
+  int iOnceResetThreshold;          /* When to reset OP_Once counters */
 };
 
 /*
@@ -3101,6 +3279,7 @@ struct Walker {
     struct SrcCount *pSrcCount;                /* Counting column references */
     struct CCurHint *pCCurHint;                /* Used by codeCursorHint() */
     int *aiCol;                                /* array of column indexes */
+    struct IdxCover *pIdxCover;                /* Check for index coverage */
   } u;
 };
 
@@ -3169,6 +3348,15 @@ int sqlite3CantopenError(int);
 #define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__)
 #define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__)
 #define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__)
+#ifdef SQLITE_DEBUG
+  int sqlite3NomemError(int);
+  int sqlite3IoerrnomemError(int);
+# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__)
+# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__)
+#else
+# define SQLITE_NOMEM_BKPT SQLITE_NOMEM
+# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM
+#endif
 
 /*
 ** FTS3 and FTS4 both require virtual table support
@@ -3209,6 +3397,7 @@ int sqlite3CantopenError(int);
 # define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
 # define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
 # define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
+# define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
 #else
 # define sqlite3Toupper(x)   toupper((unsigned char)(x))
 # define sqlite3Isspace(x)   isspace((unsigned char)(x))
@@ -3217,6 +3406,7 @@ int sqlite3CantopenError(int);
 # define sqlite3Isdigit(x)   isdigit((unsigned char)(x))
 # define sqlite3Isxdigit(x)  isxdigit((unsigned char)(x))
 # define sqlite3Tolower(x)   tolower((unsigned char)(x))
+# define sqlite3Isquote(x)   ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`')
 #endif
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
 int sqlite3IsIdChar(u8);
@@ -3225,8 +3415,9 @@ int sqlite3IsIdChar(u8);
 /*
 ** Internal function prototypes
 */
-#define sqlite3StrICmp sqlite3_stricmp
+int sqlite3StrICmp(const char*,const char*);
 int sqlite3Strlen30(const char*);
+char *sqlite3ColumnType(Column*,char*);
 #define sqlite3StrNICmp sqlite3_strnicmp
 
 int sqlite3MallocInit(void);
@@ -3265,18 +3456,22 @@ int sqlite3HeapNearlyFull(void);
 #ifdef SQLITE_USE_ALLOCA
 # define sqlite3StackAllocRaw(D,N)   alloca(N)
 # define sqlite3StackAllocZero(D,N)  memset(alloca(N), 0, N)
-# define sqlite3StackFree(D,P)       
+# define sqlite3StackFree(D,P)
 #else
 # define sqlite3StackAllocRaw(D,N)   sqlite3DbMallocRaw(D,N)
 # define sqlite3StackAllocZero(D,N)  sqlite3DbMallocZero(D,N)
 # define sqlite3StackFree(D,P)       sqlite3DbFree(D,P)
 #endif
 
-#ifdef SQLITE_ENABLE_MEMSYS3
-const sqlite3_mem_methods *sqlite3MemGetMemsys3(void);
-#endif
+/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together.  If they
+** are, disable MEMSYS3
+*/
 #ifdef SQLITE_ENABLE_MEMSYS5
 const sqlite3_mem_methods *sqlite3MemGetMemsys5(void);
+#undef SQLITE_ENABLE_MEMSYS3
+#endif
+#ifdef SQLITE_ENABLE_MEMSYS3
+const sqlite3_mem_methods *sqlite3MemGetMemsys3(void);
 #endif
 
 
@@ -3331,6 +3526,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
 
 #if defined(SQLITE_DEBUG)
   void sqlite3TreeViewExpr(TreeView*, const Expr*, u8);
+  void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*);
   void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*);
   void sqlite3TreeViewSelect(TreeView*, const Select*, u8);
   void sqlite3TreeViewWith(TreeView*, const With*, u8);
@@ -3339,7 +3535,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
 
 void sqlite3SetString(char **, sqlite3*, const char*);
 void sqlite3ErrorMsg(Parse*, const char*, ...);
-int sqlite3Dequote(char*);
+void sqlite3Dequote(char*);
 void sqlite3TokenInit(Token*,char*);
 int sqlite3KeywordCode(const unsigned char*, int);
 int sqlite3RunParser(Parse*, const char*, char **);
@@ -3349,15 +3545,20 @@ void sqlite3ReleaseTempReg(Parse*,int);
 int sqlite3GetTempRange(Parse*,int);
 void sqlite3ReleaseTempRange(Parse*,int,int);
 void sqlite3ClearTempRegCache(Parse*);
+#ifdef SQLITE_DEBUG
+int sqlite3NoTempsInRange(Parse*,int,int);
+#endif
 Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int);
 Expr *sqlite3Expr(sqlite3*,int,const char*);
 void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*);
 Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*);
+void sqlite3PExprAddSelect(Parse*, Expr*, Select*);
 Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*);
 Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*);
-void sqlite3ExprAssignVarNumber(Parse*, Expr*);
+void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32);
 void sqlite3ExprDelete(sqlite3*, Expr*);
 ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
+ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*);
 void sqlite3ExprListSetSortOrder(ExprList*,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*);
@@ -3372,6 +3573,7 @@ void sqlite3CollapseDatabaseArray(sqlite3*);
 void sqlite3CommitInternalChanges(sqlite3*);
 void sqlite3DeleteColumnNames(sqlite3*,Table*);
 int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**);
+void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*);
 Table *sqlite3ResultSetOfSelect(Parse*,Select*);
 void sqlite3OpenMasterTable(Parse *, int);
 Index *sqlite3PrimaryKeyIndex(Table*);
@@ -3382,18 +3584,16 @@ void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
 #else
 # define sqlite3ColumnPropertiesFromName(T,C) /* no-op */
 #endif
-void sqlite3AddColumn(Parse*,Token*);
+void sqlite3AddColumn(Parse*,Token*,Token*);
 void sqlite3AddNotNull(Parse*, int);
 void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int);
 void sqlite3AddCheckConstraint(Parse*, Expr*);
-void sqlite3AddColumnType(Parse*,Token*);
 void sqlite3AddDefaultValue(Parse*,ExprSpan*);
 void sqlite3AddCollateType(Parse*, Token*);
 void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*);
 int sqlite3ParseUri(const char*,const char*,unsigned int*,
                     sqlite3_vfs**,char**,char **);
 Btree *sqlite3DbNameToBtree(sqlite3*,const char*);
-int sqlite3CodeOnce(Parse *);
 
 #ifdef SQLITE_OMIT_BUILTIN_TEST
 # define sqlite3FaultSim(X) SQLITE_OK
@@ -3455,12 +3655,12 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*);
 void sqlite3IdListDelete(sqlite3*, IdList*);
 void sqlite3SrcListDelete(sqlite3*, SrcList*);
 Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**);
-Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
-                          Expr*, int, int);
+void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
+                          Expr*, int, int, u8);
 void sqlite3DropIndex(Parse*, SrcList*, int);
 int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
-                         Expr*,ExprList*,u16,Expr*,Expr*);
+                         Expr*,ExprList*,u32,Expr*,Expr*);
 void sqlite3SelectDelete(sqlite3*, Select*);
 Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, int);
@@ -3472,9 +3672,10 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
 void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
 WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int);
 void sqlite3WhereEnd(WhereInfo*);
-u64 sqlite3WhereOutputRowCount(WhereInfo*);
+LogEst sqlite3WhereOutputRowCount(WhereInfo*);
 int sqlite3WhereIsDistinct(WhereInfo*);
 int sqlite3WhereIsOrdered(WhereInfo*);
+int sqlite3WhereOrderedInnerLoop(WhereInfo*);
 int sqlite3WhereIsSorted(WhereInfo*);
 int sqlite3WhereContinueLabel(WhereInfo*);
 int sqlite3WhereBreakLabel(WhereInfo*);
@@ -3508,19 +3709,22 @@ void sqlite3ExprIfTrue(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalse(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int);
 Table *sqlite3FindTable(sqlite3*,const char*, const char*);
-Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*);
-Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *);
+#define LOCATE_VIEW    0x01
+#define LOCATE_NOERR   0x02
+Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
+Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
 Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
 void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
 void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
-void sqlite3Vacuum(Parse*);
-int sqlite3RunVacuum(char**, sqlite3*);
+void sqlite3Vacuum(Parse*,Token*);
+int sqlite3RunVacuum(char**, sqlite3*, int);
 char *sqlite3NameFromToken(sqlite3*, Token*);
 int sqlite3ExprCompare(Expr*, Expr*, int);
 int sqlite3ExprListCompare(ExprList*, ExprList*, int);
 int sqlite3ExprImpliesExpr(Expr*, Expr*, int);
 void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
 void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*);
+int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx);
 int sqlite3FunctionUsesThisSrc(Expr*, SrcList*);
 Vdbe *sqlite3GetVdbe(Parse*);
 #ifndef SQLITE_OMIT_BUILTIN_TEST
@@ -3572,11 +3776,11 @@ void sqlite3SelectSetName(Select*,const char*);
 #else
 # define sqlite3SelectSetName(A,B)
 #endif
-void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*);
-FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8);
-void sqlite3RegisterBuiltinFunctions(sqlite3*);
+void sqlite3InsertBuiltinFuncs(FuncDef*,int);
+FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
+void sqlite3RegisterBuiltinFunctions(void);
 void sqlite3RegisterDateTimeFunctions(void);
-void sqlite3RegisterGlobalFunctions(void);
+void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*);
 int sqlite3SafetyCheckOk(sqlite3*);
 int sqlite3SafetyCheckSickOrOk(sqlite3*);
 void sqlite3ChangeCookie(Parse*, int);
@@ -3655,7 +3859,11 @@ LogEst sqlite3LogEstAdd(LogEst,LogEst);
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 LogEst sqlite3LogEstFromDouble(double);
 #endif
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
+    defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \
+    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
 u64 sqlite3LogEstToInt(LogEst);
+#endif
 
 /*
 ** Routines to read and write variable-length integers.  These used to
@@ -3685,11 +3893,13 @@ const char *sqlite3IndexAffinityStr(sqlite3*, Index*);
 void sqlite3TableAffinity(Vdbe*, Table*, int);
 char sqlite3CompareAffinity(Expr *pExpr, char aff2);
 int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity);
+char sqlite3TableColumnAffinity(Table*,int);
 char sqlite3ExprAffinity(Expr *pExpr);
 int sqlite3Atoi64(const char*, i64*, int, u8);
 int sqlite3DecOrHexToI64(const char*, i64*);
 void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...);
 void sqlite3Error(sqlite3*,int);
+void sqlite3SystemError(sqlite3*,int);
 void *sqlite3HexToBlob(sqlite3*, const char *z, int n);
 u8 sqlite3HexToInt(int h);
 int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
@@ -3722,7 +3932,7 @@ u8 sqlite3GetBoolean(const char *z,u8);
 
 const void *sqlite3ValueText(sqlite3_value*, u8);
 int sqlite3ValueBytes(sqlite3_value*, u8);
-void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, 
+void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
                         void(*)(void*));
 void sqlite3ValueSetNull(sqlite3_value*);
 void sqlite3ValueFree(sqlite3_value*);
@@ -3737,7 +3947,7 @@ extern const unsigned char sqlite3UpperToLower[];
 extern const unsigned char sqlite3CtypeMap[];
 extern const Token sqlite3IntTokens[];
 extern SQLITE_WSD struct Sqlite3Config sqlite3Config;
-extern SQLITE_WSD FuncDefHash sqlite3GlobalFunctions;
+extern FuncDefHash sqlite3BuiltinFunctions;
 #ifndef SQLITE_OMIT_WSD
 extern int sqlite3PendingByte;
 #endif
@@ -3749,7 +3959,7 @@ void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
 int sqlite3GetToken(const unsigned char *, int *);
 void sqlite3NestedParse(Parse*, const char*, ...);
 void sqlite3ExpirePreparedStatements(sqlite3*);
-int sqlite3CodeSubselect(Parse *, Expr *, int, int);
+int sqlite3CodeSubselect(Parse*, Expr *, int, int);
 void sqlite3SelectPrep(Parse*, Select*, NameContext*);
 void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
 int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
@@ -3782,7 +3992,7 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*);
 #ifdef SQLITE_DEBUG
 int sqlite3KeyInfoIsWriteable(KeyInfo*);
 #endif
-int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, 
+int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *,
   void (*)(sqlite3_context*,int,sqlite3_value **),
   void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*),
   FuncDestructor *pDestructor
@@ -3804,12 +4014,20 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int);
 void sqlite3BackupRestart(sqlite3_backup *);
 void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *);
 
+#ifndef SQLITE_OMIT_SUBQUERY
+int sqlite3ExprCheckIN(Parse*, Expr*);
+#else
+# define sqlite3ExprCheckIN(x,y) SQLITE_OK
+#endif
+
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
 void sqlite3AnalyzeFunctions(void);
-int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*);
+int sqlite3Stat4ProbeSetValue(
+    Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*);
 int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**);
 void sqlite3Stat4ProbeFree(UnpackedRecord*);
 int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**);
+char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
 #endif
 
 /*
@@ -3845,7 +4063,7 @@ void sqlite3AutoLoadExtensions(sqlite3*);
 #  define sqlite3VtabRollback(X)
 #  define sqlite3VtabCommit(X)
 #  define sqlite3VtabInSync(db) 0
-#  define sqlite3VtabLock(X) 
+#  define sqlite3VtabLock(X)
 #  define sqlite3VtabUnlock(X)
 #  define sqlite3VtabUnlockList(X)
 #  define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK
@@ -3903,7 +4121,7 @@ const char *sqlite3JournalModename(int);
 ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign
 ** key functionality is available. If OMIT_TRIGGER is defined but
 ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In
-** this case foreign keys are parsed, but no other functionality is 
+** this case foreign keys are parsed, but no other functionality is
 ** provided (enforcement of FK constraints requires the triggers sub-system).
 */
 #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER)
@@ -3962,21 +4180,16 @@ const char *sqlite3JournalModename(int);
 #define IN_INDEX_NOOP_OK     0x0001  /* OK to return IN_INDEX_NOOP */
 #define IN_INDEX_MEMBERSHIP  0x0002  /* IN operator used for membership test */
 #define IN_INDEX_LOOP        0x0004  /* IN operator used as a loop */
-int sqlite3FindInIndex(Parse *, Expr *, u32, int*);
+int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*);
 
+int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
+int sqlite3JournalSize(sqlite3_vfs *);
 #ifdef SQLITE_ENABLE_ATOMIC_WRITE
-  int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
-  int sqlite3JournalSize(sqlite3_vfs *);
   int sqlite3JournalCreate(sqlite3_file *);
-  int sqlite3JournalExists(sqlite3_file *p);
-#else
-  #define sqlite3JournalSize(pVfs) ((pVfs)->szOsFile)
-  #define sqlite3JournalExists(p) 1
 #endif
 
+int sqlite3JournalIsInMemory(sqlite3_file *p);
 void sqlite3MemJournalOpen(sqlite3_file *);
-int sqlite3MemJournalSize(void);
-int sqlite3IsMemJournal(sqlite3_file *);
 
 void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p);
 #if SQLITE_MAX_EXPR_DEPTH>0
@@ -4007,7 +4220,7 @@ void sqlite3Put4byte(u8*, u32);
 /*
 ** If the SQLITE_ENABLE IOTRACE exists then the global variable
 ** sqlite3IoTrace is a pointer to a printf-like routine used to
-** print I/O tracing messages. 
+** print I/O tracing messages.
 */
 #ifdef SQLITE_ENABLE_IOTRACE
 # define IOTRACE(A)  if( sqlite3IoTrace ){ sqlite3IoTrace A; }
@@ -4041,7 +4254,7 @@ SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...);
 ** that allocations that might have been satisfied by lookaside are not
 ** passed back to non-lookaside free() routines.  Asserts such as the
 ** example above are placed on the non-lookaside free() routines to verify
-** this constraint. 
+** this constraint.
 **
 ** All of this is no-op for a production build.  It only comes into
 ** play when the SQLITE_MEMDEBUG compile-time option is used.
@@ -4072,4 +4285,9 @@ int sqlite3ThreadJoin(SQLiteThread*, void**);
 int sqlite3DbstatRegister(sqlite3*);
 #endif
 
-#endif /* _SQLITEINT_H_ */
+int sqlite3ExprVectorSize(Expr *pExpr);
+int sqlite3ExprIsVector(Expr *pExpr);
+Expr *sqlite3VectorFieldSubexpr(Expr*, int);
+Expr *sqlite3ExprForVectorField(Parse*,Expr*,int);
+
+#endif /* SQLITEINT_H */
diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h
index 75cad127..0554e615 100644
--- a/src/sqliteLimit.h
+++ b/src/sqliteLimit.h
@@ -101,13 +101,13 @@
 ** The suggested maximum number of in-memory pages to use for
 ** the main database table and for temporary tables.
 **
-** IMPLEMENTATION-OF: R-31093-59126 The default suggested cache size
-** is 2000 pages.
+** IMPLEMENTATION-OF: R-30185-15359 The default suggested cache size is -2000,
+** which means the cache size is limited to 2048000 bytes of memory.
 ** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be
 ** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options.
 */
 #ifndef SQLITE_DEFAULT_CACHE_SIZE
-# define SQLITE_DEFAULT_CACHE_SIZE  2000
+# define SQLITE_DEFAULT_CACHE_SIZE  -2000
 #endif
 
 /*
@@ -120,8 +120,9 @@
 
 /*
 ** The maximum number of attached databases.  This must be between 0
-** and 62.  The upper bound on 62 is because a 64-bit integer bitmap
-** is used internally to track attached databases.
+** and 125.  The upper bound of 125 is because the attached databases are
+** counted using a signed 8-bit integer which has a maximum value of 127
+** and we have to allow 2 extra counts for the "main" and "temp" databases.
 */
 #ifndef SQLITE_MAX_ATTACHED
 # define SQLITE_MAX_ATTACHED 10
@@ -156,7 +157,7 @@
 ** The default size of a database page.
 */
 #ifndef SQLITE_DEFAULT_PAGE_SIZE
-# define SQLITE_DEFAULT_PAGE_SIZE 1024
+# define SQLITE_DEFAULT_PAGE_SIZE 4096
 #endif
 #if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE
 # undef SQLITE_DEFAULT_PAGE_SIZE
diff --git a/src/status.c b/src/status.c
index 69f92ff7..24dcad45 100644
--- a/src/status.c
+++ b/src/status.c
@@ -158,7 +158,7 @@ int sqlite3_status64(
   return SQLITE_OK;
 }
 int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){
-  sqlite3_int64 iCur, iHwtr;
+  sqlite3_int64 iCur = 0, iHwtr = 0;
   int rc;
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT;
@@ -219,6 +219,7 @@ int sqlite3_db_status(
     ** by all pagers associated with the given database connection.  The
     ** highwater mark is meaningless and is returned as zero.
     */
+    case SQLITE_DBSTATUS_CACHE_USED_SHARED:
     case SQLITE_DBSTATUS_CACHE_USED: {
       int totalUsed = 0;
       int i;
@@ -227,7 +228,11 @@ int sqlite3_db_status(
         Btree *pBt = db->aDb[i].pBt;
         if( pBt ){
           Pager *pPager = sqlite3BtreePager(pBt);
-          totalUsed += sqlite3PagerMemUsed(pPager);
+          int nByte = sqlite3PagerMemUsed(pPager);
+          if( op==SQLITE_DBSTATUS_CACHE_USED_SHARED ){
+            nByte = nByte / sqlite3BtreeConnectionCount(pBt);
+          }
+          totalUsed += nByte;
         }
       }
       sqlite3BtreeLeaveAll(db);
diff --git a/src/table.c b/src/table.c
index 153bfb31..a50d83ce 100644
--- a/src/table.c
+++ b/src/table.c
@@ -101,7 +101,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){
   return 0;
 
 malloc_failed:
-  p->rc = SQLITE_NOMEM;
+  p->rc = SQLITE_NOMEM_BKPT;
   return 1;
 }
 
@@ -142,7 +142,7 @@ int sqlite3_get_table(
   res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc );
   if( res.azResult==0 ){
      db->errCode = SQLITE_NOMEM;
-     return SQLITE_NOMEM;
+     return SQLITE_NOMEM_BKPT;
   }
   res.azResult[0] = 0;
   rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg);
@@ -171,7 +171,7 @@ int sqlite3_get_table(
     if( azNew==0 ){
       sqlite3_free_table(&res.azResult[1]);
       db->errCode = SQLITE_NOMEM;
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }
     res.azResult = azNew;
   }
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 77da6eef..b2f13bd5 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -30,10 +30,17 @@
 ** If requested, include the SQLite compiler options file for MSVC.
 */
 #if defined(INCLUDE_MSVC_H)
-#include "msvc.h"
+# include "msvc.h"
 #endif
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"
+#else
+# include "tcl.h"
+# ifndef SQLITE_TCLAPI
+#  define SQLITE_TCLAPI
+# endif
+#endif
 #include 
 
 /*
@@ -133,6 +140,7 @@ struct SqliteDb {
   char *zBusy;               /* The busy callback routine */
   char *zCommit;             /* The commit hook callback routine */
   char *zTrace;              /* The trace callback routine */
+  char *zTraceV2;            /* The trace_v2 callback routine */
   char *zProfile;            /* The profile callback routine */
   char *zProgress;           /* The progress callback routine */
   char *zAuth;               /* The authorization callback routine */
@@ -140,6 +148,7 @@ struct SqliteDb {
   char *zNull;               /* Text to substitute for an SQL NULL value */
   SqlFunc *pFunc;            /* List of SQL functions */
   Tcl_Obj *pUpdateHook;      /* Update hook script (if any) */
+  Tcl_Obj *pPreUpdateHook;   /* Pre-update hook script (if any) */
   Tcl_Obj *pRollbackHook;    /* Rollback hook script (if any) */
   Tcl_Obj *pWalHook;         /* WAL hook script (if any) */
   Tcl_Obj *pUnlockNotify;    /* Unlock notify script (if any) */
@@ -191,7 +200,7 @@ static void closeIncrblobChannels(SqliteDb *pDb){
   for(p=pDb->pIncrblob; p; p=pNext){
     pNext = p->pNext;
 
-    /* Note: Calling unregister here call Tcl_Close on the incrblob channel, 
+    /* Note: Calling unregister here call Tcl_Close on the incrblob channel,
     ** which deletes the IncrblobChannel structure at *p. So do not
     ** call Tcl_Free() here.
     */
@@ -202,7 +211,10 @@ static void closeIncrblobChannels(SqliteDb *pDb){
 /*
 ** Close an incremental blob channel.
 */
-static int incrblobClose(ClientData instanceData, Tcl_Interp *interp){
+static int SQLITE_TCLAPI incrblobClose(
+  ClientData instanceData,
+  Tcl_Interp *interp
+){
   IncrblobChannel *p = (IncrblobChannel *)instanceData;
   int rc = sqlite3_blob_close(p->pBlob);
   sqlite3 *db = p->pDb->db;
@@ -231,9 +243,9 @@ static int incrblobClose(ClientData instanceData, Tcl_Interp *interp){
 /*
 ** Read data from an incremental blob channel.
 */
-static int incrblobInput(
-  ClientData instanceData, 
-  char *buf, 
+static int SQLITE_TCLAPI incrblobInput(
+  ClientData instanceData,
+  char *buf,
   int bufSize,
   int *errorCodePtr
 ){
@@ -263,9 +275,9 @@ static int incrblobInput(
 /*
 ** Write data to an incremental blob channel.
 */
-static int incrblobOutput(
-  ClientData instanceData, 
-  CONST char *buf, 
+static int SQLITE_TCLAPI incrblobOutput(
+  ClientData instanceData,
+  CONST char *buf,
   int toWrite,
   int *errorCodePtr
 ){
@@ -296,8 +308,8 @@ static int incrblobOutput(
 /*
 ** Seek an incremental blob channel.
 */
-static int incrblobSeek(
-  ClientData instanceData, 
+static int SQLITE_TCLAPI incrblobSeek(
+  ClientData instanceData,
   long offset,
   int seekMode,
   int *errorCodePtr
@@ -322,10 +334,17 @@ static int incrblobSeek(
 }
 
 
-static void incrblobWatch(ClientData instanceData, int mode){ 
-  /* NO-OP */ 
+static void SQLITE_TCLAPI incrblobWatch(
+  ClientData instanceData,
+  int mode
+){
+  /* NO-OP */
 }
-static int incrblobHandle(ClientData instanceData, int dir, ClientData *hPtr){
+static int SQLITE_TCLAPI incrblobHandle(
+  ClientData instanceData,
+  int dir,
+  ClientData *hPtr
+){
   return TCL_ERROR;
 }
 
@@ -351,11 +370,11 @@ static Tcl_ChannelType IncrblobChannelType = {
 ** Create a new incrblob channel.
 */
 static int createIncrblobChannel(
-  Tcl_Interp *interp, 
-  SqliteDb *pDb, 
+  Tcl_Interp *interp,
+  SqliteDb *pDb,
   const char *zDb,
-  const char *zTable, 
-  const char *zColumn, 
+  const char *zTable,
+  const char *zColumn,
   sqlite_int64 iRow,
   int isReadonly
 ){
@@ -437,7 +456,7 @@ static SqlFunc *findSqlFunc(SqliteDb *pDb, const char *zName){
   pNew = (SqlFunc*)Tcl_Alloc( sizeof(*pNew) + nName + 1 );
   pNew->zName = (char*)&pNew[1];
   memcpy(pNew->zName, zName, nName+1);
-  for(p=pDb->pFunc; p; p=p->pNext){ 
+  for(p=pDb->pFunc; p; p=p->pNext){
     if( sqlite3_stricmp(p->zName, pNew->zName)==0 ){
       Tcl_Free((char*)pNew);
       return p;
@@ -484,7 +503,7 @@ static void flushStmtCache(SqliteDb *pDb){
 ** TCL calls this procedure when an sqlite3 database command is
 ** deleted.
 */
-static void DbDeleteCmd(void *db){
+static void SQLITE_TCLAPI DbDeleteCmd(void *db){
   SqliteDb *pDb = (SqliteDb*)db;
   flushStmtCache(pDb);
   closeIncrblobChannels(pDb);
@@ -507,6 +526,9 @@ static void DbDeleteCmd(void *db){
   if( pDb->zTrace ){
     Tcl_Free(pDb->zTrace);
   }
+  if( pDb->zTraceV2 ){
+    Tcl_Free(pDb->zTraceV2);
+  }
   if( pDb->zProfile ){
     Tcl_Free(pDb->zProfile);
   }
@@ -519,6 +541,9 @@ static void DbDeleteCmd(void *db){
   if( pDb->pUpdateHook ){
     Tcl_DecrRefCount(pDb->pUpdateHook);
   }
+  if( pDb->pPreUpdateHook ){
+    Tcl_DecrRefCount(pDb->pPreUpdateHook);
+  }
   if( pDb->pRollbackHook ){
     Tcl_DecrRefCount(pDb->pRollbackHook);
   }
@@ -565,7 +590,8 @@ static int DbProgressHandler(void *cd){
 }
 #endif
 
-#ifndef SQLITE_OMIT_TRACE
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
 /*
 ** This routine is called by the SQLite trace handler whenever a new
 ** block of SQL is executed.  The TCL script in pDb->zTrace is executed.
@@ -585,6 +611,83 @@ static void DbTraceHandler(void *cd, const char *zSql){
 
 #ifndef SQLITE_OMIT_TRACE
 /*
+** This routine is called by the SQLite trace_v2 handler whenever a new
+** supported event is generated.  Unsupported event types are ignored.
+** The TCL script in pDb->zTraceV2 is executed, with the arguments for
+** the event appended to it (as list elements).
+*/
+static int DbTraceV2Handler(
+  unsigned type, /* One of the SQLITE_TRACE_* event types. */
+  void *cd,      /* The original context data pointer. */
+  void *pd,      /* Primary event data, depends on event type. */
+  void *xd       /* Extra event data, depends on event type. */
+){
+  SqliteDb *pDb = (SqliteDb*)cd;
+  Tcl_Obj *pCmd;
+
+  switch( type ){
+    case SQLITE_TRACE_STMT: {
+      sqlite3_stmt *pStmt = (sqlite3_stmt *)pd;
+      char *zSql = (char *)xd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewStringObj(zSql, -1));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+    case SQLITE_TRACE_PROFILE: {
+      sqlite3_stmt *pStmt = (sqlite3_stmt *)pd;
+      sqlite3_int64 ns = (sqlite3_int64)xd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)ns));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+    case SQLITE_TRACE_ROW: {
+      sqlite3_stmt *pStmt = (sqlite3_stmt *)pd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+    case SQLITE_TRACE_CLOSE: {
+      sqlite3 *db = (sqlite3 *)pd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)db));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+#endif
+
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
+/*
 ** This routine is called by the SQLite profile handler after a statement
 ** SQL has executed.  The TCL script in pDb->zProfile is evaluated.
 */
@@ -633,9 +736,9 @@ static void DbRollbackHandler(void *clientData){
 ** This procedure handles wal_hook callbacks.
 */
 static int DbWalHandler(
-  void *clientData, 
-  sqlite3 *db, 
-  const char *zDb, 
+  void *clientData,
+  sqlite3 *db,
+  const char *zDb,
   int nEntry
 ){
   int ret = SQLITE_OK;
@@ -649,7 +752,7 @@ static int DbWalHandler(
   Tcl_IncrRefCount(p);
   Tcl_ListObjAppendElement(interp, p, Tcl_NewStringObj(zDb, -1));
   Tcl_ListObjAppendElement(interp, p, Tcl_NewIntObj(nEntry));
-  if( TCL_OK!=Tcl_EvalObjEx(interp, p, 0) 
+  if( TCL_OK!=Tcl_EvalObjEx(interp, p, 0)
    || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &ret)
   ){
     Tcl_BackgroundError(interp);
@@ -686,23 +789,63 @@ static void DbUnlockNotify(void **apArg, int nArg){
 }
 #endif
 
-static void DbUpdateHandler(
-  void *p, 
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** Pre-update hook callback.
+*/
+static void DbPreUpdateHandler(
+  void *p,
+  sqlite3 *db,
   int op,
-  const char *zDb, 
-  const char *zTbl, 
+  const char *zDb,
+  const char *zTbl,
+  sqlite_int64 iKey1,
+  sqlite_int64 iKey2
+){
+  SqliteDb *pDb = (SqliteDb *)p;
+  Tcl_Obj *pCmd;
+  static const char *azStr[] = {"DELETE", "INSERT", "UPDATE"};
+
+  assert( (SQLITE_DELETE-1)/9 == 0 );
+  assert( (SQLITE_INSERT-1)/9 == 1 );
+  assert( (SQLITE_UPDATE-1)/9 == 2 );
+  assert( pDb->pPreUpdateHook );
+  assert( db==pDb->db );
+  assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
+
+  pCmd = Tcl_DuplicateObj(pDb->pPreUpdateHook);
+  Tcl_IncrRefCount(pCmd);
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(azStr[(op-1)/9], -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zDb, -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zTbl, -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewWideIntObj(iKey1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewWideIntObj(iKey2));
+  Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+  Tcl_DecrRefCount(pCmd);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+static void DbUpdateHandler(
+  void *p,
+  int op,
+  const char *zDb,
+  const char *zTbl,
   sqlite_int64 rowid
 ){
   SqliteDb *pDb = (SqliteDb *)p;
   Tcl_Obj *pCmd;
+  static const char *azStr[] = {"DELETE", "INSERT", "UPDATE"};
+
+  assert( (SQLITE_DELETE-1)/9 == 0 );
+  assert( (SQLITE_INSERT-1)/9 == 1 );
+  assert( (SQLITE_UPDATE-1)/9 == 2 );
 
   assert( pDb->pUpdateHook );
   assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
 
   pCmd = Tcl_DuplicateObj(pDb->pUpdateHook);
   Tcl_IncrRefCount(pCmd);
-  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(
-    ( (op==SQLITE_INSERT)?"INSERT":(op==SQLITE_UPDATE)?"UPDATE":"DELETE"), -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(azStr[(op-1)/9], -1));
   Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zDb, -1));
   Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zTbl, -1));
   Tcl_ListObjAppendElement(0, pCmd, Tcl_NewWideIntObj(rowid));
@@ -771,7 +914,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
     ** script object, lappend the arguments, then evaluate the copy.
     **
     ** By "shallow" copy, we mean only the outer list Tcl_Obj is duplicated.
-    ** The new Tcl_Obj contains pointers to the original list elements. 
+    ** The new Tcl_Obj contains pointers to the original list elements.
     ** That way, when Tcl_EvalObjv() is run and shimmers the first element
     ** of the list to tclCmdNameType, that alternate representation will
     ** be preserved and reused on the next invocation.
@@ -779,15 +922,15 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
     Tcl_Obj **aArg;
     int nArg;
     if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){
-      sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); 
+      sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
       return;
-    }     
+    }
     pCmd = Tcl_NewListObj(nArg, aArg);
     Tcl_IncrRefCount(pCmd);
     for(i=0; iinterp, pCmd, pVal);
       if( rc ){
         Tcl_DecrRefCount(pCmd);
-        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); 
+        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
         return;
       }
     }
@@ -837,7 +980,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
   }
 
   if( rc && rc!=TCL_RETURN ){
-    sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); 
+    sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
   }else{
     Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
     int n;
@@ -939,7 +1082,7 @@ static int auth_callback(
   Tcl_DStringAppendElement(&str, zArg4 ? zArg4 : "");
 #ifdef SQLITE_USER_AUTHENTICATION
   Tcl_DStringAppendElement(&str, zArg5 ? zArg5 : "");
-#endif  
+#endif
   rc = Tcl_GlobalEval(pDb->interp, Tcl_DStringValue(&str));
   Tcl_DStringFree(&str);
   zReply = rc==TCL_OK ? Tcl_GetStringResult(pDb->interp) : "SQLITE_DENY";
@@ -1010,7 +1153,7 @@ static char *local_getline(char *zPrompt, FILE *in){
 ** It is invoked after evaluating the script SCRIPT to commit or rollback
 ** the transaction or savepoint opened by the [transaction] command.
 */
-static int DbTransPostCmd(
+static int SQLITE_TCLAPI DbTransPostCmd(
   ClientData data[],                   /* data[0] is the Sqlite3Db* for $db */
   Tcl_Interp *interp,                  /* Tcl interpreter */
   int result                           /* Result of evaluating SCRIPT */
@@ -1031,12 +1174,12 @@ static int DbTransPostCmd(
   pDb->disableAuth++;
   if( sqlite3_exec(pDb->db, zEnd, 0, 0, 0) ){
       /* This is a tricky scenario to handle. The most likely cause of an
-      ** error is that the exec() above was an attempt to commit the 
+      ** error is that the exec() above was an attempt to commit the
       ** top-level transaction that returned SQLITE_BUSY. Or, less likely,
       ** that an IO-error has occurred. In either case, throw a Tcl exception
       ** and try to rollback the transaction.
       **
-      ** But it could also be that the user executed one or more BEGIN, 
+      ** But it could also be that the user executed one or more BEGIN,
       ** COMMIT, SAVEPOINT, RELEASE or ROLLBACK commands that are confusing
       ** this method's logic. Not clear how this would be best handled.
       */
@@ -1055,7 +1198,7 @@ static int DbTransPostCmd(
 ** Unless SQLITE_TEST is defined, this function is a simple wrapper around
 ** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either
 ** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending
-** on whether or not the [db_use_legacy_prepare] command has been used to 
+** on whether or not the [db_use_legacy_prepare] command has been used to
 ** configure the connection.
 */
 static int dbPrepare(
@@ -1111,7 +1254,7 @@ static int dbPrepareAndBind(
 
   for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pPreStmt->pNext){
     int n = pPreStmt->nSql;
-    if( nSql>=n 
+    if( nSql>=n
         && memcmp(pPreStmt->zSql, zSql, n)==0
         && (zSql[n]==0 || zSql[n-1]==';')
     ){
@@ -1137,7 +1280,7 @@ static int dbPrepareAndBind(
       break;
     }
   }
-  
+
   /* If no prepared statement was found. Compile the SQL text. Also allocate
   ** a new SqlPreparedStmt structure.  */
   if( pPreStmt==0 ){
@@ -1183,7 +1326,7 @@ static int dbPrepareAndBind(
   assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql );
   assert( 0==memcmp(pPreStmt->zSql, zSql, pPreStmt->nSql) );
 
-  /* Bind values to parameters that begin with $ or : */  
+  /* Bind values to parameters that begin with $ or : */
   for(i=1; i<=nVar; i++){
     const char *zVar = sqlite3_bind_parameter_name(pStmt, i);
     if( zVar!=0 && (zVar[0]=='$' || zVar[0]==':' || zVar[0]=='@') ){
@@ -1271,8 +1414,8 @@ static void dbReleaseStmt(
       assert( pDb->nStmt>0 );
     }
     pDb->nStmt++;
-   
-    /* If we have too many statement in cache, remove the surplus from 
+
+    /* If we have too many statement in cache, remove the surplus from
     ** the end of the cache list.  */
     while( pDb->nStmt>pDb->maxStmt ){
       SqlPreparedStmt *pLast = pDb->stmtLast;
@@ -1326,8 +1469,8 @@ static void dbReleaseColumnNames(DbEvalContext *p){
 ** If pArray is not NULL, then it contains the name of a Tcl array
 ** variable. The "*" member of this array is set to a list containing
 ** the names of the columns returned by the statement as part of each
-** call to dbEvalStep(), in order from left to right. e.g. if the names 
-** of the returned columns are a, b and c, it does the equivalent of the 
+** call to dbEvalStep(), in order from left to right. e.g. if the names
+** of the returned columns are a, b and c, it does the equivalent of the
 ** tcl command:
 **
 **     set ${pArray}(*) {a b c}
@@ -1448,7 +1591,7 @@ static int dbEvalStep(DbEvalContext *p){
 #if SQLITE_TEST
         if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){
           /* If the runtime error was an SQLITE_SCHEMA, and the database
-          ** handle is configured to use the legacy sqlite3_prepare() 
+          ** handle is configured to use the legacy sqlite3_prepare()
           ** interface, retry prepare()/step() on the same SQL statement.
           ** This only happens once. If there is a second SQLITE_SCHEMA
           ** error, the error will be returned to the caller. */
@@ -1536,11 +1679,11 @@ static int DbUseNre(void){
   return( (major==8 && minor>=6) || major>8 );
 }
 #else
-/* 
+/*
 ** Compiling using headers earlier than 8.6. In this case NR cannot be
 ** used, so DbUseNre() to always return zero. Add #defines for the other
 ** Tcl_NRxxx() functions to prevent them from causing compilation errors,
-** even though the only invocations of them are within conditional blocks 
+** even though the only invocations of them are within conditional blocks
 ** of the form:
 **
 **   if( DbUseNre() ) { ... }
@@ -1557,7 +1700,7 @@ static int DbUseNre(void){
 **
 **   $db eval SQL ?ARRAYNAME? SCRIPT
 */
-static int DbEvalNextCmd(
+static int SQLITE_TCLAPI DbEvalNextCmd(
   ClientData data[],                   /* data[0] is the (DbEvalContext*) */
   Tcl_Interp *interp,                  /* Tcl interpreter */
   int result                           /* Result so far */
@@ -1586,11 +1729,11 @@ static int DbEvalNextCmd(
       }
     }
 
-    /* The required interpreter variables are now populated with the data 
+    /* The required interpreter variables are now populated with the data
     ** from the current row. If using NRE, schedule callbacks to evaluate
     ** script pScript, then to invoke this function again to fetch the next
     ** row (or clean up if there is no next row or the script throws an
-    ** exception). After scheduling the callbacks, return control to the 
+    ** exception). After scheduling the callbacks, return control to the
     ** caller.
     **
     ** If not using NRE, evaluate pScript directly and continue with the
@@ -1614,6 +1757,46 @@ static int DbEvalNextCmd(
   return rc;
 }
 
+/*
+** This function is used by the implementations of the following database
+** handle sub-commands:
+**
+**   $db update_hook ?SCRIPT?
+**   $db wal_hook ?SCRIPT?
+**   $db commit_hook ?SCRIPT?
+**   $db preupdate hook ?SCRIPT?
+*/
+static void DbHookCmd(
+  Tcl_Interp *interp,             /* Tcl interpreter */
+  SqliteDb *pDb,                  /* Database handle */
+  Tcl_Obj *pArg,                  /* SCRIPT argument (or NULL) */
+  Tcl_Obj **ppHook                /* Pointer to member of SqliteDb */
+){
+  sqlite3 *db = pDb->db;
+
+  if( *ppHook ){
+    Tcl_SetObjResult(interp, *ppHook);
+    if( pArg ){
+      Tcl_DecrRefCount(*ppHook);
+      *ppHook = 0;
+    }
+  }
+  if( pArg ){
+    assert( !(*ppHook) );
+    if( Tcl_GetCharLength(pArg)>0 ){
+      *ppHook = pArg;
+      Tcl_IncrRefCount(*ppHook);
+    }
+  }
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  sqlite3_preupdate_hook(db, (pDb->pPreUpdateHook?DbPreUpdateHandler:0), pDb);
+#endif
+  sqlite3_update_hook(db, (pDb->pUpdateHook?DbUpdateHandler:0), pDb);
+  sqlite3_rollback_hook(db, (pDb->pRollbackHook?DbRollbackHandler:0), pDb);
+  sqlite3_wal_hook(db, (pDb->pWalHook?DbWalHandler:0), pDb);
+}
+
 /*
 ** The "sqlite" command below creates a new Tcl command for each
 ** connection it opens to an SQLite database.  This routine is invoked
@@ -1627,7 +1810,12 @@ static int DbEvalNextCmd(
 ** and calls that connection "db1".  The second command causes this
 ** subroutine to be invoked.
 */
-static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
+static int SQLITE_TCLAPI DbObjCmd(
+  void *cd,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *const*objv
+){
   SqliteDb *pDb = (SqliteDb*)cd;
   int choice;
   int rc = TCL_OK;
@@ -1639,11 +1827,13 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     "errorcode",          "eval",              "exists",
     "function",           "incrblob",          "interrupt",
     "last_insert_rowid",  "nullvalue",         "onecolumn",
-    "profile",            "progress",          "rekey",
-    "restore",            "rollback_hook",     "status",
-    "timeout",            "total_changes",     "trace",
-    "transaction",        "unlock_notify",     "update_hook",
-    "version",            "wal_hook",          0
+    "preupdate",          "profile",           "progress",
+    "rekey",              "restore",           "rollback_hook",
+    "status",             "timeout",           "total_changes",
+    "trace",              "trace_v2",          "transaction",
+    "unlock_notify",      "update_hook",       "version",
+    "wal_hook",
+    0
   };
   enum DB_enum {
     DB_AUTHORIZER,        DB_BACKUP,           DB_BUSY,
@@ -1653,11 +1843,12 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     DB_ERRORCODE,         DB_EVAL,             DB_EXISTS,
     DB_FUNCTION,          DB_INCRBLOB,         DB_INTERRUPT,
     DB_LAST_INSERT_ROWID, DB_NULLVALUE,        DB_ONECOLUMN,
-    DB_PROFILE,           DB_PROGRESS,         DB_REKEY,
-    DB_RESTORE,           DB_ROLLBACK_HOOK,    DB_STATUS,
-    DB_TIMEOUT,           DB_TOTAL_CHANGES,    DB_TRACE,
-    DB_TRANSACTION,       DB_UNLOCK_NOTIFY,    DB_UPDATE_HOOK,
-    DB_VERSION,           DB_WAL_HOOK
+    DB_PREUPDATE,         DB_PROFILE,          DB_PROGRESS,
+    DB_REKEY,             DB_RESTORE,          DB_ROLLBACK_HOOK,
+    DB_STATUS,            DB_TIMEOUT,          DB_TOTAL_CHANGES,
+    DB_TRACE,             DB_TRACE_V2,         DB_TRANSACTION,
+    DB_UNLOCK_NOTIFY,     DB_UPDATE_HOOK,      DB_VERSION,
+    DB_WAL_HOOK,
   };
   /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */
 
@@ -1843,7 +2034,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
         return TCL_ERROR;
       }else{
         if( TCL_ERROR==Tcl_GetIntFromObj(interp, objv[3], &n) ){
-          Tcl_AppendResult( interp, "cannot convert \"", 
+          Tcl_AppendResult( interp, "cannot convert \"",
                Tcl_GetStringFromObj(objv[3],0), "\" to integer", (char*)0);
           return TCL_ERROR;
         }else{
@@ -1857,7 +2048,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
         }
       }
     }else{
-      Tcl_AppendResult( interp, "bad option \"", 
+      Tcl_AppendResult( interp, "bad option \"",
           Tcl_GetStringFromObj(objv[2],0), "\": must be flush or size",
           (char*)0);
       return TCL_ERROR;
@@ -1868,7 +2059,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   /*     $db changes
   **
   ** Return the number of rows that were modified, inserted, or deleted by
-  ** the most recent INSERT, UPDATE or DELETE statement, not including 
+  ** the most recent INSERT, UPDATE or DELETE statement, not including
   ** any changes made by trigger programs.
   */
   case DB_CHANGES: {
@@ -1915,7 +2106,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     pCollate->zScript = (char*)&pCollate[1];
     pDb->pCollate = pCollate;
     memcpy(pCollate->zScript, zScript, nScript+1);
-    if( sqlite3_create_collation(pDb->db, zName, SQLITE_UTF8, 
+    if( sqlite3_create_collation(pDb->db, zName, SQLITE_UTF8,
         pCollate, tclSqlCollate) ){
       Tcl_SetResult(interp, (char *)sqlite3_errmsg(pDb->db), TCL_VOLATILE);
       return TCL_ERROR;
@@ -2041,7 +2232,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     const char *zSep;
     const char *zNull;
     if( objc<5 || objc>7 ){
-      Tcl_WrongNumArgs(interp, 2, objv, 
+      Tcl_WrongNumArgs(interp, 2, objv,
          "CONFLICT-ALGORITHM TABLE FILENAME ?SEPARATOR? ?NULLINDICATOR?");
       return TCL_ERROR;
     }
@@ -2070,7 +2261,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
        strcmp(zConflict, "fail"    ) != 0 &&
        strcmp(zConflict, "ignore"  ) != 0 &&
        strcmp(zConflict, "replace" ) != 0 ) {
-      Tcl_AppendResult(interp, "Error: \"", zConflict, 
+      Tcl_AppendResult(interp, "Error: \"", zConflict,
             "\", conflict-algorithm must be one of: rollback, "
             "abort, fail, ignore, or replace", (char*)0);
       return TCL_ERROR;
@@ -2159,7 +2350,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       for(i=0; i0 && strcmp(azCol[i], zNull)==0)
-          || strlen30(azCol[i])==0 
+          || strlen30(azCol[i])==0
         ){
           sqlite3_bind_null(pStmt, i+1);
         }else{
@@ -2238,8 +2429,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   ** The onecolumn method is the equivalent of:
   **     lindex [$db eval $sql] 0
   */
-  case DB_EXISTS: 
+  case DB_EXISTS:
   case DB_ONECOLUMN: {
+    Tcl_Obj *pResult = 0;
     DbEvalContext sEval;
     if( objc!=3 ){
       Tcl_WrongNumArgs(interp, 2, objv, "SQL");
@@ -2250,21 +2442,22 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     rc = dbEvalStep(&sEval);
     if( choice==DB_ONECOLUMN ){
       if( rc==TCL_OK ){
-        Tcl_SetObjResult(interp, dbEvalColumnValue(&sEval, 0));
+        pResult = dbEvalColumnValue(&sEval, 0);
       }else if( rc==TCL_BREAK ){
         Tcl_ResetResult(interp);
       }
     }else if( rc==TCL_BREAK || rc==TCL_OK ){
-      Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc==TCL_OK));
+      pResult = Tcl_NewBooleanObj(rc==TCL_OK);
     }
     dbEvalFinalize(&sEval);
+    if( pResult ) Tcl_SetObjResult(interp, pResult);
 
     if( rc==TCL_BREAK ){
       rc = TCL_OK;
     }
     break;
   }
-   
+
   /*
   **    $db eval $sql ?array? ?{  ...code... }?
   **
@@ -2310,7 +2503,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       }
       pScript = objv[objc-1];
       Tcl_IncrRefCount(pScript);
-      
+
       p = (DbEvalContext *)Tcl_Alloc(sizeof(DbEvalContext));
       dbEvalInit(p, pDb, objv[2], pArray);
 
@@ -2357,7 +2550,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       if( n>2 && strncmp(z, "-deterministic",n)==0 ){
         flags |= SQLITE_DETERMINISTIC;
       }else{
-        Tcl_AppendResult(interp, "bad option \"", z, 
+        Tcl_AppendResult(interp, "bad option \"", z,
             "\": must be -argcount or -deterministic", 0
         );
         return TCL_ERROR;
@@ -2466,7 +2659,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   }
 
   /*
-  **     $db last_insert_rowid 
+  **     $db last_insert_rowid
   **
   ** Return an integer which is the ROWID for the most recent insert.
   */
@@ -2488,7 +2681,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   */
 
   /*    $db progress ?N CALLBACK?
-  ** 
+  **
   ** Invoke the given callback every N virtual machine opcodes while executing
   ** queries.
   */
@@ -2556,7 +2749,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       }else{
         pDb->zProfile = 0;
       }
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
       if( pDb->zProfile ){
         pDb->interp = interp;
         sqlite3_profile(pDb->db, DbProfileHandler, pDb);
@@ -2574,7 +2768,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   ** Change the encryption key on the currently open database.
   */
   case DB_REKEY: {
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
     int nKey;
     void *pKey;
 #endif
@@ -2582,7 +2776,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       Tcl_WrongNumArgs(interp, 2, objv, "KEY");
       return TCL_ERROR;
     }
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
     pKey = Tcl_GetByteArrayFromObj(objv[2], &nKey);
     rc = sqlite3_rekey(pDb->db, pKey, nKey);
     if( rc ){
@@ -2595,7 +2789,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
 
   /*    $db restore ?DATABASE? FILENAME
   **
-  ** Open a database file named FILENAME.  Transfer the content 
+  ** Open a database file named FILENAME.  Transfer the content
   ** of FILENAME into the local database DATABASE (default: "main").
   */
   case DB_RESTORE: {
@@ -2656,7 +2850,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   /*
   **     $db status (step|sort|autoindex)
   **
-  ** Display SQLITE_STMTSTATUS_FULLSCAN_STEP or 
+  ** Display SQLITE_STMTSTATUS_FULLSCAN_STEP or
   ** SQLITE_STMTSTATUS_SORT for the most recent eval.
   */
   case DB_STATUS: {
@@ -2674,15 +2868,15 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     }else if( strcmp(zOp, "autoindex")==0 ){
       v = pDb->nIndex;
     }else{
-      Tcl_AppendResult(interp, 
-            "bad argument: should be autoindex, step, or sort", 
+      Tcl_AppendResult(interp,
+            "bad argument: should be autoindex, step, or sort",
             (char*)0);
       return TCL_ERROR;
     }
     Tcl_SetObjResult(interp, Tcl_NewIntObj(v));
     break;
   }
-  
+
   /*
   **     $db timeout MILLESECONDS
   **
@@ -2698,11 +2892,11 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     sqlite3_busy_timeout(pDb->db, ms);
     break;
   }
-  
+
   /*
   **     $db total_changes
   **
-  ** Return the number of rows that were modified, inserted, or deleted 
+  ** Return the number of rows that were modified, inserted, or deleted
   ** since the database handle was created.
   */
   case DB_TOTAL_CHANGES: {
@@ -2743,7 +2937,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       }else{
         pDb->zTrace = 0;
       }
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
       if( pDb->zTrace ){
         pDb->interp = interp;
         sqlite3_trace(pDb->db, DbTraceHandler, pDb);
@@ -2755,6 +2950,88 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     break;
   }
 
+  /*    $db trace_v2 ?CALLBACK? ?MASK?
+  **
+  ** Make arrangements to invoke the CALLBACK routine for each trace event
+  ** matching the mask that is generated.  The parameters are appended to
+  ** CALLBACK before it is executed.
+  */
+  case DB_TRACE_V2: {
+    if( objc>4 ){
+      Tcl_WrongNumArgs(interp, 2, objv, "?CALLBACK? ?MASK?");
+      return TCL_ERROR;
+    }else if( objc==2 ){
+      if( pDb->zTraceV2 ){
+        Tcl_AppendResult(interp, pDb->zTraceV2, (char*)0);
+      }
+    }else{
+      char *zTraceV2;
+      int len;
+      Tcl_WideInt wMask = 0;
+      if( objc==4 ){
+        static const char *TTYPE_strs[] = {
+          "statement", "profile", "row", "close", 0
+        };
+        enum TTYPE_enum {
+          TTYPE_STMT, TTYPE_PROFILE, TTYPE_ROW, TTYPE_CLOSE
+        };
+        int i;
+        if( TCL_OK!=Tcl_ListObjLength(interp, objv[3], &len) ){
+          return TCL_ERROR;
+        }
+        for(i=0; izTraceV2 ){
+        Tcl_Free(pDb->zTraceV2);
+      }
+      zTraceV2 = Tcl_GetStringFromObj(objv[2], &len);
+      if( zTraceV2 && len>0 ){
+        pDb->zTraceV2 = Tcl_Alloc( len + 1 );
+        memcpy(pDb->zTraceV2, zTraceV2, len+1);
+      }else{
+        pDb->zTraceV2 = 0;
+      }
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+      if( pDb->zTraceV2 ){
+        pDb->interp = interp;
+        sqlite3_trace_v2(pDb->db, (unsigned)wMask, DbTraceV2Handler, pDb);
+      }else{
+        sqlite3_trace_v2(pDb->db, 0, 0, 0);
+      }
+#endif
+    }
+    break;
+  }
+
   /*    $db transaction [-deferred|-immediate|-exclusive] SCRIPT
   **
   ** Start a new transaction (if we are not already in the midst of a
@@ -2807,7 +3084,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     /* If using NRE, schedule a callback to invoke the script pScript, then
     ** a second callback to commit (or rollback) the transaction or savepoint
     ** opened above. If not using NRE, evaluate the script directly, then
-    ** call function DbTransPostCmd() to commit (or rollback) the transaction 
+    ** call function DbTransPostCmd() to commit (or rollback) the transaction
     ** or savepoint.  */
     if( DbUseNre() ){
       Tcl_NRAddCallback(interp, DbTransPostCmd, cd, 0, 0, 0);
@@ -2838,14 +3115,14 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
         Tcl_DecrRefCount(pDb->pUnlockNotify);
         pDb->pUnlockNotify = 0;
       }
-  
+
       if( objc==3 ){
         xNotify = DbUnlockNotify;
         pNotifyArg = (void *)pDb;
         pDb->pUnlockNotify = objv[2];
         Tcl_IncrRefCount(pDb->pUnlockNotify);
       }
-  
+
       if( sqlite3_unlock_notify(pDb->db, xNotify, pNotifyArg) ){
         Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), (char*)0);
         rc = TCL_ERROR;
@@ -2855,50 +3132,111 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     break;
   }
 
+  /*
+  **    $db preupdate_hook count
+  **    $db preupdate_hook hook ?SCRIPT?
+  **    $db preupdate_hook new INDEX
+  **    $db preupdate_hook old INDEX
+  */
+  case DB_PREUPDATE: {
+#ifndef SQLITE_ENABLE_PREUPDATE_HOOK
+    Tcl_AppendResult(interp, "preupdate_hook was omitted at compile-time");
+    rc = TCL_ERROR;
+#else
+    static const char *azSub[] = {"count", "depth", "hook", "new", "old", 0};
+    enum DbPreupdateSubCmd {
+      PRE_COUNT, PRE_DEPTH, PRE_HOOK, PRE_NEW, PRE_OLD
+    };
+    int iSub;
+
+    if( objc<3 ){
+      Tcl_WrongNumArgs(interp, 2, objv, "SUB-COMMAND ?ARGS?");
+    }
+    if( Tcl_GetIndexFromObj(interp, objv[2], azSub, "sub-command", 0, &iSub) ){
+      return TCL_ERROR;
+    }
+
+    switch( (enum DbPreupdateSubCmd)iSub ){
+      case PRE_COUNT: {
+        int nCol = sqlite3_preupdate_count(pDb->db);
+        Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));
+        break;
+      }
+
+      case PRE_HOOK: {
+        if( objc>4 ){
+          Tcl_WrongNumArgs(interp, 2, objv, "hook ?SCRIPT?");
+          return TCL_ERROR;
+        }
+        DbHookCmd(interp, pDb, (objc==4 ? objv[3] : 0), &pDb->pPreUpdateHook);
+        break;
+      }
+
+      case PRE_DEPTH: {
+        Tcl_Obj *pRet;
+        if( objc!=3 ){
+          Tcl_WrongNumArgs(interp, 3, objv, "");
+          return TCL_ERROR;
+        }
+        pRet = Tcl_NewIntObj(sqlite3_preupdate_depth(pDb->db));
+        Tcl_SetObjResult(interp, pRet);
+        break;
+      }
+
+      case PRE_NEW:
+      case PRE_OLD: {
+        int iIdx;
+        sqlite3_value *pValue;
+        if( objc!=4 ){
+          Tcl_WrongNumArgs(interp, 3, objv, "INDEX");
+          return TCL_ERROR;
+        }
+        if( Tcl_GetIntFromObj(interp, objv[3], &iIdx) ){
+          return TCL_ERROR;
+        }
+
+        if( iSub==PRE_OLD ){
+          rc = sqlite3_preupdate_old(pDb->db, iIdx, &pValue);
+        }else{
+          assert( iSub==PRE_NEW );
+          rc = sqlite3_preupdate_new(pDb->db, iIdx, &pValue);
+        }
+
+        if( rc==SQLITE_OK ){
+          Tcl_Obj *pObj;
+          pObj = Tcl_NewStringObj((char*)sqlite3_value_text(pValue), -1);
+          Tcl_SetObjResult(interp, pObj);
+        }else{
+          Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), 0);
+          return TCL_ERROR;
+        }
+      }
+    }
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+    break;
+  }
+
   /*
   **    $db wal_hook ?script?
   **    $db update_hook ?script?
   **    $db rollback_hook ?script?
   */
-  case DB_WAL_HOOK: 
-  case DB_UPDATE_HOOK: 
+  case DB_WAL_HOOK:
+  case DB_UPDATE_HOOK:
   case DB_ROLLBACK_HOOK: {
-
-    /* set ppHook to point at pUpdateHook or pRollbackHook, depending on 
+    /* set ppHook to point at pUpdateHook or pRollbackHook, depending on
     ** whether [$db update_hook] or [$db rollback_hook] was invoked.
     */
-    Tcl_Obj **ppHook; 
-    if( choice==DB_UPDATE_HOOK ){
-      ppHook = &pDb->pUpdateHook;
-    }else if( choice==DB_WAL_HOOK ){
-      ppHook = &pDb->pWalHook;
-    }else{
-      ppHook = &pDb->pRollbackHook;
-    }
-
-    if( objc!=2 && objc!=3 ){
+    Tcl_Obj **ppHook = 0;
+    if( choice==DB_WAL_HOOK ) ppHook = &pDb->pWalHook;
+    if( choice==DB_UPDATE_HOOK ) ppHook = &pDb->pUpdateHook;
+    if( choice==DB_ROLLBACK_HOOK ) ppHook = &pDb->pRollbackHook;
+    if( objc>3 ){
        Tcl_WrongNumArgs(interp, 2, objv, "?SCRIPT?");
        return TCL_ERROR;
     }
-    if( *ppHook ){
-      Tcl_SetObjResult(interp, *ppHook);
-      if( objc==3 ){
-        Tcl_DecrRefCount(*ppHook);
-        *ppHook = 0;
-      }
-    }
-    if( objc==3 ){
-      assert( !(*ppHook) );
-      if( Tcl_GetCharLength(objv[2])>0 ){
-        *ppHook = objv[2];
-        Tcl_IncrRefCount(*ppHook);
-      }
-    }
-
-    sqlite3_update_hook(pDb->db, (pDb->pUpdateHook?DbUpdateHandler:0), pDb);
-    sqlite3_rollback_hook(pDb->db,(pDb->pRollbackHook?DbRollbackHandler:0),pDb);
-    sqlite3_wal_hook(pDb->db,(pDb->pWalHook?DbWalHandler:0),pDb);
 
+    DbHookCmd(interp, pDb, (objc==3 ? objv[2] : 0), ppHook);
     break;
   }
 
@@ -2921,7 +3259,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
 ** Adaptor that provides an objCmd interface to the NRE-enabled
 ** interface implementation.
 */
-static int DbObjCmdAdaptor(
+static int SQLITE_TCLAPI DbObjCmdAdaptor(
   void *cd,
   Tcl_Interp *interp,
   int objc,
@@ -2946,7 +3284,12 @@ static int DbObjCmdAdaptor(
 ** The second argument is the name of the database file.
 **
 */
-static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
+static int SQLITE_TCLAPI DbMain(
+  void *cd,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *const*objv
+){
   SqliteDb *p;
   const char *zArg;
   char *zErrMsg;
@@ -2955,7 +3298,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   const char *zVfs = 0;
   int flags;
   Tcl_DString translatedFilename;
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
   void *pKey = 0;
   int nKey = 0;
 #endif
@@ -2984,7 +3327,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       return TCL_OK;
     }
     if( strcmp(zArg,"-has-codec")==0 ){
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
       Tcl_AppendResult(interp,"1",(char*)0);
 #else
       Tcl_AppendResult(interp,"0",(char*)0);
@@ -2995,7 +3338,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   for(i=3; i+1db ){
     sqlite3_key(p->db, pKey, nKey);
   }
@@ -3342,7 +3685,7 @@ static void MD5Init(MD5Context *ctx){
  * Update context to reflect the concatenation of another buffer full
  * of bytes.
  */
-static 
+static
 void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
         uint32 t;
 
@@ -3388,7 +3731,7 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
 }
 
 /*
- * Final wrapup - pad to 64-byte boundary with the bit pattern 
+ * Final wrapup - pad to 64-byte boundary with the bit pattern
  * 1 0* (64-bit count of bits processed, MSB-first)
  */
 static void MD5Final(unsigned char digest[16], MD5Context *ctx){
@@ -3464,16 +3807,21 @@ static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){
 
 /*
 ** A TCL command for md5.  The argument is the text to be hashed.  The
-** Result is the hash in base64.  
+** Result is the hash in base64.
 */
-static int md5_cmd(void*cd, Tcl_Interp *interp, int argc, const char **argv){
+static int SQLITE_TCLAPI md5_cmd(
+  void*cd,
+  Tcl_Interp *interp,
+  int argc,
+  const char **argv
+){
   MD5Context ctx;
   unsigned char digest[16];
   char zBuf[50];
   void (*converter)(unsigned char*, char*);
 
   if( argc!=2 ){
-    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], 
+    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
         " TEXT\"", (char*)0);
     return TCL_ERROR;
   }
@@ -3490,7 +3838,12 @@ static int md5_cmd(void*cd, Tcl_Interp *interp, int argc, const char **argv){
 ** A TCL command to take the md5 hash of a file.  The argument is the
 ** name of the file.
 */
-static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){
+static int SQLITE_TCLAPI md5file_cmd(
+  void*cd,
+  Tcl_Interp *interp,
+  int argc,
+  const char **argv
+){
   FILE *in;
   MD5Context ctx;
   void (*converter)(unsigned char*, char*);
@@ -3498,13 +3851,13 @@ static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){
   char zBuf[10240];
 
   if( argc!=2 ){
-    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], 
+    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
         " FILENAME\"", (char*)0);
     return TCL_ERROR;
   }
   in = fopen(argv[1],"rb");
   if( in==0 ){
-    Tcl_AppendResult(interp,"unable to open file \"", argv[1], 
+    Tcl_AppendResult(interp,"unable to open file \"", argv[1],
          "\" for reading", (char*)0);
     return TCL_ERROR;
   }
@@ -3570,8 +3923,12 @@ static void md5finalize(sqlite3_context *context){
   MD5DigestToBase16(digest, zBuf);
   sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
 }
-int Md5_Register(sqlite3 *db){
-  int rc = sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0, 
+int Md5_Register(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pThunk
+){
+  int rc = sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0,
                                  md5step, md5finalize);
   sqlite3_overload_function(db, "md5sum", -1);  /* To exercise this API */
   return rc;
@@ -3618,7 +3975,7 @@ static const char *tclsh_main_loop(void);
 
 #ifdef SQLITE_TEST
 static void init_all(Tcl_Interp *);
-static int init_all_cmd(
+static int SQLITE_TCLAPI init_all_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -3648,7 +4005,7 @@ static int init_all_cmd(
 **   to use the sqlite3_prepare_v2() function to prepare statements. If it
 **   is false, sqlite3_prepare().
 */
-static int db_use_legacy_prepare_cmd(
+static int SQLITE_TCLAPI db_use_legacy_prepare_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -3685,7 +4042,7 @@ static int db_use_legacy_prepare_cmd(
 **   return the text representation of the most recently used statement
 **   handle.
 */
-static int db_last_stmt_ptr(
+static int SQLITE_TCLAPI db_last_stmt_ptr(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -3722,7 +4079,7 @@ static int db_last_stmt_ptr(
 ** Configure the interpreter passed as the first argument to have access
 ** to the commands and linked variables that make up:
 **
-**   * the [sqlite3] extension itself, 
+**   * the [sqlite3] extension itself,
 **
 **   * If SQLITE_TCLMD5 or SQLITE_TEST is defined, the Md5 commands, and
 **
@@ -3772,8 +4129,12 @@ static void init_all(Tcl_Interp *interp){
     extern int Sqlitemultiplex_Init(Tcl_Interp*);
     extern int SqliteSuperlock_Init(Tcl_Interp*);
     extern int SqlitetestSyscall_Init(Tcl_Interp*);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+    extern int TestSession_Init(Tcl_Interp*);
+#endif
     extern int Fts5tcl_Init(Tcl_Interp *);
     extern int SqliteRbu_Init(Tcl_Interp*);
+    extern int Sqlitetesttcl_Init(Tcl_Interp*);
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
     extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
 #endif
@@ -3806,7 +4167,7 @@ static void init_all(Tcl_Interp *interp){
     Sqlitetesttclvar_Init(interp);
     Sqlitetestfs_Init(interp);
     SqlitetestThread_Init(interp);
-    SqlitetestOnefile_Init(interp);
+    SqlitetestOnefile_Init();
     SqlitetestOsinst_Init(interp);
     Sqlitetestbackup_Init(interp);
     Sqlitetestintarray_Init(interp);
@@ -3816,8 +4177,12 @@ static void init_all(Tcl_Interp *interp){
     Sqlitemultiplex_Init(interp);
     SqliteSuperlock_Init(interp);
     SqlitetestSyscall_Init(interp);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+    TestSession_Init(interp);
+#endif
     Fts5tcl_Init(interp);
     SqliteRbu_Init(interp);
+    Sqlitetesttcl_Init(interp);
 
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
     Sqlitetestfts3_Init(interp);
@@ -3846,7 +4211,7 @@ static void init_all(Tcl_Interp *interp){
 #endif
 
 #define TCLSH_MAIN main   /* Needed to fake out mktclapp */
-int TCLSH_MAIN(int argc, char **argv){
+int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   Tcl_Interp *interp;
 
 #if !defined(_WIN32_WCE)
diff --git a/src/test1.c b/src/test1.c
index c5e71582..6c9a0ea1 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -19,7 +19,11 @@
 #endif
 
 #include "vdbeInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -75,7 +79,7 @@ void *sqlite3TestTextToPtr(const char *z){
 ** for an sqlite connection instance.  Bad things happen if the
 ** input is not an sqlite connection.
 */
-static int get_sqlite_pointer(
+static int SQLITE_TCLAPI get_sqlite_pointer(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -221,7 +225,7 @@ static void io_trace_callback(const char *zFormat, ...){
 ** I/O tracing begins going into FILENAME. If FILENAME is an empty
 ** string, I/O tracing is turned off.
 */
-static int test_io_trace(
+static int SQLITE_TCLAPI test_io_trace(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -262,7 +266,7 @@ static int test_io_trace(
 **
 ** Also return true if the OMIT_MISUSE environment variable exists.
 */
-static int clang_sanitize_address(
+static int SQLITE_TCLAPI clang_sanitize_address(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -289,7 +293,7 @@ static int clang_sanitize_address(
 ** DB.  The SQL is the string FORMAT.  The format string should contain
 ** one %s or %q.  STRING is the value inserted into %s or %q.
 */
-static int test_exec_printf(
+static int SQLITE_TCLAPI test_exec_printf(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -327,7 +331,7 @@ static int test_exec_printf(
 ** HEX into ASCII.  Most characters are translated as is.  %HH becomes
 ** a hex character.
 */
-static int test_exec_hex(
+static int SQLITE_TCLAPI test_exec_hex(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -373,7 +377,7 @@ static int test_exec_hex(
 **
 ** Enter or leave the mutex on a database connection.
 */
-static int db_enter(
+static int SQLITE_TCLAPI db_enter(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -389,7 +393,7 @@ static int db_enter(
   sqlite3_mutex_enter(db->mutex);
   return TCL_OK;
 }
-static int db_leave(
+static int SQLITE_TCLAPI db_leave(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -411,7 +415,7 @@ static int db_leave(
 **
 ** Invoke the sqlite3_exec interface using the open database DB
 */
-static int test_exec(
+static int SQLITE_TCLAPI test_exec(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -458,7 +462,7 @@ static int test_exec(
 ** Invoke the sqlite3_exec interface using the open database DB.  Discard
 ** all results
 */
-static int test_exec_nr(
+static int SQLITE_TCLAPI test_exec_nr(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -485,7 +489,7 @@ static int test_exec_nr(
 ** concatenate arg0 through argn using separator as the separator.
 ** Return the result.
 */
-static int test_mprintf_z(
+static int SQLITE_TCLAPI test_mprintf_z(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -508,7 +512,7 @@ static int test_mprintf_z(
 ** Test the %n format of sqlite_mprintf().  Return the length of the
 ** input string.
 */
-static int test_mprintf_n(
+static int SQLITE_TCLAPI test_mprintf_n(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -532,7 +536,7 @@ static int test_mprintf_n(
 ** You pass in a format string that requires more than one argument,
 ** bad things will happen.
 */
-static int test_snprintf_int(
+static int SQLITE_TCLAPI test_snprintf_int(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -558,7 +562,7 @@ static int test_snprintf_int(
 ** DB.  The SQL is the string FORMAT.  The format string should contain
 ** one %s or %q.  STRING is the value inserted into %s or %q.
 */
-static int test_get_table_printf(
+static int SQLITE_TCLAPI test_get_table_printf(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -621,7 +625,7 @@ static int test_get_table_printf(
 **
 ** Returns the integer ROWID of the most recent insert.
 */
-static int test_last_rowid(
+static int SQLITE_TCLAPI test_last_rowid(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -645,13 +649,13 @@ static int test_last_rowid(
 **
 ** Set the codec key.
 */
-static int test_key(
+static int SQLITE_TCLAPI test_key(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
   char **argv            /* Text of each argument */
 ){
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
   sqlite3 *db;
   const char *zKey;
   int nKey;
@@ -673,7 +677,7 @@ static int test_key(
 **
 ** Change the codec key.
 */
-static int test_rekey(
+static int SQLITE_TCLAPI test_rekey(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -701,7 +705,7 @@ static int test_rekey(
 **
 ** Closes the database opened by sqlite3_open.
 */
-static int sqlite_test_close(
+static int SQLITE_TCLAPI sqlite_test_close(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -725,7 +729,7 @@ static int sqlite_test_close(
 **
 ** Closes the database opened by sqlite3_open.
 */
-static int sqlite_test_close_v2(
+static int SQLITE_TCLAPI sqlite_test_close_v2(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1009,7 +1013,7 @@ static void nondeterministicFunction(
 ** sqlite3_create_function function while a query is in progress in order
 ** to test the SQLITE_MISUSE detection logic.
 */
-static int test_create_function(
+static int SQLITE_TCLAPI test_create_function(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1166,7 +1170,7 @@ static void legacyCountFinalize(sqlite3_context *context){
 ** "legacy_count()" with the supplied database handle. This is used
 ** to test the deprecated sqlite3_aggregate_count() API.
 */
-static int test_create_aggregate(
+static int SQLITE_TCLAPI test_create_aggregate(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1207,7 +1211,7 @@ static int test_create_aggregate(
 ** Puts uses a separate buffer and debugging statements will be out of
 ** sequence if it is used.
 */
-static int test_printf(
+static int SQLITE_TCLAPI test_printf(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1229,7 +1233,7 @@ static int test_printf(
 **
 ** Call mprintf with three integer arguments
 */
-static int sqlite3_mprintf_int(
+static int SQLITE_TCLAPI sqlite3_mprintf_int(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1256,7 +1260,7 @@ static int sqlite3_mprintf_int(
 **
 ** Call mprintf with three 64-bit integer arguments
 */
-static int sqlite3_mprintf_int64(
+static int SQLITE_TCLAPI sqlite3_mprintf_int64(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1271,7 +1275,7 @@ static int sqlite3_mprintf_int64(
     return TCL_ERROR;
   }
   for(i=2; i<5; i++){
-    if( sqlite3Atoi64(argv[i], &a[i-2], 1000000, SQLITE_UTF8) ){
+    if( sqlite3Atoi64(argv[i], &a[i-2], sqlite3Strlen30(argv[i]), SQLITE_UTF8) ){
       Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", 0);
       return TCL_ERROR;
     }
@@ -1289,7 +1293,7 @@ static int sqlite3_mprintf_int64(
 ** same as sqlite3_mprintf_int or sqlite3_mprintf_int64, depending on
 ** platform.
 */
-static int sqlite3_mprintf_long(
+static int SQLITE_TCLAPI sqlite3_mprintf_long(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1320,7 +1324,7 @@ static int sqlite3_mprintf_long(
 **
 ** Call mprintf with two integer arguments and one string argument
 */
-static int sqlite3_mprintf_str(
+static int SQLITE_TCLAPI sqlite3_mprintf_str(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1347,7 +1351,7 @@ static int sqlite3_mprintf_str(
 **
 ** Call mprintf with two integer arguments and one string argument
 */
-static int sqlite3_snprintf_str(
+static int SQLITE_TCLAPI sqlite3_snprintf_str(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1381,7 +1385,7 @@ static int sqlite3_snprintf_str(
 **
 ** Call mprintf with two integer arguments and one double argument
 */
-static int sqlite3_mprintf_double(
+static int SQLITE_TCLAPI sqlite3_mprintf_double(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1412,7 +1416,7 @@ static int sqlite3_mprintf_double(
 ** two arguments given above.  This is used to generate overflow and underflow
 ** doubles to test that they are converted properly.
 */
-static int sqlite3_mprintf_scaled(
+static int SQLITE_TCLAPI sqlite3_mprintf_scaled(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1442,7 +1446,7 @@ static int sqlite3_mprintf_scaled(
 ** two arguments given above.  This is used to generate overflow and underflow
 ** doubles to test that they are converted properly.
 */
-static int sqlite3_mprintf_stronly(
+static int SQLITE_TCLAPI sqlite3_mprintf_stronly(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1466,7 +1470,7 @@ static int sqlite3_mprintf_stronly(
 ** Call mprintf with a single double argument which is derived from the
 ** hexadecimal encoding of an IEEE double.
 */
-static int sqlite3_mprintf_hexdouble(
+static int SQLITE_TCLAPI sqlite3_mprintf_hexdouble(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1499,7 +1503,7 @@ static int sqlite3_mprintf_hexdouble(
 **
 */
 #if !defined(SQLITE_OMIT_SHARED_CACHE)
-static int test_enable_shared(
+static int SQLITE_TCLAPI test_enable_shared(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1536,7 +1540,7 @@ static int test_enable_shared(
 ** Usage: sqlite3_extended_result_codes   DB    BOOLEAN
 **
 */
-static int test_extended_result_codes(
+static int SQLITE_TCLAPI test_extended_result_codes(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1559,7 +1563,7 @@ static int test_extended_result_codes(
 ** Usage: sqlite3_libversion_number
 **
 */
-static int test_libversion_number(
+static int SQLITE_TCLAPI test_libversion_number(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1573,7 +1577,7 @@ static int test_libversion_number(
 ** Usage: sqlite3_table_column_metadata DB dbname tblname colname
 **
 */
-static int test_table_column_metadata(
+static int SQLITE_TCLAPI test_table_column_metadata(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1624,7 +1628,7 @@ static int test_table_column_metadata(
 
 #ifndef SQLITE_OMIT_INCRBLOB
 
-static int blobHandleFromObj(
+static int SQLITE_TCLAPI blobHandleFromObj(
   Tcl_Interp *interp, 
   Tcl_Obj *pObj,
   sqlite3_blob **ppBlob
@@ -1653,7 +1657,7 @@ static int blobHandleFromObj(
   return TCL_OK;
 }
 
-static int test_blob_reopen(
+static int SQLITE_TCLAPI test_blob_reopen(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1729,7 +1733,7 @@ static int testCreateCollationCmp(
 
   return iRes;
 }
-static int test_create_collation_v2(
+static int SQLITE_TCLAPI test_create_collation_v2(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1804,7 +1808,7 @@ static void cf2Destroy(void *pUser){
   if( p->pDestroy ) Tcl_DecrRefCount(p->pDestroy); 
   sqlite3_free(p);
 }
-static int test_create_function_v2(
+static int SQLITE_TCLAPI test_create_function_v2(
   ClientData clientData,          /* Not used */
   Tcl_Interp *interp,             /* The invoking TCL interpreter */
   int objc,                       /* Number of arguments */
@@ -1892,7 +1896,7 @@ static int test_create_function_v2(
 /*
 ** Usage: sqlite3_load_extension DB-HANDLE FILE ?PROC?
 */
-static int test_load_extension(
+static int SQLITE_TCLAPI test_load_extension(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1931,6 +1935,8 @@ static int test_load_extension(
 #ifdef SQLITE_OMIT_LOAD_EXTENSION
   rc = SQLITE_ERROR;
   zErr = sqlite3_mprintf("this build omits sqlite3_load_extension()");
+  (void)zProc;
+  (void)zFile;
 #else
   rc = sqlite3_load_extension(db, zFile, zProc, &zErr);
 #endif
@@ -1948,7 +1954,7 @@ static int test_load_extension(
 /*
 ** Usage: sqlite3_enable_load_extension DB-HANDLE ONOFF
 */
-static int test_enable_load(
+static int SQLITE_TCLAPI test_enable_load(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1994,7 +2000,7 @@ static int test_enable_load(
 ** This command is used to test the recoverability of a database in
 ** the event of a program crash.
 */
-static int sqlite_abort(
+static int SQLITE_TCLAPI sqlite_abort(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -2053,7 +2059,7 @@ error_out:
 **
 ** Register the test SQL function on the database DB under the name NAME.
 */
-static int test_register_func(
+static int SQLITE_TCLAPI test_register_func(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -2082,7 +2088,7 @@ static int test_register_func(
 **
 ** Finalize a statement handle.
 */
-static int test_finalize(
+static int SQLITE_TCLAPI test_finalize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2114,7 +2120,7 @@ static int test_finalize(
 **
 ** Get the value of a status counter from a statement.
 */
-static int test_stmt_status(
+static int SQLITE_TCLAPI test_stmt_status(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2159,7 +2165,7 @@ static int test_stmt_status(
 /*
 ** Usage:  sqlite3_stmt_scanstatus STMT IDX
 */
-static int test_stmt_scanstatus(
+static int SQLITE_TCLAPI test_stmt_scanstatus(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2209,7 +2215,7 @@ static int test_stmt_scanstatus(
 /*
 ** Usage:  sqlite3_stmt_scanstatus_reset  STMT
 */
-static int test_stmt_scanstatus_reset(
+static int SQLITE_TCLAPI test_stmt_scanstatus_reset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2232,7 +2238,7 @@ static int test_stmt_scanstatus_reset(
 **
 ** Zero the SQLITE_CONFIG_SQLLOG configuration
 */
-static int test_config_sqllog(
+static int SQLITE_TCLAPI test_config_sqllog(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2252,7 +2258,7 @@ static int test_config_sqllog(
 **
 ** Return the value returned by the default VFS's xCurrentTimeInt64 method.
 */
-static int vfsCurrentTimeInt64(
+static int SQLITE_TCLAPI vfsCurrentTimeInt64(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2273,7 +2279,7 @@ static int vfsCurrentTimeInt64(
 /*
 ** Usage: sqlite3_snapshot_get DB DBNAME
 */
-static int test_snapshot_get(
+static int SQLITE_TCLAPI test_snapshot_get(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2308,7 +2314,7 @@ static int test_snapshot_get(
 /*
 ** Usage: sqlite3_snapshot_open DB DBNAME SNAPSHOT
 */
-static int test_snapshot_open(
+static int SQLITE_TCLAPI test_snapshot_open(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2340,7 +2346,7 @@ static int test_snapshot_open(
 /*
 ** Usage: sqlite3_snapshot_free SNAPSHOT
 */
-static int test_snapshot_free(
+static int SQLITE_TCLAPI test_snapshot_free(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2357,12 +2363,60 @@ static int test_snapshot_free(
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** Usage: sqlite3_snapshot_cmp SNAPSHOT1 SNAPSHOT2
+*/
+static int SQLITE_TCLAPI test_snapshot_cmp(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int res;
+  sqlite3_snapshot *p1;
+  sqlite3_snapshot *p2;
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SNAPSHOT1 SNAPSHOT2");
+    return TCL_ERROR;
+  }
+  p1 = (sqlite3_snapshot*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
+  p2 = (sqlite3_snapshot*)sqlite3TestTextToPtr(Tcl_GetString(objv[2]));
+  res = sqlite3_snapshot_cmp(p1, p2);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(res));
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
+/*
+** Usage: sqlite3_delete_database FILENAME
+*/
+int sqlite3_delete_database(const char*);   /* in test_delete.c */
+static int SQLITE_TCLAPI test_delete_database(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  const char *zFile;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "FILE");
+    return TCL_ERROR;
+  }
+  zFile = (const char*)Tcl_GetString(objv[1]);
+  rc = sqlite3_delete_database(zFile);
+
+  Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_next_stmt  DB  STMT
 **
 ** Return the next statment in sequence after STMT.
 */
-static int test_next_stmt(
+static int SQLITE_TCLAPI test_next_stmt(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2394,7 +2448,7 @@ static int test_next_stmt(
 ** Return true if STMT is a NULL pointer or a pointer to a statement
 ** that is guaranteed to leave the database unmodified.
 */
-static int test_stmt_readonly(
+static int SQLITE_TCLAPI test_stmt_readonly(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2421,7 +2475,7 @@ static int test_stmt_readonly(
 ** Return true if STMT is a non-NULL pointer to a statement
 ** that has been stepped but not to completion.
 */
-static int test_stmt_busy(
+static int SQLITE_TCLAPI test_stmt_busy(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2447,7 +2501,7 @@ static int test_stmt_busy(
 **
 ** Return true if STMT uses a statement journal.
 */
-static int uses_stmt_journal(
+static int SQLITE_TCLAPI uses_stmt_journal(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2473,7 +2527,7 @@ static int uses_stmt_journal(
 **
 ** Reset a statement handle.
 */
-static int test_reset(
+static int SQLITE_TCLAPI test_reset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2508,7 +2562,7 @@ static int test_reset(
 **
 ** Return TRUE if a recompilation of the statement is recommended.
 */
-static int test_expired(
+static int SQLITE_TCLAPI test_expired(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2532,7 +2586,7 @@ static int test_expired(
 **
 ** Transfer all bindings from FROMSTMT over to TOSTMT
 */
-static int test_transfer_bind(
+static int SQLITE_TCLAPI test_transfer_bind(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2559,7 +2613,7 @@ static int test_transfer_bind(
 ** Return the number of changes made to the database by the last SQL
 ** execution.
 */
-static int test_changes(
+static int SQLITE_TCLAPI test_changes(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2594,7 +2648,7 @@ static int sqlite_static_bind_nbyte = 0;
 ** of the VALUE is made.  If FLAGS=="blob10" then a VALUE is ignored
 ** an a 10-byte blob "abc\000xyz\000pq" is inserted.
 */
-static int test_bind(
+static int SQLITE_TCLAPI test_bind(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -2716,7 +2770,7 @@ static int test_collate_func(
   Tcl_GetIntFromObj(i, Tcl_GetObjResult(i), &res);
   return res;
 }
-static int test_collate(
+static int SQLITE_TCLAPI test_collate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2790,7 +2844,7 @@ static int test_utf16bin_collate_func(
   if( res==0 ) res = nA - nB;
   return res;
 }
-static int test_utf16bin_collate(
+static int SQLITE_TCLAPI test_utf16bin_collate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2847,7 +2901,7 @@ static void test_collate_needed_cb(
 /*
 ** Usage: add_test_collate_needed DB
 */
-static int test_collate_needed(
+static int SQLITE_TCLAPI test_collate_needed(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2898,7 +2952,7 @@ static int alignmentCollFunc(
   }
   return rc;
 }
-static int add_alignment_test_collations(
+static int SQLITE_TCLAPI add_alignment_test_collations(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3016,7 +3070,7 @@ static void test_function_utf16be(
   sqlite3ValueFree(pVal);
 }
 #endif /* SQLITE_OMIT_UTF16 */
-static int test_function(
+static int SQLITE_TCLAPI test_function(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3061,7 +3115,7 @@ bad_args:
 ** The result is a list of two elements, the string representation of the
 ** error code and the english language explanation.
 */
-static int test_errstr(
+static int SQLITE_TCLAPI test_errstr(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3094,7 +3148,7 @@ static int test_errstr(
 ** Then run testfixture in the debugger and wait for the breakpoint to
 ** fire.  Then additional breakpoints can be set to trace down the bug.
 */
-static int test_breakpoint(
+static int SQLITE_TCLAPI test_breakpoint(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -3110,7 +3164,7 @@ static int test_breakpoint(
 ** IDX is the index of a wildcard in the prepared statement.  This command
 ** binds a N-byte zero-filled BLOB to the wildcard.
 */
-static int test_bind_zeroblob(
+static int SQLITE_TCLAPI test_bind_zeroblob(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3146,7 +3200,7 @@ static int test_bind_zeroblob(
 ** IDX is the index of a wildcard in the prepared statement.  This command
 ** binds a N-byte zero-filled BLOB to the wildcard.
 */
-static int test_bind_zeroblob64(
+static int SQLITE_TCLAPI test_bind_zeroblob64(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3154,7 +3208,7 @@ static int test_bind_zeroblob64(
 ){
   sqlite3_stmt *pStmt;
   int idx;
-  i64 n;
+  Tcl_WideInt n;
   int rc;
 
   if( objc!=4 ){
@@ -3183,7 +3237,7 @@ static int test_bind_zeroblob64(
 ** N is the index of a wildcard in the prepared statement.  This command
 ** binds a 32-bit integer VALUE to that wildcard.
 */
-static int test_bind_int(
+static int SQLITE_TCLAPI test_bind_int(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3214,6 +3268,145 @@ static int test_bind_int(
 }
 
 
+/*
+** Usage:   intarray_addr  INT  ...
+**
+** Return the address of a C-language array of 32-bit integers.
+**
+** Space to hold the array is obtained from malloc().  Call this procedure once
+** with no arguments in order to release memory.  Each call to this procedure
+** overwrites the previous array.
+*/
+static int SQLITE_TCLAPI test_intarray_addr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int i;
+  static int *p = 0;
+
+  sqlite3_free(p);
+  p = 0;
+  if( objc>1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; ilen ){
+    char zBuf[200];
+    sqlite3_snprintf(sizeof(zBuf), zBuf,
+                     "cannot use %d blob bytes, have %d", bytes, len);
+    Tcl_AppendResult(interp, zBuf, -1);
+    return TCL_ERROR;
+  }
+
   rc = sqlite3_bind_blob(pStmt, idx, value, bytes, xDestructor);
   if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR;
   if( rc!=SQLITE_OK ){
@@ -3503,7 +3705,7 @@ static int test_bind_blob(
 **
 ** Return the number of wildcards in the given statement.
 */
-static int test_bind_parameter_count(
+static int SQLITE_TCLAPI test_bind_parameter_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3527,7 +3729,7 @@ static int test_bind_parameter_count(
 ** An empty string is returned if N is out of range or if the wildcard
 ** is nameless.
 */
-static int test_bind_parameter_name(
+static int SQLITE_TCLAPI test_bind_parameter_name(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3554,7 +3756,7 @@ static int test_bind_parameter_name(
 ** Return the index of the wildcard called NAME.  Return 0 if there is
 ** no such wildcard.
 */
-static int test_bind_parameter_index(
+static int SQLITE_TCLAPI test_bind_parameter_index(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3579,7 +3781,7 @@ static int test_bind_parameter_index(
 ** Usage:   sqlite3_clear_bindings STMT
 **
 */
-static int test_clear_bindings(
+static int SQLITE_TCLAPI test_clear_bindings(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3599,7 +3801,7 @@ static int test_clear_bindings(
 /*
 ** Usage:   sqlite3_sleep MILLISECONDS
 */
-static int test_sleep(
+static int SQLITE_TCLAPI test_sleep(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3624,7 +3826,7 @@ static int test_sleep(
 ** Return the string representation of the most recent sqlite3_* API
 ** error code. e.g. "SQLITE_ERROR".
 */
-static int test_ex_errcode(
+static int SQLITE_TCLAPI test_ex_errcode(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3651,7 +3853,7 @@ static int test_ex_errcode(
 ** Return the string representation of the most recent sqlite3_* API
 ** error code. e.g. "SQLITE_ERROR".
 */
-static int test_errcode(
+static int SQLITE_TCLAPI test_errcode(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3677,7 +3879,7 @@ static int test_errcode(
 ** Returns the UTF-8 representation of the error message string for the
 ** most recent sqlite3_* API call.
 */
-static int test_errmsg(
+static int SQLITE_TCLAPI test_errmsg(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3706,7 +3908,7 @@ static int test_errmsg(
 ** level, and it includes the 0x00 0x00 terminator bytes at the end of the
 ** UTF-16 string.
 */
-static int test_errmsg16(
+static int SQLITE_TCLAPI test_errmsg16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3743,7 +3945,7 @@ static int test_errmsg16(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare(
+static int SQLITE_TCLAPI test_prepare(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3800,7 +4002,7 @@ static int test_prepare(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare_v2(
+static int SQLITE_TCLAPI test_prepare_v2(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3868,7 +4070,7 @@ static int test_prepare_v2(
 ** Generate a prepared statement for a zero-byte string as a test
 ** for ticket #3134.  The string should be preceded by a zero byte.
 */
-static int test_prepare_tkt3134(
+static int SQLITE_TCLAPI test_prepare_tkt3134(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3911,7 +4113,7 @@ static int test_prepare_tkt3134(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare16(
+static int SQLITE_TCLAPI test_prepare16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3971,7 +4173,7 @@ static int test_prepare16(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare16_v2(
+static int SQLITE_TCLAPI test_prepare16_v2(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4026,7 +4228,7 @@ static int test_prepare16_v2(
 /*
 ** Usage: sqlite3_open filename ?options-list?
 */
-static int test_open(
+static int SQLITE_TCLAPI test_open(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4053,7 +4255,7 @@ static int test_open(
 /*
 ** Usage: sqlite3_open_v2 FILENAME FLAGS VFS
 */
-static int test_open_v2(
+static int SQLITE_TCLAPI test_open_v2(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4123,7 +4325,7 @@ static int test_open_v2(
 /*
 ** Usage: sqlite3_open16 filename options
 */
-static int test_open16(
+static int SQLITE_TCLAPI test_open16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4155,7 +4357,7 @@ static int test_open16(
 ** Return 1 if the supplied argument is a complete SQL statement, or zero
 ** otherwise.
 */
-static int test_complete16(
+static int SQLITE_TCLAPI test_complete16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4180,7 +4382,7 @@ static int test_complete16(
 **
 ** Advance the statement to the next row.
 */
-static int test_step(
+static int SQLITE_TCLAPI test_step(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4203,7 +4405,7 @@ static int test_step(
   return TCL_OK;
 }
 
-static int test_sql(
+static int SQLITE_TCLAPI test_sql(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4220,13 +4422,33 @@ static int test_sql(
   Tcl_SetResult(interp, (char *)sqlite3_sql(pStmt), TCL_VOLATILE);
   return TCL_OK;
 }
+static int SQLITE_TCLAPI test_ex_sql(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+  char *z;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "STMT");
+    return TCL_ERROR;
+  }
+
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  z = sqlite3_expanded_sql(pStmt);
+  Tcl_SetResult(interp, z, TCL_VOLATILE);
+  sqlite3_free(z);
+  return TCL_OK;
+}
 
 /*
 ** Usage: sqlite3_column_count STMT 
 **
 ** Return the number of columns returned by the sql statement STMT.
 */
-static int test_column_count(
+static int SQLITE_TCLAPI test_column_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4251,7 +4473,7 @@ static int test_column_count(
 **
 ** Return the type of the data in column 'column' of the current row.
 */
-static int test_column_type(
+static int SQLITE_TCLAPI test_column_type(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4300,7 +4522,7 @@ static int test_column_type(
 ** Return the data in column 'column' of the current row cast as an
 ** wide (64-bit) integer.
 */
-static int test_column_int64(
+static int SQLITE_TCLAPI test_column_int64(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4327,7 +4549,7 @@ static int test_column_int64(
 /*
 ** Usage: sqlite3_column_blob STMT column
 */
-static int test_column_blob(
+static int SQLITE_TCLAPI test_column_blob(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4359,7 +4581,7 @@ static int test_column_blob(
 **
 ** Return the data in column 'column' of the current row cast as a double.
 */
-static int test_column_double(
+static int SQLITE_TCLAPI test_column_double(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4388,7 +4610,7 @@ static int test_column_double(
 **
 ** Return the number of columns returned by the sql statement STMT.
 */
-static int test_data_count(
+static int SQLITE_TCLAPI test_data_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4415,7 +4637,7 @@ static int test_data_count(
 **
 ** Usage: sqlite3_column_name STMT column
 */
-static int test_stmt_utf8(
+static int SQLITE_TCLAPI test_stmt_utf8(
   void * clientData,        /* Pointer to SQLite API function to be invoke */
   Tcl_Interp *interp,
   int objc,
@@ -4442,7 +4664,7 @@ static int test_stmt_utf8(
   return TCL_OK;
 }
 
-static int test_global_recover(
+static int SQLITE_TCLAPI test_global_recover(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4467,7 +4689,7 @@ static int test_global_recover(
 **
 ** Usage: sqlite3_column_name STMT column
 */
-static int test_stmt_utf16(
+static int SQLITE_TCLAPI test_stmt_utf16(
   void * clientData,     /* Pointer to SQLite API function to be invoked */
   Tcl_Interp *interp,
   int objc,
@@ -4511,7 +4733,7 @@ static int test_stmt_utf16(
 ** Usage: sqlite3_column_bytes16 STMT column
 **
 */
-static int test_stmt_int(
+static int SQLITE_TCLAPI test_stmt_int(
   void * clientData,    /* Pointer to SQLite API function to be invoked */
   Tcl_Interp *interp,
   int objc,
@@ -4540,7 +4762,7 @@ static int test_stmt_int(
 **
 ** Set the db->magic value.  This is used to test error recovery logic.
 */
-static int sqlite_set_magic(
+static int SQLITE_TCLAPI sqlite_set_magic(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4572,7 +4794,7 @@ static int sqlite_set_magic(
 **
 ** Trigger an interrupt on DB
 */
-static int test_interrupt(
+static int SQLITE_TCLAPI test_interrupt(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4613,7 +4835,7 @@ u64 sqlite3StackDepth(void){
 **
 ** Try to measure the amount of stack space used by a call to sqlite3_exec
 */
-static int test_stack_used(
+static int SQLITE_TCLAPI test_stack_used(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4641,7 +4863,7 @@ static int test_stack_used(
 ** is assumed that the user function was created as UTF8, any number of
 ** arguments (the way the TCL interface does it).
 */
-static int delete_function(
+static int SQLITE_TCLAPI delete_function(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4667,7 +4889,7 @@ static int delete_function(
 ** DB. It is assumed that the collation sequence was created as UTF8 (the 
 ** way the TCL interface does it).
 */
-static int delete_collation(
+static int SQLITE_TCLAPI delete_collation(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4692,7 +4914,7 @@ static int delete_collation(
 ** Return true if the database DB is currently in auto-commit mode.
 ** Return false if not.
 */
-static int get_autocommit(
+static int SQLITE_TCLAPI get_autocommit(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4718,7 +4940,7 @@ static int get_autocommit(
 ** method of the TCL interface.  But we need a way to test the case
 ** where it returns SQLITE_MISUSE.
 */
-static int test_busy_timeout(
+static int SQLITE_TCLAPI test_busy_timeout(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4744,7 +4966,7 @@ static int test_busy_timeout(
 ** Return the name of the internal representation for the
 ** value of the given variable.
 */
-static int tcl_variable_type(
+static int SQLITE_TCLAPI tcl_variable_type(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4770,7 +4992,7 @@ static int tcl_variable_type(
 ** The integer N is the number of bytes we are trying to release.  The 
 ** return value is the amount of memory actually released.
 */
-static int test_release_memory(
+static int SQLITE_TCLAPI test_release_memory(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4801,7 +5023,7 @@ static int test_release_memory(
 ** Attempt to release memory currently held by database DB.  Return the
 ** result code (which in the current implementation is always zero).
 */
-static int test_db_release_memory(
+static int SQLITE_TCLAPI test_db_release_memory(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4824,7 +5046,7 @@ static int test_db_release_memory(
 **
 ** Attempt to flush any dirty pages to disk.
 */
-static int test_db_cacheflush(
+static int SQLITE_TCLAPI test_db_cacheflush(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4847,12 +5069,35 @@ static int test_db_cacheflush(
   return TCL_OK;
 }
 
+/*
+** Usage:  sqlite3_system_errno DB
+**
+** Return the low-level system errno value.
+*/
+static int SQLITE_TCLAPI test_system_errno(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  int iErrno;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  iErrno = sqlite3_system_errno(db);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(iErrno));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_db_filename DB DBNAME
 **
 ** Return the name of a file associated with a database.
 */
-static int test_db_filename(
+static int SQLITE_TCLAPI test_db_filename(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4876,7 +5121,7 @@ static int test_db_filename(
 ** Return 1 or 0 if DBNAME is readonly or not.  Return -1 if DBNAME does
 ** not exist.
 */
-static int test_db_readonly(
+static int SQLITE_TCLAPI test_db_readonly(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4901,7 +5146,7 @@ static int test_db_readonly(
 ** limit is only changed if the N is present.  The previous limit
 ** is returned.
 */
-static int test_soft_heap_limit(
+static int SQLITE_TCLAPI test_soft_heap_limit(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4926,7 +5171,7 @@ static int test_soft_heap_limit(
 **
 ** Call the sqlite3_thread_cleanup API.
 */
-static int test_thread_cleanup(
+static int SQLITE_TCLAPI test_thread_cleanup(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4944,7 +5189,7 @@ static int test_thread_cleanup(
 ** Return a list of numbers which are the PagerRefcount for all
 ** pagers on each database connection.
 */
-static int test_pager_refcounts(
+static int SQLITE_TCLAPI test_pager_refcounts(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4990,7 +5235,7 @@ static int test_pager_refcounts(
 ** and that the errors they are seeing in the test scripts might be
 ** a result of their defective TCL rather than problems in SQLite.
 */
-static int working_64bit_int(
+static int SQLITE_TCLAPI working_64bit_int(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5015,7 +5260,7 @@ static int working_64bit_int(
 ** VFS when none are previously registered, and the ability to 
 ** unregister the only available VFS.  Ticket #2738
 */
-static int vfs_unlink_test(
+static int SQLITE_TCLAPI vfs_unlink_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5117,7 +5362,7 @@ static int vfs_unlink_test(
 ** This TCL command attempts to vfs_find and vfs_register when the
 ** sqlite3_initialize() interface is failing.  All calls should fail.
 */
-static int vfs_initfail_test(
+static int SQLITE_TCLAPI vfs_initfail_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5145,7 +5390,7 @@ static int nVfs = 0;
 **
 ** Unregister all VFSes.
 */
-static int vfs_unregister_all(
+static int SQLITE_TCLAPI vfs_unregister_all(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5163,17 +5408,19 @@ static int vfs_unregister_all(
 /*
 ** tclcmd:   vfs_reregister_all
 **
-** Restore all VFSes that were removed using vfs_unregister_all
+** Restore all VFSes that were removed using vfs_unregister_all. Taking
+** care to put the linked list back together in the same order as it was
+** in before vfs_unregister_all was invoked.
 */
-static int vfs_reregister_all(
+static int SQLITE_TCLAPI vfs_reregister_all(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
   Tcl_Obj *CONST objv[]  /* Command arguments */
 ){
   int i;
-  for(i=0; i=0; i--){
+    sqlite3_vfs_register(apVfs[i], 1);
   }
   return TCL_OK;
 }
@@ -5185,7 +5432,7 @@ static int vfs_reregister_all(
 ** This TCL command runs the sqlite3_file_control interface and
 ** verifies correct operation of the same.
 */
-static int file_control_test(
+static int SQLITE_TCLAPI file_control_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5220,7 +5467,7 @@ static int file_control_test(
 ** This TCL command runs the sqlite3_file_control interface and
 ** verifies correct operation of the SQLITE_LAST_ERRNO verb.
 */
-static int file_control_lasterrno_test(
+static int SQLITE_TCLAPI file_control_lasterrno_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5258,7 +5505,7 @@ static int file_control_lasterrno_test(
 ** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and
 ** SQLITE_SET_LOCKPROXYFILE verbs.
 */
-static int file_control_chunksize_test(
+static int SQLITE_TCLAPI file_control_chunksize_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5295,7 +5542,7 @@ static int file_control_chunksize_test(
 ** This TCL command runs the sqlite3_file_control interface 
 ** with SQLITE_FCNTL_SIZE_HINT
 */
-static int file_control_sizehint_test(
+static int SQLITE_TCLAPI file_control_sizehint_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5333,7 +5580,7 @@ static int file_control_sizehint_test(
 ** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and
 ** SQLITE_SET_LOCKPROXYFILE verbs.
 */
-static int file_control_lockproxy_test(
+static int SQLITE_TCLAPI file_control_lockproxy_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5403,7 +5650,7 @@ static int file_control_lockproxy_test(
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_WIN32_AV_RETRY opcode.
 */
-static int file_control_win32_av_retry(
+static int SQLITE_TCLAPI file_control_win32_av_retry(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5430,13 +5677,45 @@ static int file_control_win32_av_retry(
   return TCL_OK;  
 }
 
+/*
+** tclcmd:   file_control_win32_get_handle DB
+**
+** This TCL command runs the sqlite3_file_control interface with
+** the SQLITE_FCNTL_WIN32_GET_HANDLE opcode.
+*/
+static int file_control_win32_get_handle(
+  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int objc,              /* Number of arguments */
+  Tcl_Obj *CONST objv[]  /* Command arguments */
+){
+  sqlite3 *db;
+  int rc;
+  HANDLE hFile = NULL;
+  char z[100];
+
+  if( objc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"",
+        Tcl_GetStringFromObj(objv[0], 0), " DB", 0);
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
+    return TCL_ERROR;
+  }
+  rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_GET_HANDLE,
+                            (void*)&hFile);
+  sqlite3_snprintf(sizeof(z), z, "%d %p", rc, (void*)hFile);
+  Tcl_AppendResult(interp, z, (char*)0);
+  return TCL_OK;
+}
+
 /*
 ** tclcmd:   file_control_win32_set_handle DB HANDLE
 **
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_WIN32_SET_HANDLE opcode.
 */
-static int file_control_win32_set_handle(
+static int SQLITE_TCLAPI file_control_win32_set_handle(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5472,7 +5751,7 @@ static int file_control_win32_set_handle(
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_PERSIST_WAL opcode.
 */
-static int file_control_persist_wal(
+static int SQLITE_TCLAPI file_control_persist_wal(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5504,7 +5783,7 @@ static int file_control_persist_wal(
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_POWERSAFE_OVERWRITE opcode.
 */
-static int file_control_powersafe_overwrite(
+static int SQLITE_TCLAPI file_control_powersafe_overwrite(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5536,7 +5815,7 @@ static int file_control_powersafe_overwrite(
 **
 ** Return a string that describes the stack of VFSes.
 */
-static int file_control_vfsname(
+static int SQLITE_TCLAPI file_control_vfsname(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5568,7 +5847,7 @@ static int file_control_vfsname(
 **
 ** Return a string that is a temporary filename
 */
-static int file_control_tempfilename(
+static int SQLITE_TCLAPI file_control_tempfilename(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5601,7 +5880,7 @@ static int file_control_tempfilename(
 **
 **   Return a tcl list containing the names of all registered vfs's.
 */
-static int vfs_list(
+static int SQLITE_TCLAPI vfs_list(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5626,7 +5905,7 @@ static int vfs_list(
 ** This TCL command runs the sqlite3_limit interface and
 ** verifies correct operation of the same.
 */
-static int test_limit(
+static int SQLITE_TCLAPI test_limit(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5689,7 +5968,7 @@ static int test_limit(
 ** At the same time, verify that sqlite3_test_control works even when
 ** called with an out-of-range opcode.
 */
-static int save_prng_state(
+static int SQLITE_TCLAPI save_prng_state(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5705,7 +5984,7 @@ static int save_prng_state(
 /*
 ** tclcmd:  restore_prng_state
 */
-static int restore_prng_state(
+static int SQLITE_TCLAPI restore_prng_state(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5717,7 +5996,7 @@ static int restore_prng_state(
 /*
 ** tclcmd:  reset_prng_state
 */
-static int reset_prng_state(
+static int SQLITE_TCLAPI reset_prng_state(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5733,7 +6012,7 @@ static int reset_prng_state(
 ** Indicate that database files might be corrupt.  In other words, set the normal
 ** state of operation.
 */
-static int database_may_be_corrupt(
+static int SQLITE_TCLAPI database_may_be_corrupt(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5748,7 +6027,7 @@ static int database_may_be_corrupt(
 ** Indicate that database files are always well-formed.  This enables extra assert()
 ** statements that test conditions that are always true for well-formed databases.
 */
-static int database_never_corrupt(
+static int SQLITE_TCLAPI database_never_corrupt(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5761,7 +6040,7 @@ static int database_never_corrupt(
 /*
 ** tclcmd:  pcache_stats
 */
-static int test_pcache_stats(
+static int SQLITE_TCLAPI test_pcache_stats(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5803,7 +6082,7 @@ static void test_unlock_notify_cb(void **aArg, int nArg){
 ** tclcmd:  sqlite3_unlock_notify db
 */
 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
-static int test_unlock_notify(
+static int SQLITE_TCLAPI test_unlock_notify(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5829,7 +6108,7 @@ static int test_unlock_notify(
 /*
 ** tclcmd:  sqlite3_wal_checkpoint db ?NAME?
 */
-static int test_wal_checkpoint(
+static int SQLITE_TCLAPI test_wal_checkpoint(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5873,7 +6152,7 @@ static int test_wal_checkpoint(
 ** the number of frames in the log and the number of frames in the log
 ** that have been checkpointed.
 */
-static int test_wal_checkpoint_v2(
+static int SQLITE_TCLAPI test_wal_checkpoint_v2(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5929,7 +6208,7 @@ static int test_wal_checkpoint_v2(
 /*
 ** tclcmd:  sqlite3_wal_autocheckpoint db VALUE
 */
-static int test_wal_autocheckpoint(
+static int SQLITE_TCLAPI test_wal_autocheckpoint(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5980,7 +6259,7 @@ static void xLogcallback(void *unused, int err, char *zMsg){
   Tcl_EvalObjEx(logcallback.pInterp, pNew, TCL_EVAL_GLOBAL|TCL_EVAL_DIRECT);
   Tcl_DecrRefCount(pNew);
 }
-static int test_sqlite3_log(
+static int SQLITE_TCLAPI test_sqlite3_log(
   ClientData clientData,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6011,7 +6290,7 @@ static int test_sqlite3_log(
 ** Run a TCL command using its objProc interface.  Throw an error if
 ** the command has no objProc interface.
 */
-static int runAsObjProc(
+static int SQLITE_TCLAPI runAsObjProc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6074,7 +6353,7 @@ int printExplainQueryPlan(sqlite3_stmt *pStmt){
   return sqlite3_finalize(pExplain);
 }
 
-static int test_print_eqp(
+static int SQLITE_TCLAPI test_print_eqp(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6102,7 +6381,7 @@ static int test_print_eqp(
 /*
 ** sqlite3_test_control VERB ARGS...
 */
-static int test_test_control(
+static int SQLITE_TCLAPI test_test_control(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6181,7 +6460,7 @@ static int test_test_control(
 #include 
 #include 
 
-static int test_getrusage(
+static int SQLITE_TCLAPI test_getrusage(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6224,7 +6503,7 @@ struct win32FileLocker {
 /*
 ** The background thread that does file locking.
 */
-static void win32_file_locker(void *pAppData){
+static void SQLITE_CDECL win32_file_locker(void *pAppData){
   struct win32FileLocker *p = (struct win32FileLocker*)pAppData;
   if( p->evName ){
     HANDLE ev = OpenEvent(EVENT_MODIFY_STATE, FALSE, p->evName);
@@ -6255,7 +6534,7 @@ static void win32_file_locker(void *pAppData){
 ** Get an exclusive manditory lock on file for DELAY2 milliseconds.
 ** Wait DELAY1 milliseconds before acquiring the lock.
 */
-static int win32_file_lock(
+static int SQLITE_TCLAPI win32_file_lock(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6319,7 +6598,7 @@ static int win32_file_lock(
 ** Returns non-zero if the specified path exists, whose fully qualified name
 ** may exceed 260 characters if it is prefixed with "\\?\".
 */
-static int win32_exists_path(
+static int SQLITE_TCLAPI win32_exists_path(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6341,7 +6620,7 @@ static int win32_exists_path(
 ** whose fully qualified name may exceed 248 characters if it is prefixed with
 ** "\\?\".
 */
-static int win32_find_file(
+static int SQLITE_TCLAPI win32_find_file(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6386,7 +6665,7 @@ static int win32_find_file(
 ** Deletes the specified file, whose fully qualified name may exceed 260
 ** characters if it is prefixed with "\\?\".
 */
-static int win32_delete_file(
+static int SQLITE_TCLAPI win32_delete_file(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6410,7 +6689,7 @@ static int win32_delete_file(
 ** Creates the specified directory, whose fully qualified name may exceed 248
 ** characters if it is prefixed with "\\?\".
 */
-static int win32_mkdir(
+static int SQLITE_TCLAPI win32_mkdir(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6434,7 +6713,7 @@ static int win32_mkdir(
 ** Removes the specified directory, whose fully qualified name may exceed 248
 ** characters if it is prefixed with "\\?\".
 */
-static int win32_rmdir(
+static int SQLITE_TCLAPI win32_rmdir(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6461,7 +6740,7 @@ static int win32_rmdir(
 ** interface.  Disable if BOOLEAN is false and enable if BOOLEAN is true.
 ** OPT is the name of the optimization to be disabled.
 */
-static int optimization_control(
+static int SQLITE_TCLAPI optimization_control(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6518,20 +6797,21 @@ static int optimization_control(
   return TCL_OK;
 }
 
-typedef struct sqlite3_api_routines sqlite3_api_routines;
 /*
 **     load_static_extension DB NAME ...
 **
 ** Load one or more statically linked extensions.
 */
-static int tclLoadStaticExtensionCmd(
+static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
   extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_carray_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_csv_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6548,7 +6828,9 @@ static int tclLoadStaticExtensionCmd(
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
   } aExtension[] = {
     { "amatch",                sqlite3_amatch_init               },
+    { "carray",                sqlite3_carray_init               },
     { "closure",               sqlite3_closure_init              },
+    { "csv",                   sqlite3_csv_init                  },
     { "eval",                  sqlite3_eval_init                 },
     { "fileio",                sqlite3_fileio_init               },
     { "fuzzer",                sqlite3_fuzzer_init               },
@@ -6598,7 +6880,7 @@ static int tclLoadStaticExtensionCmd(
 **     sorter_test_fakeheap BOOL
 **
 */
-static int sorter_test_fakeheap(
+static int SQLITE_TCLAPI sorter_test_fakeheap(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6638,7 +6920,7 @@ static int sorter_test_fakeheap(
 ** Then execute statement $SQL2. Check that the statement returns the same
 ** set of integers in the same order as in the previous step (using $SQL1).
 */
-static int sorter_test_sort4_helper(
+static int SQLITE_TCLAPI sorter_test_sort4_helper(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6648,8 +6930,8 @@ static int sorter_test_sort4_helper(
   const char *zSql2;
   int nStep; 
   int iStep; 
-  int iCksum1 = 0; 
-  int iCksum2 = 0; 
+  unsigned int iCksum1 = 0; 
+  unsigned int iCksum2 = 0; 
   int rc;
   int iB;
   sqlite3 *db;
@@ -6676,7 +6958,7 @@ static int sorter_test_sort4_helper(
       return TCL_ERROR;
     }
 
-    iCksum1 += (iCksum1 << 3) + a;
+    iCksum1 += (iCksum1 << 3) + (unsigned int)a;
   }
   rc = sqlite3_finalize(pStmt);
   if( rc!=SQLITE_OK ) goto sql_error;
@@ -6685,7 +6967,7 @@ static int sorter_test_sort4_helper(
   if( rc!=SQLITE_OK ) goto sql_error;
   for(iStep=0; SQLITE_ROW==sqlite3_step(pStmt); iStep++){
     int a = sqlite3_column_int(pStmt, 0);
-    iCksum2 += (iCksum2 << 3) + a;
+    iCksum2 += (iCksum2 << 3) + (unsigned int)a;
   }
   rc = sqlite3_finalize(pStmt);
   if( rc!=SQLITE_OK ) goto sql_error;
@@ -6707,7 +6989,7 @@ static int sorter_test_sort4_helper(
 /*
 ** tclcmd:  sqlite3_user_authenticate DB USERNAME PASSWORD
 */
-static int test_user_authenticate(
+static int SQLITE_TCLAPI test_user_authenticate(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6738,7 +7020,7 @@ static int test_user_authenticate(
 /*
 ** tclcmd:  sqlite3_user_add DB USERNAME PASSWORD ISADMIN
 */
-static int test_user_add(
+static int SQLITE_TCLAPI test_user_add(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6771,7 +7053,7 @@ static int test_user_add(
 /*
 ** tclcmd:  sqlite3_user_change DB USERNAME PASSWORD ISADMIN
 */
-static int test_user_change(
+static int SQLITE_TCLAPI test_user_change(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6804,7 +7086,7 @@ static int test_user_change(
 /*
 ** tclcmd:  sqlite3_user_delete DB USERNAME
 */
-static int test_user_delete(
+static int SQLITE_TCLAPI test_user_delete(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6841,7 +7123,7 @@ static int test_user_delete(
 **       3          Read after free
 **       4          Panic
 */
-static int test_bad_behavior(
+static int SQLITE_TCLAPI test_bad_behavior(
   ClientData clientData, /* Pointer to an integer containing zero */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6891,7 +7173,7 @@ static int test_bad_behavior(
 **
 ** Cause the dbstat virtual table to be available on the connection DB
 */
-static int test_register_dbstat_vtab(
+static int SQLITE_TCLAPI test_register_dbstat_vtab(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6920,6 +7202,77 @@ static int test_register_dbstat_vtab(
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 }
 
+/*
+** tclcmd:   sqlite3_db_config DB SETTING VALUE
+**
+** Invoke sqlite3_db_config() for one of the setting values.
+*/
+static int SQLITE_TCLAPI test_sqlite3_db_config(
+  void *clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  static const struct {
+    const char *zName;
+    int eVal;
+  } aSetting[] = {
+    { "FKEY",            SQLITE_DBCONFIG_ENABLE_FKEY },
+    { "TRIGGER",         SQLITE_DBCONFIG_ENABLE_TRIGGER },
+    { "FTS3_TOKENIZER",  SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER },
+    { "LOAD_EXTENSION",  SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION },
+  };
+  int i;
+  int v;
+  const char *zSetting;
+  sqlite3 *db;
+
+  if( objc!=4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB SETTING VALUE");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zSetting = Tcl_GetString(objv[2]);
+  if( sqlite3_strglob("SQLITE_*", zSetting)==0 ) zSetting += 7;
+  if( sqlite3_strglob("DBCONFIG_*", zSetting)==0 ) zSetting += 9;
+  if( sqlite3_strglob("ENABLE_*", zSetting)==0 ) zSetting += 7;
+  for(i=0; i=ArraySize(aSetting) ){
+    Tcl_SetObjResult(interp,
+      Tcl_NewStringObj("unknown sqlite3_db_config setting", -1));
+    return TCL_ERROR;
+  }
+  if( Tcl_GetIntFromObj(interp, objv[3], &v) ) return TCL_ERROR;
+  sqlite3_db_config(db, aSetting[i].eVal, v, &v);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(v));
+  return TCL_OK;
+}
+
+/*
+** Change the name of the main database schema from "main" to "icecube".
+*/
+static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**);
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }else{
+    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_MAINDBNAME, "icecube");
+    Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
+    return TCL_OK;
+  }
+}
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -6934,7 +7287,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
   extern int sqlite3_hostid_num;
 #endif
   extern int sqlite3_max_blobsize;
-  extern int sqlite3BtreeSharedCacheReport(void*,
+  extern int SQLITE_TCLAPI sqlite3BtreeSharedCacheReport(void*,
                                           Tcl_Interp*,int,Tcl_Obj*CONST*);
   static int iZero = 0;
   static struct {
@@ -6989,9 +7342,14 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      Tcl_ObjCmdProc *xProc;
      void *clientData;
   } aObjCmd[] = {
+     { "sqlite3_db_config",             test_sqlite3_db_config, 0 },
      { "bad_behavior",                  test_bad_behavior,  (void*)&iZero },
      { "register_dbstat_vtab",          test_register_dbstat_vtab  },
      { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
+     { "intarray_addr",                 test_intarray_addr, 0 },
+     { "int64array_addr",               test_int64array_addr, 0 },
+     { "doublearray_addr",              test_doublearray_addr, 0 },
+     { "textarray_addr",                test_textarray_addr, 0 },
      { "sqlite3_bind_int",              test_bind_int,      0 },
      { "sqlite3_bind_zeroblob",         test_bind_zeroblob, 0 },
      { "sqlite3_bind_zeroblob64",       test_bind_zeroblob64, 0 },
@@ -7028,6 +7386,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_changes",               test_changes       ,0 },
      { "sqlite3_step",                  test_step          ,0 },
      { "sqlite3_sql",                   test_sql           ,0 },
+     { "sqlite3_expanded_sql",          test_ex_sql        ,0 },
      { "sqlite3_next_stmt",             test_next_stmt     ,0 },
      { "sqlite3_stmt_readonly",         test_stmt_readonly ,0 },
      { "sqlite3_stmt_busy",             test_stmt_busy     ,0 },
@@ -7036,6 +7395,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_release_memory",        test_release_memory,     0},
      { "sqlite3_db_release_memory",     test_db_release_memory,  0},
      { "sqlite3_db_cacheflush",         test_db_cacheflush,      0},
+     { "sqlite3_system_errno",          test_system_errno,       0},
      { "sqlite3_db_filename",           test_db_filename,        0},
      { "sqlite3_db_readonly",           test_db_readonly,        0},
      { "sqlite3_soft_heap_limit",       test_soft_heap_limit,    0},
@@ -7046,6 +7406,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_enable_load_extension", test_enable_load,        0},
      { "sqlite3_extended_result_codes", test_extended_result_codes, 0},
      { "sqlite3_limit",                 test_limit,                 0},
+     { "dbconfig_maindbname_icecube",   test_dbconfig_maindbname_icecube },
 
      { "save_prng_state",               save_prng_state,    0 },
      { "restore_prng_state",            restore_prng_state, 0 },
@@ -7112,6 +7473,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "file_control_sizehint_test",  file_control_sizehint_test,   0   },
 #if SQLITE_OS_WIN
      { "file_control_win32_av_retry", file_control_win32_av_retry,  0   },
+     { "file_control_win32_get_handle", file_control_win32_get_handle, 0  },
      { "file_control_win32_set_handle", file_control_win32_set_handle, 0  },
 #endif
      { "file_control_persist_wal",    file_control_persist_wal,     0   },
@@ -7175,7 +7537,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_snapshot_get", test_snapshot_get, 0 },
      { "sqlite3_snapshot_open", test_snapshot_open, 0 },
      { "sqlite3_snapshot_free", test_snapshot_free, 0 },
+     { "sqlite3_snapshot_cmp", test_snapshot_cmp, 0 },
 #endif
+     { "sqlite3_delete_database", test_delete_database, 0 },
   };
   static int bitmask_size = sizeof(Bitmask)*8;
   static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
diff --git a/src/test2.c b/src/test2.c
index d16ba501..dfca1d2b 100644
--- a/src/test2.c
+++ b/src/test2.c
@@ -14,7 +14,11 @@
 ** testing of the SQLite library.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -38,7 +42,7 @@ static void pager_test_reiniter(DbPage *pNotUsed){
 **
 ** Open a new pager
 */
-static int pager_open(
+static int SQLITE_TCLAPI pager_open(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -75,7 +79,7 @@ static int pager_open(
 **
 ** Close the given pager.
 */
-static int pager_close(
+static int SQLITE_TCLAPI pager_close(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -102,7 +106,7 @@ static int pager_close(
 **
 ** Rollback changes
 */
-static int pager_rollback(
+static int SQLITE_TCLAPI pager_rollback(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -129,7 +133,7 @@ static int pager_rollback(
 **
 ** Commit all changes
 */
-static int pager_commit(
+static int SQLITE_TCLAPI pager_commit(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -161,7 +165,7 @@ static int pager_commit(
 **
 ** Start a new checkpoint.
 */
-static int pager_stmt_begin(
+static int SQLITE_TCLAPI pager_stmt_begin(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -188,7 +192,7 @@ static int pager_stmt_begin(
 **
 ** Rollback changes to a checkpoint
 */
-static int pager_stmt_rollback(
+static int SQLITE_TCLAPI pager_stmt_rollback(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -216,7 +220,7 @@ static int pager_stmt_rollback(
 **
 ** Commit changes to a checkpoint
 */
-static int pager_stmt_commit(
+static int SQLITE_TCLAPI pager_stmt_commit(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -243,7 +247,7 @@ static int pager_stmt_commit(
 **
 ** Return pager statistics.
 */
-static int pager_stats(
+static int SQLITE_TCLAPI pager_stats(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -276,7 +280,7 @@ static int pager_stats(
 **
 ** Return the size of the database file.
 */
-static int pager_pagecount(
+static int SQLITE_TCLAPI pager_pagecount(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -302,7 +306,7 @@ static int pager_pagecount(
 **
 ** Return a pointer to a page from the database.
 */
-static int page_get(
+static int SQLITE_TCLAPI page_get(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -339,7 +343,7 @@ static int page_get(
 ** Return a pointer to a page if the page is already in cache.
 ** If not in cache, return an empty string.
 */
-static int page_lookup(
+static int SQLITE_TCLAPI page_lookup(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -367,7 +371,7 @@ static int page_lookup(
 /*
 ** Usage:   pager_truncate ID PGNO
 */
-static int pager_truncate(
+static int SQLITE_TCLAPI pager_truncate(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -392,7 +396,7 @@ static int pager_truncate(
 **
 ** Drop a pointer to a page.
 */
-static int page_unref(
+static int SQLITE_TCLAPI page_unref(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -414,7 +418,7 @@ static int page_unref(
 **
 ** Return the content of a page
 */
-static int page_read(
+static int SQLITE_TCLAPI page_read(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -438,7 +442,7 @@ static int page_read(
 **
 ** Return the page number for a page.
 */
-static int page_number(
+static int SQLITE_TCLAPI page_number(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -462,7 +466,7 @@ static int page_number(
 **
 ** Write something into a page.
 */
-static int page_write(
+static int SQLITE_TCLAPI page_write(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -498,7 +502,7 @@ static int page_write(
 ** new pages after N.  If N is 2096 or bigger, this will test the
 ** ability of SQLite to write to large files.
 */
-static int fake_big_file(
+static int SQLITE_TCLAPI fake_big_file(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -551,7 +555,7 @@ static int fake_big_file(
 **
 ** Set the PENDING_BYTE using the sqlite3_test_control() interface.
 */
-static int testPendingByte(
+static int SQLITE_TCLAPI testPendingByte(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -616,7 +620,7 @@ static int faultSimCallback(int x){
 ** appended, whenever sqlite3FaultSim() is called.  Or, if SCRIPT is the
 ** empty string, cancel the sqlite3FaultSim() callback.
 */
-static int faultInstallCmd(
+static int SQLITE_TCLAPI faultInstallCmd(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -659,7 +663,7 @@ static int faultInstallCmd(
 ** Invoke the SQLITE_TESTCTRL_BITVEC_TEST operator on test_control.
 ** See comments on sqlite3BitvecBuiltinTest() for additional information.
 */
-static int testBitvecBuiltinTest(
+static int SQLITE_TCLAPI testBitvecBuiltinTest(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
diff --git a/src/test3.c b/src/test3.c
index 2a41068e..6b4bfedb 100644
--- a/src/test3.c
+++ b/src/test3.c
@@ -15,7 +15,11 @@
 */
 #include "sqliteInt.h"
 #include "btreeInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -33,7 +37,7 @@ static int nRefSqlite3 = 0;
 **
 ** Open a new database
 */
-static int btree_open(
+static int SQLITE_TCLAPI btree_open(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -79,7 +83,7 @@ static int btree_open(
 **
 ** Close the given database.
 */
-static int btree_close(
+static int SQLITE_TCLAPI btree_close(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -114,7 +118,7 @@ static int btree_close(
 **
 ** Start a new transaction
 */
-static int btree_begin_transaction(
+static int SQLITE_TCLAPI btree_begin_transaction(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -143,7 +147,7 @@ static int btree_begin_transaction(
 **
 ** Returns pager statistics
 */
-static int btree_pager_stats(
+static int SQLITE_TCLAPI btree_pager_stats(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -193,7 +197,7 @@ static int btree_pager_stats(
 **
 ** Create a new cursor.  Return the ID for the cursor.
 */
-static int btree_cursor(
+static int SQLITE_TCLAPI btree_cursor(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -242,14 +246,13 @@ static int btree_cursor(
 **
 ** Close a cursor opened using btree_cursor.
 */
-static int btree_close_cursor(
+static int SQLITE_TCLAPI btree_close_cursor(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
   const char **argv      /* Text of each argument */
 ){
   BtCursor *pCur;
-  Btree *pBt;
   int rc;
 
   if( argc!=2 ){
@@ -258,12 +261,18 @@ static int btree_close_cursor(
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
-  pBt = pCur->pBtree;
-  sqlite3_mutex_enter(pBt->db->mutex);
-  sqlite3BtreeEnter(pBt);
+#if SQLITE_THREADSAFE>0
+  {
+    Btree *pBt = pCur->pBtree;
+    sqlite3_mutex_enter(pBt->db->mutex);
+    sqlite3BtreeEnter(pBt);
+    rc = sqlite3BtreeCloseCursor(pCur);
+    sqlite3BtreeLeave(pBt);
+    sqlite3_mutex_leave(pBt->db->mutex);
+  }
+#else
   rc = sqlite3BtreeCloseCursor(pCur);
-  sqlite3BtreeLeave(pBt);
-  sqlite3_mutex_leave(pBt->db->mutex);
+#endif
   ckfree((char *)pCur);
   if( rc ){
     Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
@@ -279,7 +288,7 @@ static int btree_close_cursor(
 ** or 1 if the cursor was already on the last entry in the table or if
 ** the table is empty.
 */
-static int btree_next(
+static int SQLITE_TCLAPI btree_next(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -314,7 +323,7 @@ static int btree_next(
 ** Move the cursor to the first entry in the table.  Return 0 if the
 ** cursor was left point to something and 1 if the table is empty.
 */
-static int btree_first(
+static int SQLITE_TCLAPI btree_first(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -349,7 +358,7 @@ static int btree_first(
 ** Return TRUE if the given cursor is not pointing at a valid entry.
 ** Return FALSE if the cursor does point to a valid entry.
 */
-static int btree_eof(
+static int SQLITE_TCLAPI btree_eof(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -378,15 +387,14 @@ static int btree_eof(
 **
 ** Return the number of bytes of payload
 */
-static int btree_payload_size(
+static int SQLITE_TCLAPI btree_payload_size(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
   const char **argv      /* Text of each argument */
 ){
   BtCursor *pCur;
-  int n2;
-  u64 n1;
+  u32 n;
   char zBuf[50];
 
   if( argc!=2 ){
@@ -396,17 +404,9 @@ static int btree_payload_size(
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
   sqlite3BtreeEnter(pCur->pBtree);
-
-  /* The cursor may be in "require-seek" state. If this is the case, the
-  ** call to BtreeDataSize() will fix it. */
-  sqlite3BtreeDataSize(pCur, (u32*)&n2);
-  if( pCur->apPage[pCur->iPage]->intKey ){
-    n1 = 0;
-  }else{
-    sqlite3BtreeKeySize(pCur, (i64*)&n1);
-  }
+  n = sqlite3BtreePayloadSize(pCur);
   sqlite3BtreeLeave(pCur->pBtree);
-  sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", (int)(n1+n2));
+  sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n);
   Tcl_AppendResult(interp, zBuf, 0);
   return SQLITE_OK;
 }
@@ -425,7 +425,7 @@ static int btree_payload_size(
 ** This command returns nothing if it works.  It returns an error message
 ** if something goes wrong.
 */
-static int btree_varint_test(
+static int SQLITE_TCLAPI btree_varint_test(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -509,7 +509,7 @@ static int btree_varint_test(
 ** sqlite3 db test.db
 ** set bt [btree_from_db db]
 */
-static int btree_from_db(
+static int SQLITE_TCLAPI btree_from_db(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -547,9 +547,9 @@ static int btree_from_db(
 /*
 ** Usage:   btree_ismemdb ID
 **
-** Return true if the B-Tree is in-memory.
+** Return true if the B-Tree is currently stored entirely in memory.
 */
-static int btree_ismemdb(
+static int SQLITE_TCLAPI btree_ismemdb(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -557,6 +557,7 @@ static int btree_ismemdb(
 ){
   Btree *pBt;
   int res;
+  sqlite3_file *pFile;
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
@@ -566,7 +567,8 @@ static int btree_ismemdb(
   pBt = sqlite3TestTextToPtr(argv[1]);
   sqlite3_mutex_enter(pBt->db->mutex);
   sqlite3BtreeEnter(pBt);
-  res = sqlite3PagerIsMemdb(sqlite3BtreePager(pBt));
+  pFile = sqlite3PagerFile(sqlite3BtreePager(pBt));
+  res = (pFile->pMethods==0);
   sqlite3BtreeLeave(pBt);
   sqlite3_mutex_leave(pBt->db->mutex);
   Tcl_SetObjResult(interp, Tcl_NewBooleanObj(res));
@@ -578,7 +580,7 @@ static int btree_ismemdb(
 **
 ** Set the size of the cache used by btree $ID.
 */
-static int btree_set_cache_size(
+static int SQLITE_TCLAPI btree_set_cache_size(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -608,7 +610,7 @@ static int btree_set_cache_size(
 **
 ** Set the size of the cache used by btree $ID.
 */
-static int btree_insert(
+static int SQLITE_TCLAPI btree_insert(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -616,27 +618,27 @@ static int btree_insert(
 ){
   BtCursor *pCur;
   int rc;
-  void *pKey = 0;
-  int nKey = 0;
-  void *pData = 0;
-  int nData = 0;
+  BtreePayload x;
 
   if( objc!=4 && objc!=3 ){
     Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE");
     return TCL_ERROR;
   }
 
+  memset(&x, 0, sizeof(x));
   if( objc==4 ){
-    if( Tcl_GetIntFromObj(interp, objv[2], &nKey) ) return TCL_ERROR;
-    pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &nData);
+    if( Tcl_GetIntFromObj(interp, objv[2], &rc) ) return TCL_ERROR;
+    x.nKey = rc;
+    x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &x.nData);
   }else{
-    pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &nKey);
+    x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &rc);
+    x.nKey = rc;
   }
   pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
 
   sqlite3_mutex_enter(pCur->pBtree->db->mutex);
   sqlite3BtreeEnter(pCur->pBtree);
-  rc = sqlite3BtreeInsert(pCur, pKey, nKey, pData, nData, 0, 0, 0);
+  rc = sqlite3BtreeInsert(pCur, &x, 0, 0);
   sqlite3BtreeLeave(pCur->pBtree);
   sqlite3_mutex_leave(pCur->pBtree->db->mutex);
 
diff --git a/src/test4.c b/src/test4.c
index d6890303..c5b5ec12 100644
--- a/src/test4.c
+++ b/src/test4.c
@@ -12,7 +12,11 @@
 ** Code for testing the SQLite library in a multithreaded environment.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #if SQLITE_OS_UNIX && SQLITE_THREADSAFE
 #include 
 #include 
@@ -121,7 +125,7 @@ static int parse_thread_id(Tcl_Interp *interp, const char *zArg){
 ** NAME should be an upper case letter.  Start the thread running with
 ** an open connection to the given database.
 */
-static int tcl_thread_create(
+static int SQLITE_TCLAPI tcl_thread_create(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -170,7 +174,7 @@ static void thread_wait(Thread *p){
 **
 ** Wait on thread ID to reach its idle state.
 */
-static int tcl_thread_wait(
+static int SQLITE_TCLAPI tcl_thread_wait(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -214,7 +218,7 @@ static void stop_thread(Thread *p){
 ** Cause a thread to shut itself down.  Wait for the shutdown to be
 ** completed.  If ID is "*" then stop all threads.
 */
-static int tcl_thread_halt(
+static int SQLITE_TCLAPI tcl_thread_halt(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -249,7 +253,7 @@ static int tcl_thread_halt(
 ** Wait on the most recent thread_step to complete, then return the
 ** number of columns in the result set.
 */
-static int tcl_thread_argc(
+static int SQLITE_TCLAPI tcl_thread_argc(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -281,7 +285,7 @@ static int tcl_thread_argc(
 ** Wait on the most recent thread_step to complete, then return the
 ** value of the N-th columns in the result set.
 */
-static int tcl_thread_argv(
+static int SQLITE_TCLAPI tcl_thread_argv(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -317,7 +321,7 @@ static int tcl_thread_argv(
 ** Wait on the most recent thread_step to complete, then return the
 ** name of the N-th columns in the result set.
 */
-static int tcl_thread_colname(
+static int SQLITE_TCLAPI tcl_thread_colname(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -353,7 +357,7 @@ static int tcl_thread_colname(
 ** Wait on the most recent operation to complete, then return the
 ** result code from that operation.
 */
-static int tcl_thread_result(
+static int SQLITE_TCLAPI tcl_thread_result(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -385,7 +389,7 @@ static int tcl_thread_result(
 ** Wait on the most recent operation to complete, then return the
 ** error string.
 */
-static int tcl_thread_error(
+static int SQLITE_TCLAPI tcl_thread_error(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -430,7 +434,7 @@ static void do_compile(Thread *p){
 **
 ** Compile a new virtual machine.
 */
-static int tcl_thread_compile(
+static int SQLITE_TCLAPI tcl_thread_compile(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -483,7 +487,7 @@ static void do_step(Thread *p){
 **
 ** Advance the virtual machine by one step
 */
-static int tcl_thread_step(
+static int SQLITE_TCLAPI tcl_thread_step(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -525,7 +529,7 @@ static void do_finalize(Thread *p){
 **
 ** Finalize the virtual machine.
 */
-static int tcl_thread_finalize(
+static int SQLITE_TCLAPI tcl_thread_finalize(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -556,7 +560,7 @@ static int tcl_thread_finalize(
 **
 ** Interchange the sqlite* pointer between two threads.
 */
-static int tcl_thread_swap(
+static int SQLITE_TCLAPI tcl_thread_swap(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -596,7 +600,7 @@ static int tcl_thread_swap(
 ** remove the pointer from the thread itself.  Afterwards, the thread
 ** can be stopped and the connection can be used by the main thread.
 */
-static int tcl_thread_db_get(
+static int SQLITE_TCLAPI tcl_thread_db_get(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -627,7 +631,7 @@ static int tcl_thread_db_get(
 ** Usage: thread_db_put ID DB
 **
 */
-static int tcl_thread_db_put(
+static int SQLITE_TCLAPI tcl_thread_db_put(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -659,7 +663,7 @@ static int tcl_thread_db_put(
 ** Return the database stmt pointer for the given thread.  Then
 ** remove the pointer from the thread itself. 
 */
-static int tcl_thread_stmt_get(
+static int SQLITE_TCLAPI tcl_thread_stmt_get(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
diff --git a/src/test5.c b/src/test5.c
index 952e3325..0d924286 100644
--- a/src/test5.c
+++ b/src/test5.c
@@ -17,7 +17,11 @@
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -26,7 +30,7 @@
 ** object with the encoded representation of the string, including
 ** the NULL terminator.
 */
-static int binarize(
+static int SQLITE_TCLAPI binarize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -54,7 +58,7 @@ static int binarize(
 ** If  is 0, then the calls to sqlite3_value_text() are not
 ** actually made.
 */
-static int test_value_overhead(
+static int SQLITE_TCLAPI test_value_overhead(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -118,7 +122,7 @@ static u8 name_to_enc(Tcl_Interp *interp, Tcl_Obj *pObj){
 ** Usage:   test_translate    ??
 **
 */
-static int test_translate(
+static int SQLITE_TCLAPI test_translate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -182,7 +186,7 @@ static int test_translate(
 ** translation. If there is a problem an assert() will fail.
 **/
 void sqlite3UtfSelfTest(void);
-static int test_translate_selftest(
+static int SQLITE_TCLAPI test_translate_selftest(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test6.c b/src/test6.c
index 306482dc..5304bcc3 100644
--- a/src/test6.c
+++ b/src/test6.c
@@ -16,7 +16,11 @@
 */
 #if SQLITE_TEST          /* This file is used for testing only */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #ifndef SQLITE_OMIT_DISKIO  /* This file is a no-op if disk I/O is disabled */
 
@@ -215,7 +219,9 @@ static int writeListSync(CrashFile *pFile, int isCrash){
   }
 
 #ifdef TRACE_CRASHTEST
-  printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
+  if( pFile ){
+    printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
+  }
 #endif
 
   ppPtr = &g.pWriteList;
@@ -701,6 +707,10 @@ static int cfCurrentTime(sqlite3_vfs *pCfVfs, double *pTimeOut){
   sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
   return pVfs->xCurrentTime(pVfs, pTimeOut);
 }
+static int cfGetLastError(sqlite3_vfs *pCfVfs, int n, char *z){
+  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
+  return pVfs->xGetLastError(pVfs, n, z);
+}
 
 static int processDevSymArgs(
   Tcl_Interp *interp,
@@ -795,13 +805,34 @@ static int processDevSymArgs(
   return TCL_OK;
 }
 
+/*
+** tclcmd:   sqlite3_crash_now
+**
+** Simulate a crash immediately. This function does not return 
+** (writeListSync() calls exit(-1)).
+*/
+static int SQLITE_TCLAPI crashNowCmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+  writeListSync(0, 1);
+  assert( 0 );
+  return TCL_OK;
+}
+
 /*
 ** tclcmd:   sqlite_crash_enable ENABLE
 **
 ** Parameter ENABLE must be a boolean value. If true, then the "crash"
 ** vfs is added to the system. If false, it is removed.
 */
-static int crashEnableCmd(
+static int SQLITE_TCLAPI crashEnableCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -827,7 +858,7 @@ static int crashEnableCmd(
     cfRandomness,         /* xRandomness */
     cfSleep,              /* xSleep */
     cfCurrentTime,        /* xCurrentTime */
-    0,                    /* xGetlastError */
+    cfGetLastError,       /* xGetLastError */
     0,                    /* xCurrentTimeInt64 */
   };
 
@@ -876,7 +907,7 @@ static int crashEnableCmd(
 **   sqlite_crashparams -sect 1024 -char {atomic sequential} ./test.db 1
 **
 */
-static int crashParamsObjCmd(
+static int SQLITE_TCLAPI crashParamsObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -923,7 +954,7 @@ error:
   return TCL_ERROR;
 }
 
-static int devSymObjCmd(
+static int SQLITE_TCLAPI devSymObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -940,12 +971,33 @@ static int devSymObjCmd(
   devsym_register(iDc, iSectorSize);
 
   return TCL_OK;
+
+}
+
+/*
+** tclcmd: unregister_devsim
+*/
+static int SQLITE_TCLAPI dsUnregisterObjCmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  void devsym_unregister(void);
+
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+
+  devsym_unregister();
+  return TCL_OK;
 }
 
 /*
 ** tclcmd: register_jt_vfs ?-default? PARENT-VFS
 */
-static int jtObjCmd(
+static int SQLITE_TCLAPI jtObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -983,7 +1035,7 @@ static int jtObjCmd(
 /*
 ** tclcmd: unregister_jt_vfs
 */
-static int jtUnregisterObjCmd(
+static int SQLITE_TCLAPI jtUnregisterObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1009,7 +1061,9 @@ int Sqlitetest6_Init(Tcl_Interp *interp){
 #ifndef SQLITE_OMIT_DISKIO
   Tcl_CreateObjCommand(interp, "sqlite3_crash_enable", crashEnableCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "sqlite3_crash_now", crashNowCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "unregister_devsim", dsUnregisterObjCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "register_jt_vfs", jtObjCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "unregister_jt_vfs", jtUnregisterObjCmd, 0, 0);
 #endif
diff --git a/src/test7.c b/src/test7.c
index 6ba3631b..d57e4b82 100644
--- a/src/test7.c
+++ b/src/test7.c
@@ -13,7 +13,11 @@
 ** Derived from test4.c.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 /*
 ** This test only works on UNIX with a SQLITE_THREADSAFE build that includes
@@ -149,7 +153,7 @@ static int parse_client_id(Tcl_Interp *interp, const char *zArg){
 ** NAME should be an upper case letter.  Start the thread running with
 ** an open connection to the given database.
 */
-static int tcl_client_create(
+static int SQLITE_TCLAPI tcl_client_create(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -202,7 +206,7 @@ static void client_wait(Thread *p){
 **
 ** Wait on thread ID to reach its idle state.
 */
-static int tcl_client_wait(
+static int SQLITE_TCLAPI tcl_client_wait(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -246,7 +250,7 @@ static void stop_thread(Thread *p){
 ** Cause a client thread to shut itself down.  Wait for the shutdown to be
 ** completed.  If ID is "*" then stop all client threads.
 */
-static int tcl_client_halt(
+static int SQLITE_TCLAPI tcl_client_halt(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -294,7 +298,7 @@ static int tcl_client_halt(
 ** Wait on the most recent client_step to complete, then return the
 ** number of columns in the result set.
 */
-static int tcl_client_argc(
+static int SQLITE_TCLAPI tcl_client_argc(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -326,7 +330,7 @@ static int tcl_client_argc(
 ** Wait on the most recent client_step to complete, then return the
 ** value of the N-th columns in the result set.
 */
-static int tcl_client_argv(
+static int SQLITE_TCLAPI tcl_client_argv(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -362,7 +366,7 @@ static int tcl_client_argv(
 ** Wait on the most recent client_step to complete, then return the
 ** name of the N-th columns in the result set.
 */
-static int tcl_client_colname(
+static int SQLITE_TCLAPI tcl_client_colname(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -400,7 +404,7 @@ extern const char *sqlite3ErrName(int);
 ** Wait on the most recent operation to complete, then return the
 ** result code from that operation.
 */
-static int tcl_client_result(
+static int SQLITE_TCLAPI tcl_client_result(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -432,7 +436,7 @@ static int tcl_client_result(
 ** Wait on the most recent operation to complete, then return the
 ** error string.
 */
-static int tcl_client_error(
+static int SQLITE_TCLAPI tcl_client_error(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -477,7 +481,7 @@ static void do_compile(Thread *p){
 **
 ** Compile a new virtual machine.
 */
-static int tcl_client_compile(
+static int SQLITE_TCLAPI tcl_client_compile(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -530,7 +534,7 @@ static void do_step(Thread *p){
 **
 ** Advance the virtual machine by one step
 */
-static int tcl_client_step(
+static int SQLITE_TCLAPI tcl_client_step(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -572,7 +576,7 @@ static void do_finalize(Thread *p){
 **
 ** Finalize the virtual machine.
 */
-static int tcl_client_finalize(
+static int SQLITE_TCLAPI tcl_client_finalize(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -616,7 +620,7 @@ static void do_reset(Thread *p){
 **
 ** Finalize the virtual machine.
 */
-static int tcl_client_reset(
+static int SQLITE_TCLAPI tcl_client_reset(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -647,7 +651,7 @@ static int tcl_client_reset(
 **
 ** Interchange the sqlite* pointer between two threads.
 */
-static int tcl_client_swap(
+static int SQLITE_TCLAPI tcl_client_swap(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
diff --git a/src/test8.c b/src/test8.c
index fb781ac8..daab504e 100644
--- a/src/test8.c
+++ b/src/test8.c
@@ -14,7 +14,11 @@
 ** testing of the SQLite library.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -1353,7 +1357,7 @@ static void moduleDestroy(void *p){
 /*
 ** Register the echo virtual table module.
 */
-static int register_echo_module(
+static int SQLITE_TCLAPI register_echo_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1393,7 +1397,7 @@ static int register_echo_module(
 **
 ** sqlite3_declare_vtab DB SQL
 */
-static int declare_vtab(
+static int SQLITE_TCLAPI declare_vtab(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test9.c b/src/test9.c
index e5993e8f..5b139e8a 100644
--- a/src/test9.c
+++ b/src/test9.c
@@ -15,14 +15,18 @@
 ** as there is not much point in binding to Tcl.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
 /*
 ** c_collation_test
 */
-static int c_collation_test(
+static int SQLITE_TCLAPI c_collation_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -63,7 +67,7 @@ error_out:
 /*
 ** c_realloc_test
 */
-static int c_realloc_test(
+static int SQLITE_TCLAPI c_realloc_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -104,7 +108,7 @@ error_out:
 /*
 ** c_misuse_test
 */
-static int c_misuse_test(
+static int SQLITE_TCLAPI c_misuse_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_async.c b/src/test_async.c
index b0b94318..c32c74c6 100644
--- a/src/test_async.c
+++ b/src/test_async.c
@@ -15,7 +15,14 @@
 */
 
 #define TCL_THREADS 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #ifdef SQLITE_ENABLE_ASYNCIO
 
@@ -36,7 +43,7 @@ TCL_DECLARE_MUTEX(testasync_g_writerMutex);
 /*
 ** sqlite3async_initialize PARENT-VFS ISDEFAULT
 */
-static int testAsyncInit(
+static int SQLITE_TCLAPI testAsyncInit(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -69,7 +76,7 @@ static int testAsyncInit(
 /*
 ** sqlite3async_shutdown
 */
-static int testAsyncShutdown(
+static int SQLITE_TCLAPI testAsyncShutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -93,7 +100,7 @@ static Tcl_ThreadCreateType tclWriterThread(ClientData pIsStarted){
 **
 ** Start a new writer thread.
 */
-static int testAsyncStart(
+static int SQLITE_TCLAPI testAsyncStart(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -125,7 +132,7 @@ static int testAsyncStart(
 ** If the current writer thread is set to run forever then this
 ** command would block forever.  To prevent that, an error is returned. 
 */
-static int testAsyncWait(
+static int SQLITE_TCLAPI testAsyncWait(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -151,7 +158,7 @@ static int testAsyncWait(
 /*
 ** sqlite3async_control OPTION ?VALUE?
 */
-static int testAsyncControl(
+static int SQLITE_TCLAPI testAsyncControl(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_autoext.c b/src/test_autoext.c
index a5236d23..e23e41a0 100644
--- a/src/test_autoext.c
+++ b/src/test_autoext.c
@@ -11,7 +11,14 @@
 *************************************************************************
 ** Test extension for testing the sqlite3_auto_extension() function.
 */
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include "sqlite3ext.h"
 
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
@@ -87,13 +94,13 @@ static int broken_init(
 **
 ** Register the "sqr" extension to be loaded automatically.
 */
-static int autoExtSqrObjCmd(
+static int SQLITE_TCLAPI autoExtSqrObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_auto_extension((void*)sqr_init);
+  int rc = sqlite3_auto_extension((void(*)(void))sqr_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -103,13 +110,13 @@ static int autoExtSqrObjCmd(
 **
 ** Unregister the "sqr" extension.
 */
-static int cancelAutoExtSqrObjCmd(
+static int SQLITE_TCLAPI cancelAutoExtSqrObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_cancel_auto_extension((void*)sqr_init);
+  int rc = sqlite3_cancel_auto_extension((void(*)(void))sqr_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -119,13 +126,13 @@ static int cancelAutoExtSqrObjCmd(
 **
 ** Register the "cube" extension to be loaded automatically.
 */
-static int autoExtCubeObjCmd(
+static int SQLITE_TCLAPI autoExtCubeObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_auto_extension((void*)cube_init);
+  int rc = sqlite3_auto_extension((void(*)(void))cube_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -135,13 +142,13 @@ static int autoExtCubeObjCmd(
 **
 ** Unregister the "cube" extension.
 */
-static int cancelAutoExtCubeObjCmd(
+static int SQLITE_TCLAPI cancelAutoExtCubeObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_cancel_auto_extension((void*)cube_init);
+  int rc = sqlite3_cancel_auto_extension((void(*)(void))cube_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -151,13 +158,13 @@ static int cancelAutoExtCubeObjCmd(
 **
 ** Register the broken extension to be loaded automatically.
 */
-static int autoExtBrokenObjCmd(
+static int SQLITE_TCLAPI autoExtBrokenObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_auto_extension((void*)broken_init);
+  int rc = sqlite3_auto_extension((void(*)(void))broken_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -167,13 +174,13 @@ static int autoExtBrokenObjCmd(
 **
 ** Unregister the broken extension.
 */
-static int cancelAutoExtBrokenObjCmd(
+static int SQLITE_TCLAPI cancelAutoExtBrokenObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_cancel_auto_extension((void*)broken_init);
+  int rc = sqlite3_cancel_auto_extension((void(*)(void))broken_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -186,7 +193,7 @@ static int cancelAutoExtBrokenObjCmd(
 **
 ** Reset all auto-extensions
 */
-static int resetAutoExtObjCmd(
+static int SQLITE_TCLAPI resetAutoExtObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_backup.c b/src/test_backup.c
index 6b4d6b9b..9b684a28 100644
--- a/src/test_backup.c
+++ b/src/test_backup.c
@@ -13,7 +13,14 @@
 **
 */
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include "sqlite3.h"
 #include 
 
@@ -23,7 +30,7 @@ extern const char *sqlite3ErrName(int);
 /* These functions are implemented in test1.c. */
 extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **);
 
-static int backupTestCmd(
+static int SQLITE_TCLAPI backupTestCmd(
   ClientData clientData, 
   Tcl_Interp *interp, 
   int objc,
@@ -98,7 +105,7 @@ static int backupTestCmd(
   return TCL_OK;
 }
 
-static void backupTestFinish(ClientData clientData){
+static void SQLITE_TCLAPI backupTestFinish(ClientData clientData){
   sqlite3_backup *pBackup = (sqlite3_backup *)clientData;
   sqlite3_backup_finish(pBackup);
 }
@@ -107,7 +114,7 @@ static void backupTestFinish(ClientData clientData){
 **     sqlite3_backup CMDNAME DESTHANDLE DESTNAME SRCHANDLE SRCNAME
 **
 */
-static int backupTestInit(
+static int SQLITE_TCLAPI backupTestInit(
   ClientData clientData, 
   Tcl_Interp *interp, 
   int objc,
diff --git a/src/test_bestindex.c b/src/test_bestindex.c
new file mode 100644
index 00000000..94017f03
--- /dev/null
+++ b/src/test_bestindex.c
@@ -0,0 +1,610 @@
+/*
+** 2016-03-01
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Code for testing the virtual table xBestIndex method and the query
+** planner.
+*/
+
+
+/*
+** INSTRUCTIONS
+**
+** This module exports a single tcl command - [register_tcl_module]. When
+** invoked, it registers a special virtual table module with a database
+** connection.
+**
+** The virtual table is currently read-only. And always returns zero rows.
+** It is created with a single argument - the name of a Tcl command - as
+** follows:
+**
+**   CREATE VIRTUAL TABLE x1 USING tcl(tcl_command);
+**
+** The command [tcl_command] is invoked when the table is first created (or
+** connected), when the xBestIndex() method is invoked and when the xFilter()
+** method is called. When it is created (or connected), it is invoked as
+** follows:
+**
+**   tcl_command xConnect
+**
+** In this case the return value of the script is passed to the
+** sqlite3_declare_vtab() function to create the virtual table schema.
+**
+** When the xBestIndex() method is called by SQLite, the Tcl command is
+** invoked as:
+**
+**   tcl_command xBestIndex CONSTRAINTS ORDERBY MASK
+**
+** where CONSTRAINTS is a tcl representation of the aConstraints[] array,
+** ORDERBY is a representation of the contents of the aOrderBy[] array and
+** MASK is a copy of sqlite3_index_info.colUsed. For example if the virtual
+** table is declared as:
+**
+**   CREATE TABLE x1(a, b, c)
+**
+** and the query is:
+**
+**   SELECT * FROM x1 WHERE a=? AND c module name  ("fs")
+**   argv[1]   -> database name
+**   argv[2]   -> table name
+**   argv[...] -> other module argument fields.
+*/
+static int tclConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  Tcl_Interp *interp = (Tcl_Interp*)pAux;
+  tcl_vtab *pTab = 0;
+  char *zCmd = 0;
+  Tcl_Obj *pScript = 0;
+  int rc = SQLITE_OK;
+
+  if( argc!=4 ){
+    *pzErr = sqlite3_mprintf("wrong number of arguments");
+    return SQLITE_ERROR;
+  }
+
+  zCmd = sqlite3_malloc64(strlen(argv[3])+1);
+  pTab = (tcl_vtab*)sqlite3_malloc64(sizeof(tcl_vtab));
+  if( zCmd && pTab ){
+    memcpy(zCmd, argv[3], strlen(argv[3])+1);
+    tclDequote(zCmd);
+    memset(pTab, 0, sizeof(tcl_vtab));
+
+    pTab->pCmd = Tcl_NewStringObj(zCmd, -1);
+    pTab->interp = interp;
+    pTab->db = db;
+    Tcl_IncrRefCount(pTab->pCmd);
+
+    pScript = Tcl_DuplicateObj(pTab->pCmd);
+    Tcl_IncrRefCount(pScript);
+    Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("xConnect", -1));
+
+    rc = Tcl_EvalObjEx(interp, pScript, TCL_EVAL_GLOBAL);
+    if( rc!=TCL_OK ){
+      *pzErr = sqlite3_mprintf("%s", Tcl_GetStringResult(interp));
+      rc = SQLITE_ERROR;
+    }else{
+      rc = sqlite3_declare_vtab(db, Tcl_GetStringResult(interp));
+    }
+
+    if( rc!=SQLITE_OK ){
+      sqlite3_free(pTab);
+      pTab = 0;
+    }
+  }else{
+    rc = SQLITE_NOMEM;
+  }
+
+  sqlite3_free(zCmd);
+  *ppVtab = &pTab->base;
+  return rc;
+}
+
+/* The xDisconnect and xDestroy methods are also the same */
+static int tclDisconnect(sqlite3_vtab *pVtab){
+  tcl_vtab *pTab = (tcl_vtab*)pVtab;
+  Tcl_DecrRefCount(pTab->pCmd);
+  sqlite3_free(pTab);
+  return SQLITE_OK;
+}
+
+/*
+** Open a new tcl cursor.
+*/
+static int tclOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  tcl_cursor *pCur;
+  pCur = sqlite3_malloc(sizeof(tcl_cursor));
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(tcl_cursor));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tcl cursor.
+*/
+static int tclClose(sqlite3_vtab_cursor *cur){
+  tcl_cursor *pCur = (tcl_cursor *)cur;
+  if( pCur ){
+    sqlite3_finalize(pCur->pStmt);
+    sqlite3_free(pCur);
+  }
+  return SQLITE_OK;
+}
+
+static int tclNext(sqlite3_vtab_cursor *pVtabCursor){
+  tcl_cursor *pCsr = (tcl_cursor*)pVtabCursor;
+  if( pCsr->pStmt ){
+    tcl_vtab *pTab = (tcl_vtab*)(pVtabCursor->pVtab);
+    int rc = sqlite3_step(pCsr->pStmt);
+    if( rc!=SQLITE_ROW ){
+      const char *zErr;
+      rc = sqlite3_finalize(pCsr->pStmt);
+      pCsr->pStmt = 0;
+      if( rc!=SQLITE_OK ){
+        zErr = sqlite3_errmsg(pTab->db);
+        pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr);
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+static int tclFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  tcl_cursor *pCsr = (tcl_cursor*)pVtabCursor;
+  tcl_vtab *pTab = (tcl_vtab*)(pVtabCursor->pVtab);
+  Tcl_Interp *interp = pTab->interp;
+  Tcl_Obj *pScript;
+  Tcl_Obj *pArg;
+  int ii;
+  int rc;
+
+  pScript = Tcl_DuplicateObj(pTab->pCmd);
+  Tcl_IncrRefCount(pScript);
+  Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("xFilter", -1));
+  Tcl_ListObjAppendElement(interp, pScript, Tcl_NewIntObj(idxNum));
+  if( idxStr ){
+    Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj(idxStr, -1));
+  }else{
+    Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("", -1));
+  }
+
+  pArg = Tcl_NewObj();
+  Tcl_IncrRefCount(pArg);
+  for(ii=0; iibase.zErrMsg = sqlite3_mprintf("%s", zErr);
+  }else{
+    /* Analyze the scripts return value. The return value should be a tcl 
+    ** list object with an even number of elements. The first element of each
+    ** pair must be one of:
+    ** 
+    **   "sql"          (SQL statement to return data)
+    */
+    Tcl_Obj *pRes = Tcl_GetObjResult(interp);
+    Tcl_Obj **apElem = 0;
+    int nElem;
+    rc = Tcl_ListObjGetElements(interp, pRes, &nElem, &apElem);
+    if( rc!=TCL_OK ){
+      const char *zErr = Tcl_GetStringResult(interp);
+      rc = SQLITE_ERROR;
+      pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr);
+    }else{
+      for(ii=0; rc==SQLITE_OK && iidb, zSql, -1, &pCsr->pStmt, 0);
+          if( rc!=SQLITE_OK ){
+            const char *zErr = sqlite3_errmsg(pTab->db);
+            pTab->base.zErrMsg = sqlite3_mprintf("unexpected: %s", zErr);
+          }
+        }else{
+          rc = SQLITE_ERROR;
+          pTab->base.zErrMsg = sqlite3_mprintf("unexpected: %s", zCmd);
+        }
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = tclNext(pVtabCursor);
+  }
+  return rc;
+}
+
+static int tclColumn(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  sqlite3_context *ctx, 
+  int i
+){
+  tcl_cursor *pCsr = (tcl_cursor*)pVtabCursor;
+  sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1));
+  return SQLITE_OK;
+}
+
+static int tclRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){
+  tcl_cursor *pCsr = (tcl_cursor*)pVtabCursor;
+  *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+  return SQLITE_OK;
+}
+
+static int tclEof(sqlite3_vtab_cursor *pVtabCursor){
+  tcl_cursor *pCsr = (tcl_cursor*)pVtabCursor;
+  return (pCsr->pStmt==0);
+}
+
+static int tclBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+  tcl_vtab *pTab = (tcl_vtab*)tab;
+  Tcl_Interp *interp = pTab->interp;
+  Tcl_Obj *pArg;
+  Tcl_Obj *pScript;
+  int ii;
+  int rc = SQLITE_OK;
+
+  pScript = Tcl_DuplicateObj(pTab->pCmd);
+  Tcl_IncrRefCount(pScript);
+  Tcl_ListObjAppendElement(interp, pScript, Tcl_NewStringObj("xBestIndex", -1));
+
+  pArg = Tcl_NewObj();
+  Tcl_IncrRefCount(pArg);
+  for(ii=0; iinConstraint; ii++){
+    struct sqlite3_index_constraint const *pCons = &pIdxInfo->aConstraint[ii];
+    Tcl_Obj *pElem = Tcl_NewObj();
+    const char *zOp = "?";
+
+    Tcl_IncrRefCount(pElem);
+
+    switch( pCons->op ){
+      case SQLITE_INDEX_CONSTRAINT_EQ:
+        zOp = "eq"; break;
+      case SQLITE_INDEX_CONSTRAINT_GT:
+        zOp = "gt"; break;
+      case SQLITE_INDEX_CONSTRAINT_LE:
+        zOp = "le"; break;
+      case SQLITE_INDEX_CONSTRAINT_LT:
+        zOp = "lt"; break;
+      case SQLITE_INDEX_CONSTRAINT_GE:
+        zOp = "ge"; break;
+      case SQLITE_INDEX_CONSTRAINT_MATCH:
+        zOp = "match"; break;
+      case SQLITE_INDEX_CONSTRAINT_LIKE:
+        zOp = "like"; break;
+      case SQLITE_INDEX_CONSTRAINT_GLOB:
+        zOp = "glob"; break;
+      case SQLITE_INDEX_CONSTRAINT_REGEXP:
+        zOp = "regexp"; break;
+    }
+
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj("op", -1));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj(zOp, -1));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj("column", -1));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewIntObj(pCons->iColumn));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj("usable", -1));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewIntObj(pCons->usable));
+
+    Tcl_ListObjAppendElement(0, pArg, pElem);
+    Tcl_DecrRefCount(pElem);
+  }
+
+  Tcl_ListObjAppendElement(0, pScript, pArg);
+  Tcl_DecrRefCount(pArg);
+
+  pArg = Tcl_NewObj();
+  Tcl_IncrRefCount(pArg);
+  for(ii=0; iinOrderBy; ii++){
+    struct sqlite3_index_orderby const *pOrder = &pIdxInfo->aOrderBy[ii];
+    Tcl_Obj *pElem = Tcl_NewObj();
+    Tcl_IncrRefCount(pElem);
+
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj("column", -1));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewIntObj(pOrder->iColumn));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewStringObj("desc", -1));
+    Tcl_ListObjAppendElement(0, pElem, Tcl_NewIntObj(pOrder->desc));
+
+    Tcl_ListObjAppendElement(0, pArg, pElem);
+    Tcl_DecrRefCount(pElem);
+  }
+
+  Tcl_ListObjAppendElement(0, pScript, pArg);
+  Tcl_DecrRefCount(pArg);
+
+  Tcl_ListObjAppendElement(0, pScript, Tcl_NewWideIntObj(pIdxInfo->colUsed));
+
+  rc = Tcl_EvalObjEx(interp, pScript, TCL_EVAL_GLOBAL);
+  Tcl_DecrRefCount(pScript);
+  if( rc!=TCL_OK ){
+    const char *zErr = Tcl_GetStringResult(interp);
+    rc = SQLITE_ERROR;
+    pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr);
+  }else{
+    /* Analyze the scripts return value. The return value should be a tcl 
+    ** list object with an even number of elements. The first element of each
+    ** pair must be one of:
+    ** 
+    **   "orderby"          (value of orderByConsumed flag)
+    **   "cost"             (value of estimatedCost field)
+    **   "rows"             (value of estimatedRows field)
+    **   "use"              (index of used constraint in aConstraint[])
+    **   "idxnum"           (value of idxNum field)
+    **   "idxstr"           (value of idxStr field)
+    **   "omit"             (index of omitted constraint in aConstraint[])
+    */
+    Tcl_Obj *pRes = Tcl_GetObjResult(interp);
+    Tcl_Obj **apElem = 0;
+    int nElem;
+    rc = Tcl_ListObjGetElements(interp, pRes, &nElem, &apElem);
+    if( rc!=TCL_OK ){
+      const char *zErr = Tcl_GetStringResult(interp);
+      rc = SQLITE_ERROR;
+      pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr);
+    }else{
+      int iArgv = 1;
+      for(ii=0; rc==SQLITE_OK && iiestimatedCost);
+        }else
+        if( sqlite3_stricmp("orderby", zCmd)==0 ){
+          rc = Tcl_GetIntFromObj(interp, p, &pIdxInfo->orderByConsumed);
+        }else
+        if( sqlite3_stricmp("idxnum", zCmd)==0 ){
+          rc = Tcl_GetIntFromObj(interp, p, &pIdxInfo->idxNum);
+        }else
+        if( sqlite3_stricmp("idxstr", zCmd)==0 ){
+          sqlite3_free(pIdxInfo->idxStr);
+          pIdxInfo->idxStr = sqlite3_mprintf("%s", Tcl_GetString(p));
+          pIdxInfo->needToFreeIdxStr = 1;
+        }else
+        if( sqlite3_stricmp("rows", zCmd)==0 ){
+          Tcl_WideInt x = 0;
+          rc = Tcl_GetWideIntFromObj(interp, p, &x);
+          pIdxInfo->estimatedRows = (tRowcnt)x;
+        }else
+        if( sqlite3_stricmp("use", zCmd)==0 
+         || sqlite3_stricmp("omit", zCmd)==0 
+        ){
+          int iCons;
+          rc = Tcl_GetIntFromObj(interp, p, &iCons);
+          if( rc==SQLITE_OK ){
+            if( iCons<0 || iCons>=pIdxInfo->nConstraint ){
+              rc = SQLITE_ERROR;
+              pTab->base.zErrMsg = sqlite3_mprintf("unexpected: %d", iCons);
+            }else{
+              int bOmit = (zCmd[0]=='o' || zCmd[0]=='O');
+              pIdxInfo->aConstraintUsage[iCons].argvIndex = iArgv++;
+              pIdxInfo->aConstraintUsage[iCons].omit = bOmit;
+            }
+          }
+        }else{
+          rc = SQLITE_ERROR;
+          pTab->base.zErrMsg = sqlite3_mprintf("unexpected: %s", zCmd);
+        }
+        if( rc!=SQLITE_OK && pTab->base.zErrMsg==0 ){
+          const char *zErr = Tcl_GetStringResult(interp);
+          pTab->base.zErrMsg = sqlite3_mprintf("%s", zErr);
+        }
+      }
+    }
+  }
+
+  return rc;
+}
+
+/*
+** A virtual table module that provides read-only access to a
+** Tcl global variable namespace.
+*/
+static sqlite3_module tclModule = {
+  0,                         /* iVersion */
+  tclConnect,
+  tclConnect,
+  tclBestIndex,
+  tclDisconnect, 
+  tclDisconnect,
+  tclOpen,                      /* xOpen - open a cursor */
+  tclClose,                     /* xClose - close a cursor */
+  tclFilter,                    /* xFilter - configure scan constraints */
+  tclNext,                      /* xNext - advance a cursor */
+  tclEof,                       /* xEof - check for end of scan */
+  tclColumn,                    /* xColumn - read data */
+  tclRowid,                     /* xRowid - read data */
+  0,                           /* xUpdate */
+  0,                           /* xBegin */
+  0,                           /* xSync */
+  0,                           /* xCommit */
+  0,                           /* xRollback */
+  0,                           /* xFindMethod */
+  0,                           /* xRename */
+};
+
+/*
+** Decode a pointer to an sqlite3 object.
+*/
+extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
+
+/*
+** Register the echo virtual table module.
+*/
+static int SQLITE_TCLAPI register_tcl_module(
+  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int objc,              /* Number of arguments */
+  Tcl_Obj *CONST objv[]  /* Command arguments */
+){
+  sqlite3 *db;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  sqlite3_create_module(db, "tcl", &tclModule, (void *)interp);
+#endif
+  return TCL_OK;
+}
+
+#endif
+
+
+/*
+** Register commands with the TCL interpreter.
+*/
+int Sqlitetesttcl_Init(Tcl_Interp *interp){
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  static struct {
+     char *zName;
+     Tcl_ObjCmdProc *xProc;
+     void *clientData;
+  } aObjCmd[] = {
+     { "register_tcl_module",   register_tcl_module, 0 },
+  };
+  int i;
+  for(i=0; i
 #include 
 #include 
@@ -95,7 +99,7 @@ static char *blobStringFromObj(Tcl_Obj *pObj){
 **
 ** Tcl test harness for the sqlite3_blob_open() function.
 */
-static int test_blob_open(
+static int SQLITE_TCLAPI test_blob_open(
   ClientData clientData,          /* Not used */
   Tcl_Interp *interp,             /* Calling TCL interpreter */
   int objc,                       /* Number of arguments */
@@ -105,7 +109,7 @@ static int test_blob_open(
   const char *zDb;
   const char *zTable;
   const char *zColumn;
-  sqlite_int64 iRowid;
+  Tcl_WideInt iRowid;
   int flags;
   const char *zVarname;
   int nVarname;
@@ -146,7 +150,7 @@ static int test_blob_open(
 /*
 ** sqlite3_blob_close  HANDLE
 */
-static int test_blob_close(
+static int SQLITE_TCLAPI test_blob_close(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -174,7 +178,7 @@ static int test_blob_close(
 /*
 ** sqlite3_blob_bytes  HANDLE
 */
-static int test_blob_bytes(
+static int SQLITE_TCLAPI test_blob_bytes(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -210,7 +214,7 @@ static int test_blob_bytes(
 **   text representation of the returned error code (i.e. "SQLITE_NOMEM")
 **   and a Tcl exception is thrown.
 */
-static int test_blob_read(
+static int SQLITE_TCLAPI test_blob_read(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -262,7 +266,7 @@ static int test_blob_read(
 **   result is set to the text representation of the returned error code 
 **   (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown.
 */
-static int test_blob_write(
+static int SQLITE_TCLAPI test_blob_write(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_btree.c b/src/test_btree.c
index dfe77051..03b8b207 100644
--- a/src/test_btree.c
+++ b/src/test_btree.c
@@ -14,7 +14,11 @@
 ** testing of the SQLite library.
 */
 #include "btreeInt.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 /*
 ** Usage: sqlite3_shared_cache_report
@@ -22,7 +26,7 @@
 ** Return a list of file that are shared and the number of
 ** references to each file.
 */
-int sqlite3BtreeSharedCacheReport(
+int SQLITE_TCLAPI sqlite3BtreeSharedCacheReport(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_config.c b/src/test_config.c
index 5db71175..1e68e58d 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -24,7 +24,11 @@
 #  include "os_win.h"
 #endif
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -79,6 +83,13 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "debug", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC
+  Tcl_SetVar2(interp, "sqlite_options", "default_ckptfullfsync", 
+              SQLITE_DEFAULT_CKPTFULLFSYNC ? "1" : "0", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "default_ckptfullfsync", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_DIRECT_OVERFLOW_READ
   Tcl_SetVar2(interp, "sqlite_options", "direct_read", "1", TCL_GLOBAL_ONLY);
 #else
@@ -97,6 +108,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS
+  Tcl_SetVar2(interp, "sqlite_options", "pagecache_overflow_stats","0",TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "pagecache_overflow_stats","1",TCL_GLOBAL_ONLY);
+#endif
+
 #if SQLITE_MAX_MMAP_SIZE>0
   Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY);
 #else
@@ -143,6 +160,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "mem5", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  Tcl_SetVar2(interp, "sqlite_options", "preupdate", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "preupdate", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_SNAPSHOT
   Tcl_SetVar2(interp, "sqlite_options", "snapshot", "1", TCL_GLOBAL_ONLY);
 #else
@@ -370,12 +393,6 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
 #endif
 
-#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
-  Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "1", TCL_GLOBAL_ONLY);
-#else
-  Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "0", TCL_GLOBAL_ONLY);
-#endif
-
 #ifdef SQLITE_ENABLE_FTS5
   Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
 #else
@@ -527,6 +544,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "schema_version", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_SESSION
+  Tcl_SetVar2(interp, "sqlite_options", "session", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "session", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_STAT4
   Tcl_SetVar2(interp, "sqlite_options", "stat4", "1", TCL_GLOBAL_ONLY);
 #else
@@ -582,7 +605,11 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
 #endif
 
   Tcl_SetVar2(interp, "sqlite_options", "threadsafe", 
-      STRINGVALUE(SQLITE_THREADSAFE), TCL_GLOBAL_ONLY);
+      SQLITE_THREADSAFE ? "1" : "0", TCL_GLOBAL_ONLY);
+  Tcl_SetVar2(interp, "sqlite_options", "threadsafe1", 
+      SQLITE_THREADSAFE==1 ? "1" : "0", TCL_GLOBAL_ONLY);
+  Tcl_SetVar2(interp, "sqlite_options", "threadsafe2", 
+      SQLITE_THREADSAFE==2 ? "1" : "0", TCL_GLOBAL_ONLY);
   assert( sqlite3_threadsafe()==SQLITE_THREADSAFE );
 
 #ifdef SQLITE_OMIT_TEMPDB
diff --git a/src/test_delete.c b/src/test_delete.c
new file mode 100644
index 00000000..6b3d9e85
--- /dev/null
+++ b/src/test_delete.c
@@ -0,0 +1,138 @@
+/*
+** 2016 September 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains test code to delete an SQLite database and all
+** of its associated files. Associated files include:
+**
+**   * The journal file.
+**   * The wal file.
+**   * The SQLITE_ENABLE_8_3_NAMES version of the db, journal or wal files.
+**   * Files created by the test_multiplex.c module to extend any of the 
+**     above.
+*/
+
+#if SQLITE_OS_WIN
+#  include 
+#  define F_OK 0
+#else
+#  include 
+#endif
+#include 
+#include 
+#include "sqlite3.h"
+
+/* The following #defines are copied from test_multiplex.c */
+#ifndef MX_CHUNK_NUMBER 
+# define MX_CHUNK_NUMBER 299
+#endif
+#ifndef SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET
+# define SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET 400
+#endif
+#ifndef SQLITE_MULTIPLEX_WAL_8_3_OFFSET
+# define SQLITE_MULTIPLEX_WAL_8_3_OFFSET 700
+#endif
+
+/*
+** This routine is a copy of (most of) the code from SQLite function
+** sqlite3FileSuffix3(). It modifies the filename in buffer z in the
+** same way as SQLite does when in 8.3 filenames mode.
+*/
+static void sqlite3Delete83Name(char *z){
+  int i, sz;
+  sz = strlen(z);
+  for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
+  if( z[i]=='.' && (sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4);
+}
+
+/*
+** zFile is a filename. Assuming no error occurs, if this file exists, 
+** set *pbExists to true and unlink it. Or, if the file does not exist,
+** set *pbExists to false before returning.
+**
+** If an error occurs, the value of errno is returned. Or, if no error
+** occurs, zero is returned.
+*/
+static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){
+  int rc;
+  rc = access(zFile, F_OK);
+  if( rc ){
+    if( errno==ENOENT ){ 
+      if( pbExists ) *pbExists = 0;
+      return 0; 
+    }
+    return errno;
+  }
+  if( pbExists ) *pbExists = 1;
+  rc = unlink(zFile);
+  if( rc ) return errno;
+  return 0;
+}
+
+/*
+** Delete the database file identified by the string argument passed to this
+** function. The string must contain a filename, not an SQLite URI.
+*/
+int sqlite3_delete_database(
+  const char *zFile               /* File to delete */
+){
+  char *zBuf;                     /* Buffer to sprintf() filenames to */
+  int nBuf;                       /* Size of buffer in bytes */
+  int rc = 0;                     /* System error code */
+  int i;                          /* Iterate through azFmt[] and aMFile[] */
+
+  const char *azFmt[] = { "%s", "%s-journal", "%s-wal", "%s-shm" };
+
+  struct MFile {
+    const char *zFmt;
+    int iOffset;
+    int b83;
+  } aMFile[] = {
+    { "%s%03d",         0,   0 },
+    { "%s-journal%03d", 0,   0 },
+    { "%s-wal%03d",     0,   0 },
+    { "%s%03d",         0,   1 },
+    { "%s-journal%03d", SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET, 1 },
+    { "%s-wal%03d",     SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 },
+  };
+
+  /* Allocate a buffer large enough for any of the files that need to be
+  ** deleted.  */
+  nBuf = strlen(zFile) + 100;
+  zBuf = (char*)sqlite3_malloc(nBuf);
+  if( zBuf==0 ) return SQLITE_NOMEM;
+
+  /* Delete both the regular and 8.3 filenames versions of the database,
+  ** journal, wal and shm files.  */
+  for(i=0; rc==0 && izFmt, zFile, iChunk+p->iOffset);
+      if( p->b83 ) sqlite3Delete83Name(zBuf);
+      rc = sqlite3DeleteUnlinkIfExists(zBuf, &bExists);
+      if( bExists==0 || rc!=0 ) break;
+    }
+  }
+
+  sqlite3_free(zBuf);
+  return (rc ? SQLITE_ERROR : SQLITE_OK);
+}
diff --git a/src/test_demovfs.c b/src/test_demovfs.c
index 9410a309..ba7af863 100644
--- a/src/test_demovfs.c
+++ b/src/test_demovfs.c
@@ -641,10 +641,17 @@ sqlite3_vfs *sqlite3_demovfs(void){
 
 #ifdef SQLITE_TEST
 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #if SQLITE_OS_UNIX
-static int register_demovfs(
+static int SQLITE_TCLAPI register_demovfs(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -653,7 +660,7 @@ static int register_demovfs(
   sqlite3_vfs_register(sqlite3_demovfs(), 1);
   return TCL_OK;
 }
-static int unregister_demovfs(
+static int SQLITE_TCLAPI unregister_demovfs(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_devsym.c b/src/test_devsym.c
index 21f0f684..9a1ba09d 100644
--- a/src/test_devsym.c
+++ b/src/test_devsym.c
@@ -133,7 +133,8 @@ struct DevsymGlobal g = {0, 0, 512};
 */
 static int devsymClose(sqlite3_file *pFile){
   devsym_file *p = (devsym_file *)pFile;
-  return sqlite3OsClose(p->pReal);
+  sqlite3OsClose(p->pReal);
+  return SQLITE_OK;
 }
 
 /*
@@ -395,4 +396,11 @@ void devsym_register(int iDeviceChar, int iSectorSize){
   }
 }
 
+void devsym_unregister(){
+  sqlite3_vfs_unregister(&devsym_vfs);
+  g.pVfs = 0;
+  g.iDeviceChar = 0;
+  g.iSectorSize = 0;
+}
+
 #endif
diff --git a/src/test_fs.c b/src/test_fs.c
index 45db0b53..dd590a66 100644
--- a/src/test_fs.c
+++ b/src/test_fs.c
@@ -62,7 +62,11 @@
 **   SELECT * FROM fstree WHERE path LIKE '/home/dan/sqlite/%'
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #include 
 #include 
@@ -495,10 +499,14 @@ static int fstreeFilter(
   char aWild[2] = { '\0', '\0' };
 
 #if SQLITE_OS_WIN
-  zRoot = sqlite3_mprintf("%s%c", getenv("SystemDrive"), '/');
-  nRoot = strlen(zRoot);
-  zPrefix = sqlite3_mprintf("%s", getenv("SystemDrive"));
-  nPrefix = strlen(zPrefix);
+  const char *zDrive = windirent_getenv("fstreeDrive");
+  if( zDrive==0 ){
+    zDrive = windirent_getenv("SystemDrive");
+  }
+  zRoot = sqlite3_mprintf("%s%c", zDrive, '/');
+  nRoot = sqlite3Strlen30(zRoot);
+  zPrefix = sqlite3_mprintf("%s", zDrive);
+  nPrefix = sqlite3Strlen30(zPrefix);
 #else
   zRoot = "/";
   nRoot = 1;
@@ -871,7 +879,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the echo virtual table module.
 */
-static int register_fs_module(
+static int SQLITE_TCLAPI register_fs_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_func.c b/src/test_func.c
index 579177b7..26f0d369 100644
--- a/src/test_func.c
+++ b/src/test_func.c
@@ -13,7 +13,11 @@
 ** implements new SQL functions used by the test scripts.
 */
 #include "sqlite3.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -21,7 +25,6 @@
 #include "sqliteInt.h"
 #include "vdbeInt.h"
 
-
 /*
 ** Allocate nByte bytes of space using sqlite3_malloc(). If the
 ** allocation fails, call sqlite3_result_error_nomem() to notify
@@ -640,7 +643,11 @@ static void test_setsubtype(
   sqlite3_result_subtype(context, (unsigned int)sqlite3_value_int(argv[1]));
 }
 
-static int registerTestFunctions(sqlite3 *db){
+static int registerTestFunctions(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pThunk
+){
   static const struct {
      char *zName;
      signed char nArg;
@@ -689,16 +696,16 @@ static int registerTestFunctions(sqlite3 *db){
 ** the standard set of test functions to be loaded into each new
 ** database connection.
 */
-static int autoinstall_test_funcs(
+static int SQLITE_TCLAPI autoinstall_test_funcs(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  extern int Md5_Register(sqlite3*);
-  int rc = sqlite3_auto_extension((void*)registerTestFunctions);
+  extern int Md5_Register(sqlite3 *, char **, const sqlite3_api_routines *);
+  int rc = sqlite3_auto_extension((void(*)(void))registerTestFunctions);
   if( rc==SQLITE_OK ){
-    rc = sqlite3_auto_extension((void*)Md5_Register);
+    rc = sqlite3_auto_extension((void(*)(void))Md5_Register);
   }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return TCL_OK;
@@ -717,7 +724,7 @@ static void tFinal(sqlite3_context *a){}
 ** Make various calls to sqlite3_create_function that do not have valid
 ** parameters.  Verify that the error condition is detected and reported.
 */
-static int abuse_create_function(
+static int SQLITE_TCLAPI abuse_create_function(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -783,6 +790,7 @@ abuse_err:
   return TCL_ERROR;
 }
 
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -795,13 +803,13 @@ int Sqlitetest_func_Init(Tcl_Interp *interp){
      { "abuse_create_function",         abuse_create_function  },
   };
   int i;
-  extern int Md5_Register(sqlite3*);
+  extern int Md5_Register(sqlite3 *, char **, const sqlite3_api_routines *);
 
   for(i=0; i
 #include 
 #include 
@@ -94,7 +98,7 @@ int sqlite3TestHexToBin(const unsigned char *zIn, int N, unsigned char *aOut){
 ** beginning of the file.  Convert that information to hexadecimal
 ** and return the resulting HEX string.
 */
-static int hexio_read(
+static int SQLITE_TCLAPI hexio_read(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -144,7 +148,7 @@ static int hexio_read(
 ** Write DATA into file FILENAME beginning at OFFSET from the
 ** beginning of the file.  DATA is expressed in hexadecimal.
 */
-static int hexio_write(
+static int SQLITE_TCLAPI hexio_write(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -192,7 +196,7 @@ static int hexio_write(
 ** the value of that integer.  HEXDATA can contain between 2 and 8
 ** hexadecimal digits.
 */
-static int hexio_get_int(
+static int SQLITE_TCLAPI hexio_get_int(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -232,7 +236,7 @@ static int hexio_get_int(
 **
 ** Render INTEGER has a 16-bit big-endian integer in hexadecimal.
 */
-static int hexio_render_int16(
+static int SQLITE_TCLAPI hexio_render_int16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -259,7 +263,7 @@ static int hexio_render_int16(
 **
 ** Render INTEGER has a 32-bit big-endian integer in hexadecimal.
 */
-static int hexio_render_int32(
+static int SQLITE_TCLAPI hexio_render_int32(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -289,7 +293,7 @@ static int hexio_render_int32(
 ** The UTF8 might not be well-formed.  Run this string through
 ** sqlite3Utf8to8() convert it back to hex and return the result.
 */
-static int utf8_to_utf8(
+static int SQLITE_TCLAPI utf8_to_utf8(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -340,7 +344,7 @@ static int getFts3Varint(const char *p, sqlite_int64 *v){
 ** Read a varint from the start of BLOB. Set variable VARNAME to contain
 ** the interpreted value. Return the number of bytes of BLOB consumed.
 */
-static int read_fts3varint(
+static int SQLITE_TCLAPI read_fts3varint(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_init.c b/src/test_init.c
index 502d95c0..58465785 100644
--- a/src/test_init.c
+++ b/src/test_init.c
@@ -27,7 +27,11 @@
 
 #include "sqliteInt.h"
 #include 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 static struct Wrapped {
   sqlite3_pcache_methods2 pcache;
@@ -184,7 +188,7 @@ static void installInitWrappers(void){
   sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcachemethods);
 }
 
-static int init_wrapper_install(
+static int SQLITE_TCLAPI init_wrapper_install(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -208,7 +212,7 @@ static int init_wrapper_install(
   return TCL_OK;
 }
 
-static int init_wrapper_uninstall(
+static int SQLITE_TCLAPI init_wrapper_uninstall(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -226,7 +230,7 @@ static int init_wrapper_uninstall(
   return TCL_OK;
 }
 
-static int init_wrapper_clear(
+static int SQLITE_TCLAPI init_wrapper_clear(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -243,7 +247,7 @@ static int init_wrapper_clear(
   return TCL_OK;
 }
 
-static int init_wrapper_query(
+static int SQLITE_TCLAPI init_wrapper_query(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_intarray.c b/src/test_intarray.c
index 70e34db3..ddfb13fe 100644
--- a/src/test_intarray.c
+++ b/src/test_intarray.c
@@ -270,7 +270,14 @@ SQLITE_API int sqlite3_intarray_bind(
 ** Everything below is interface for testing this module.
 */
 #ifdef SQLITE_TEST
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 /*
 ** Routines to encode and decode pointers
@@ -286,7 +293,7 @@ extern const char *sqlite3ErrName(int);
 ** Invoke the sqlite3_intarray_create interface.  A string that becomes
 ** the first parameter to sqlite3_intarray_bind.
 */
-static int test_intarray_create(
+static int SQLITE_TCLAPI test_intarray_create(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -322,7 +329,7 @@ static int test_intarray_create(
 **
 ** Invoke the sqlite3_intarray_bind interface on the given array of integers.
 */
-static int test_intarray_bind(
+static int SQLITE_TCLAPI test_intarray_bind(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_intarray.h b/src/test_intarray.h
index 84b1f3fe..cee55b5d 100644
--- a/src/test_intarray.h
+++ b/src/test_intarray.h
@@ -75,8 +75,8 @@
 ** action to free the intarray objects.
 */
 #include "sqlite3.h"
-#ifndef _INTARRAY_H_
-#define _INTARRAY_H_
+#ifndef SQLITE_INTARRAY_H
+#define SQLITE_INTARRAY_H
 
 /*
 ** Make sure we can call this stuff from C++.
@@ -125,4 +125,4 @@ SQLITE_API int sqlite3_intarray_bind(
 #ifdef __cplusplus
 }  /* End of the 'extern "C"' block */
 #endif
-#endif /* _INTARRAY_H_ */
+#endif /* SQLITE_INTARRAY_H */
diff --git a/src/test_journal.c b/src/test_journal.c
index 6e320b7a..4e63bccf 100644
--- a/src/test_journal.c
+++ b/src/test_journal.c
@@ -160,6 +160,7 @@ static int jtRandomness(sqlite3_vfs*, int nByte, char *zOut);
 static int jtSleep(sqlite3_vfs*, int microseconds);
 static int jtCurrentTime(sqlite3_vfs*, double*);
 static int jtCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int jtGetLastError(sqlite3_vfs*, int, char*);
 
 static sqlite3_vfs jt_vfs = {
   2,                             /* iVersion */
@@ -179,7 +180,7 @@ static sqlite3_vfs jt_vfs = {
   jtRandomness,                  /* xRandomness */
   jtSleep,                       /* xSleep */
   jtCurrentTime,                 /* xCurrentTime */
-  0,                             /* xGetLastError */
+  jtGetLastError,                /* xGetLastError */
   jtCurrentTimeInt64             /* xCurrentTimeInt64 */
 };
 
@@ -256,7 +257,8 @@ static int jtClose(sqlite3_file *pFile){
     *pp = p->pNext;
   }
   leaveJtMutex();
-  return sqlite3OsClose(p->pReal);
+  sqlite3OsClose(p->pReal);
+  return SQLITE_OK;
 }
 
 /*
@@ -284,9 +286,10 @@ static int jtRead(
 **   b) The file-name specified when the file was opened matches
 **      all but the final 8 characters of the journal file name.
 **
-**   c) There is currently a reserved lock on the file.
+**   c) There is currently a reserved lock on the file. This 
+**      condition is waived if the noLock argument is non-zero.
 **/
-static jt_file *locateDatabaseHandle(const char *zJournal){
+static jt_file *locateDatabaseHandle(const char *zJournal, int noLock){
   jt_file *pMain = 0;
   enterJtMutex();
   for(pMain=g.pList; pMain; pMain=pMain->pNext){
@@ -294,7 +297,7 @@ static jt_file *locateDatabaseHandle(const char *zJournal){
     if( (pMain->flags&SQLITE_OPEN_MAIN_DB)
      && ((int)strlen(pMain->zName)==nName)
      && 0==memcmp(pMain->zName, zJournal, nName)
-     && (pMain->eLock>=SQLITE_LOCK_RESERVED)
+     && ((pMain->eLock>=SQLITE_LOCK_RESERVED) || noLock)
     ){
       break;
     }
@@ -516,7 +519,7 @@ static int jtWrite(
   jt_file *p = (jt_file *)pFile;
   if( p->flags&SQLITE_OPEN_MAIN_JOURNAL ){
     if( iOfst==0 ){
-      jt_file *pMain = locateDatabaseHandle(p->zName);
+      jt_file *pMain = locateDatabaseHandle(p->zName, 0);
       assert( pMain );
   
       if( iAmt==28 ){
@@ -561,7 +564,7 @@ static int jtWrite(
 
   rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst);
   if( (p->flags&SQLITE_OPEN_MAIN_JOURNAL) && iAmt==12 ){
-    jt_file *pMain = locateDatabaseHandle(p->zName);
+    jt_file *pMain = locateDatabaseHandle(p->zName, 0);
     int rc2 = readJournalFile(p, pMain);
     if( rc==SQLITE_OK ) rc = rc2;
   }
@@ -575,7 +578,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){
   jt_file *p = (jt_file *)pFile;
   if( p->flags&SQLITE_OPEN_MAIN_JOURNAL && size==0 ){
     /* Truncating a journal file. This is the end of a transaction. */
-    jt_file *pMain = locateDatabaseHandle(p->zName);
+    jt_file *pMain = locateDatabaseHandle(p->zName, 0);
     closeTransaction(pMain);
   }
   if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){
@@ -603,11 +606,10 @@ static int jtSync(sqlite3_file *pFile, int flags){
     ** jt_file.pWritable bitvec of the main database file associated with
     ** this journal file.
     */
-    pMain = locateDatabaseHandle(p->zName);
-    assert(pMain);
+    pMain = locateDatabaseHandle(p->zName, 0);
 
     /* Set the bitvec values */
-    if( pMain->pWritable ){
+    if( pMain && pMain->pWritable ){
       pMain->nSync++;
       rc = readJournalFile(p, pMain);
       if( rc!=SQLITE_OK ){
@@ -729,7 +731,7 @@ static int jtDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   int nPath = (int)strlen(zPath);
   if( nPath>8 && 0==strcmp("-journal", &zPath[nPath-8]) ){
     /* Deleting a journal file. The end of a transaction. */
-    jt_file *pMain = locateDatabaseHandle(zPath);
+    jt_file *pMain = locateDatabaseHandle(zPath, 0);
     if( pMain ){
       closeTransaction(pMain);
     }
@@ -824,6 +826,10 @@ static int jtCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){
   return g.pVfs->xCurrentTimeInt64(g.pVfs, pTimeOut);
 }
 
+static int jtGetLastError(sqlite3_vfs *pVfs, int n, char *z){
+  return g.pVfs->xGetLastError(g.pVfs, n, z);
+}
+
 /**************************************************************************
 ** Start of public API.
 */
diff --git a/src/test_malloc.c b/src/test_malloc.c
index aaa640b0..e8c248f9 100644
--- a/src/test_malloc.c
+++ b/src/test_malloc.c
@@ -14,7 +14,11 @@
 ** memory allocation subsystem.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -306,7 +310,7 @@ static int textToPointer(const char *z, void **pp){
 **
 ** Raw test interface for sqlite3_malloc().
 */
-static int test_malloc(
+static int SQLITE_TCLAPI test_malloc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -331,7 +335,7 @@ static int test_malloc(
 **
 ** Raw test interface for sqlite3_realloc().
 */
-static int test_realloc(
+static int SQLITE_TCLAPI test_realloc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -360,7 +364,7 @@ static int test_realloc(
 **
 ** Raw test interface for sqlite3_free().
 */
-static int test_free(
+static int SQLITE_TCLAPI test_free(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -391,7 +395,7 @@ int sqlite3TestBinToHex(char*,int);
 ** Set a chunk of memory (obtained from malloc, probably) to a
 ** specified hex pattern.
 */
-static int test_memset(
+static int SQLITE_TCLAPI test_memset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -437,7 +441,7 @@ static int test_memset(
 **
 ** Return memory as hexadecimal text.
 */
-static int test_memget(
+static int SQLITE_TCLAPI test_memget(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -484,7 +488,7 @@ static int test_memget(
 **
 ** Raw test interface for sqlite3_memory_used().
 */
-static int test_memory_used(
+static int SQLITE_TCLAPI test_memory_used(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -499,7 +503,7 @@ static int test_memory_used(
 **
 ** Raw test interface for sqlite3_memory_highwater().
 */
-static int test_memory_highwater(
+static int SQLITE_TCLAPI test_memory_highwater(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -524,7 +528,7 @@ static int test_memory_highwater(
 ** Set the depth of backtracing.  If SQLITE_MEMDEBUG is not defined
 ** then this routine is a no-op.
 */
-static int test_memdebug_backtrace(
+static int SQLITE_TCLAPI test_memdebug_backtrace(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -550,7 +554,7 @@ static int test_memdebug_backtrace(
 **
 ** Write a summary of unfreed memory to FILENAME.
 */
-static int test_memdebug_dump(
+static int SQLITE_TCLAPI test_memdebug_dump(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -575,7 +579,7 @@ static int test_memdebug_dump(
 **
 ** Return the total number of times malloc() has been called.
 */
-static int test_memdebug_malloc_count(
+static int SQLITE_TCLAPI test_memdebug_malloc_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -615,7 +619,7 @@ static int test_memdebug_malloc_count(
 **
 ** To disable simulated failures, use a COUNTER of -1.
 */
-static int test_memdebug_fail(
+static int SQLITE_TCLAPI test_memdebug_fail(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -681,7 +685,7 @@ static int test_memdebug_fail(
 ** simulated failure occurs. A negative return value indicates that
 ** no malloc() failure is scheduled.
 */
-static int test_memdebug_pending(
+static int SQLITE_TCLAPI test_memdebug_pending(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -714,7 +718,7 @@ static int sqlite3_memdebug_title_count = 0;
 **
 ** Each title overwrite the previous.
 */
-static int test_memdebug_settitle(
+static int SQLITE_TCLAPI test_memdebug_settitle(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -795,7 +799,7 @@ static void test_memdebug_log_clear(void){
   Tcl_InitHashTable(&aMallocLog, MALLOC_LOG_KEYINTS);
 }
 
-static int test_memdebug_log(
+static int SQLITE_TCLAPI test_memdebug_log(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -893,7 +897,7 @@ static int test_memdebug_log(
 **
 ** A negative SIZE causes the buffer pointer to be NULL.
 */
-static int test_config_scratch(
+static int SQLITE_TCLAPI test_config_scratch(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -933,7 +937,7 @@ static int test_config_scratch(
 **
 ** A negative SIZE causes the buffer pointer to be NULL.
 */
-static int test_config_pagecache(
+static int SQLITE_TCLAPI test_config_pagecache(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -976,7 +980,7 @@ static int test_config_pagecache(
 ** is certainty.  0 is never.  PRNG_SEED is the pseudo-random number generator
 ** seed.
 */
-static int test_alt_pcache(
+static int SQLITE_TCLAPI test_alt_pcache(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1017,7 +1021,7 @@ static int test_alt_pcache(
 **
 ** Enable or disable memory status reporting using SQLITE_CONFIG_MEMSTATUS.
 */
-static int test_config_memstatus(
+static int SQLITE_TCLAPI test_config_memstatus(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1038,7 +1042,7 @@ static int test_config_memstatus(
 ** Usage:    sqlite3_config_lookaside  SIZE  COUNT
 **
 */
-static int test_config_lookaside(
+static int SQLITE_TCLAPI test_config_lookaside(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1072,7 +1076,7 @@ static int test_config_lookaside(
 ** is 10KB in size.  A BUFID of 0 indicates that the buffer should be NULL
 ** which will cause sqlite3_db_config() to allocate space on its own.
 */
-static int test_db_config_lookaside(
+static int SQLITE_TCLAPI test_db_config_lookaside(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1093,7 +1097,7 @@ static int test_db_config_lookaside(
   if( Tcl_GetIntFromObj(interp, objv[3], &sz) ) return TCL_ERROR;
   if( Tcl_GetIntFromObj(interp, objv[4], &cnt) ) return TCL_ERROR;
   if( bufid==0 ){
-    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, 0, sz, cnt);
+    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, (void*)0, sz, cnt);
   }else if( bufid>=1 && bufid<=2 && sz*cnt<=sizeof(azBuf[0]) ){
     rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, azBuf[bufid], sz,cnt);
   }else{
@@ -1107,7 +1111,7 @@ static int test_db_config_lookaside(
 /*
 ** Usage:    sqlite3_config_heap NBYTE NMINALLOC
 */
-static int test_config_heap(
+static int SQLITE_TCLAPI test_config_heap(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1144,7 +1148,7 @@ static int test_config_heap(
 /*
 ** Usage:    sqlite3_config_heap_size NBYTE
 */
-static int test_config_heap_size(
+static int SQLITE_TCLAPI test_config_heap_size(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1174,7 +1178,7 @@ static int test_config_heap_size(
 ** Invoke sqlite3_config() or sqlite3_db_config() with invalid
 ** opcodes and verify that they return errors.
 */
-static int test_config_error(
+static int SQLITE_TCLAPI test_config_error(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1212,7 +1216,7 @@ static int test_config_error(
 ** Enables or disables interpretation of URI parameters by default using
 ** SQLITE_CONFIG_URI.
 */
-static int test_config_uri(
+static int SQLITE_TCLAPI test_config_uri(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1241,7 +1245,7 @@ static int test_config_uri(
 ** Enables or disables the use of the covering-index scan optimization.
 ** SQLITE_CONFIG_COVERING_INDEX_SCAN.
 */
-static int test_config_cis(
+static int SQLITE_TCLAPI test_config_cis(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1269,7 +1273,7 @@ static int test_config_cis(
 **
 ** Set the minimum PMA size.
 */
-static int test_config_pmasz(
+static int SQLITE_TCLAPI test_config_pmasz(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1299,7 +1303,7 @@ static int test_config_pmasz(
 **
 ** Write a summary of unfreed memsys3 allocations to FILENAME.
 */
-static int test_dump_memsys3(
+static int SQLITE_TCLAPI test_dump_memsys3(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1335,7 +1339,7 @@ static int test_dump_memsys3(
 ** Return a list of three elements which are the sqlite3_status() return
 ** code, the current value, and the high-water mark value.
 */
-static int test_status(
+static int SQLITE_TCLAPI test_status(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1392,7 +1396,7 @@ static int test_status(
 ** Return a list of three elements which are the sqlite3_db_status() return
 ** code, the current value, and the high-water mark value.
 */
-static int test_db_status(
+static int SQLITE_TCLAPI test_db_status(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1417,7 +1421,8 @@ static int test_db_status(
     { "CACHE_HIT",           SQLITE_DBSTATUS_CACHE_HIT           },
     { "CACHE_MISS",          SQLITE_DBSTATUS_CACHE_MISS          },
     { "CACHE_WRITE",         SQLITE_DBSTATUS_CACHE_WRITE         },
-    { "DEFERRED_FKS",        SQLITE_DBSTATUS_DEFERRED_FKS        }
+    { "DEFERRED_FKS",        SQLITE_DBSTATUS_DEFERRED_FKS        },
+    { "CACHE_USED_SHARED",   SQLITE_DBSTATUS_CACHE_USED_SHARED   },
   };
   Tcl_Obj *pResult;
   if( objc!=4 ){
@@ -1452,7 +1457,7 @@ static int test_db_status(
 /*
 ** install_malloc_faultsim BOOLEAN
 */
-static int test_install_malloc_faultsim(
+static int SQLITE_TCLAPI test_install_malloc_faultsim(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1476,7 +1481,7 @@ static int test_install_malloc_faultsim(
 /*
 ** sqlite3_install_memsys3
 */
-static int test_install_memsys3(
+static int SQLITE_TCLAPI test_install_memsys3(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1491,7 +1496,7 @@ static int test_install_memsys3(
   return TCL_OK;
 }
 
-static int test_vfs_oom_test(
+static int SQLITE_TCLAPI test_vfs_oom_test(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_multiplex.c b/src/test_multiplex.c
index 82845ea7..1027aa13 100644
--- a/src/test_multiplex.c
+++ b/src/test_multiplex.c
@@ -539,7 +539,7 @@ static int multiplexOpen(
     memset(pGroup, 0, sz);
     pMultiplexOpen->pGroup = pGroup;
     pGroup->bEnabled = (unsigned char)-1;
-    pGroup->bTruncate = sqlite3_uri_boolean(zUri, "truncate", 
+    pGroup->bTruncate = (unsigned char)sqlite3_uri_boolean(zUri, "truncate", 
                                    (flags & SQLITE_OPEN_MAIN_DB)==0);
     pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize",
                                         SQLITE_MULTIPLEX_CHUNK_SIZE);
@@ -717,7 +717,11 @@ static int multiplexCurrentTime(sqlite3_vfs *a, double *b){
   return gMultiplex.pOrigVfs->xCurrentTime(gMultiplex.pOrigVfs, b);
 }
 static int multiplexGetLastError(sqlite3_vfs *a, int b, char *c){
-  return gMultiplex.pOrigVfs->xGetLastError(gMultiplex.pOrigVfs, b, c);
+  if( gMultiplex.pOrigVfs->xGetLastError ){
+    return gMultiplex.pOrigVfs->xGetLastError(gMultiplex.pOrigVfs, b, c);
+  }else{
+    return 0;
+  }
 }
 static int multiplexCurrentTimeInt64(sqlite3_vfs *a, sqlite3_int64 *b){
   return gMultiplex.pOrigVfs->xCurrentTimeInt64(gMultiplex.pOrigVfs, b);
@@ -974,7 +978,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){
     case MULTIPLEX_CTRL_ENABLE:
       if( pArg ) {
         int bEnabled = *(int *)pArg;
-        pGroup->bEnabled = bEnabled;
+        pGroup->bEnabled = (unsigned char)bEnabled;
         rc = SQLITE_OK;
       }
       break;
@@ -1193,7 +1197,7 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){
   gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap;
   sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault);
 
-  sqlite3_auto_extension((void*)multiplexFuncInit);
+  sqlite3_auto_extension((void(*)(void))multiplexFuncInit);
 
   return SQLITE_OK;
 }
@@ -1225,14 +1229,21 @@ int sqlite3_multiplex_shutdown(int eForce){
 
 /***************************** Test Code ***********************************/
 #ifdef SQLITE_TEST
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 extern const char *sqlite3ErrName(int);
 
 
 /*
 ** tclcmd: sqlite3_multiplex_initialize NAME MAKEDEFAULT
 */
-static int test_multiplex_initialize(
+static int SQLITE_TCLAPI test_multiplex_initialize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1263,7 +1274,7 @@ static int test_multiplex_initialize(
 /*
 ** tclcmd: sqlite3_multiplex_shutdown
 */
-static int test_multiplex_shutdown(
+static int SQLITE_TCLAPI test_multiplex_shutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1291,7 +1302,7 @@ static int test_multiplex_shutdown(
 /*
 ** tclcmd:  sqlite3_multiplex_dump
 */
-static int test_multiplex_dump(
+static int SQLITE_TCLAPI test_multiplex_dump(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1346,7 +1357,7 @@ static int test_multiplex_dump(
 /*
 ** Tclcmd: test_multiplex_control HANDLE DBNAME SUB-COMMAND ?INT-VALUE?
 */
-static int test_multiplex_control(
+static int SQLITE_TCLAPI test_multiplex_control(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_multiplex.h b/src/test_multiplex.h
index d973e4af..790c778a 100644
--- a/src/test_multiplex.h
+++ b/src/test_multiplex.h
@@ -20,8 +20,8 @@
 **
 */
 
-#ifndef _TEST_MULTIPLEX_H
-#define _TEST_MULTIPLEX_H
+#ifndef SQLITE_TEST_MULTIPLEX_H
+#define SQLITE_TEST_MULTIPLEX_H
 
 /*
 ** CAPI: File-control Operations Supported by Multiplex VFS
@@ -96,4 +96,4 @@ extern int sqlite3_multiplex_shutdown(int eForce);
 }  /* End of the 'extern "C"' block */
 #endif
 
-#endif /* _TEST_MULTIPLEX_H */
+#endif /* SQLITE_TEST_MULTIPLEX_H */
diff --git a/src/test_mutex.c b/src/test_mutex.c
index 995b89a4..8f43e5ad 100644
--- a/src/test_mutex.c
+++ b/src/test_mutex.c
@@ -12,7 +12,11 @@
 ** This file contains test logic for the sqlite3_mutex interfaces.
 */
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include "sqlite3.h"
 #include "sqliteInt.h"
 #include 
@@ -152,7 +156,7 @@ static void counterMutexLeave(sqlite3_mutex *p){
 /*
 ** sqlite3_shutdown
 */
-static int test_shutdown(
+static int SQLITE_TCLAPI test_shutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -173,7 +177,7 @@ static int test_shutdown(
 /*
 ** sqlite3_initialize
 */
-static int test_initialize(
+static int SQLITE_TCLAPI test_initialize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -194,7 +198,7 @@ static int test_initialize(
 /*
 ** install_mutex_counters BOOLEAN
 */
-static int test_install_mutex_counters(
+static int SQLITE_TCLAPI test_install_mutex_counters(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -255,7 +259,7 @@ static int test_install_mutex_counters(
 /*
 ** read_mutex_counters
 */
-static int test_read_mutex_counters(
+static int SQLITE_TCLAPI test_read_mutex_counters(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -284,7 +288,7 @@ static int test_read_mutex_counters(
 /*
 ** clear_mutex_counters
 */
-static int test_clear_mutex_counters(
+static int SQLITE_TCLAPI test_clear_mutex_counters(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -308,7 +312,7 @@ static int test_clear_mutex_counters(
 ** will be invalid since the mutex has already been freed.  The
 ** return pointer just checks to see if the mutex really was allocated.
 */
-static int test_alloc_mutex(
+static int SQLITE_TCLAPI test_alloc_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -335,7 +339,7 @@ static int test_alloc_mutex(
 **
 ** Or OPTION can be an raw integer.
 */
-static int test_config(
+static int SQLITE_TCLAPI test_config(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -397,7 +401,7 @@ static sqlite3_mutex *getStaticMutexPointer(
   return counterMutexAlloc(iMutex);
 }
 
-static int test_enter_static_mutex(
+static int SQLITE_TCLAPI test_enter_static_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -416,7 +420,7 @@ static int test_enter_static_mutex(
   return TCL_OK;
 }
 
-static int test_leave_static_mutex(
+static int SQLITE_TCLAPI test_leave_static_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -435,7 +439,7 @@ static int test_leave_static_mutex(
   return TCL_OK;
 }
 
-static int test_enter_db_mutex(
+static int SQLITE_TCLAPI test_enter_db_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -454,7 +458,7 @@ static int test_enter_db_mutex(
   return TCL_OK;
 }
 
-static int test_leave_db_mutex(
+static int SQLITE_TCLAPI test_leave_db_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_onefile.c b/src/test_onefile.c
index 122be700..46c8bc9c 100644
--- a/src/test_onefile.c
+++ b/src/test_onefile.c
@@ -508,7 +508,7 @@ static int fsSync(sqlite3_file *pFile, int flags){
   if( p->eType==DATABASE_FILE ){
     unsigned char zSize[4];
     zSize[0] = (pReal->nDatabase&0xFF000000)>>24;
-    zSize[1] = (pReal->nDatabase&0x00FF0000)>>16;
+    zSize[1] = (unsigned char)((pReal->nDatabase&0x00FF0000)>>16);
     zSize[2] = (pReal->nDatabase&0x0000FF00)>>8;
     zSize[3] = (pReal->nDatabase&0x000000FF);
     rc = pRealFile->pMethods->xWrite(pRealFile, zSize, 4, 0);
diff --git a/src/test_osinst.c b/src/test_osinst.c
index 4ae23a87..a008baba 100644
--- a/src/test_osinst.c
+++ b/src/test_osinst.c
@@ -644,9 +644,9 @@ static void vfslog_flush(VfslogVfs *p){
 
 static void put32bits(unsigned char *p, unsigned int v){
   p[0] = v>>24;
-  p[1] = v>>16;
-  p[2] = v>>8;
-  p[3] = v;
+  p[1] = (unsigned char)(v>>16);
+  p[2] = (unsigned char)(v>>8);
+  p[3] = (unsigned char)v;
 }
 
 static void vfslog_call(
@@ -1104,9 +1104,16 @@ int sqlite3_vfslog_register(sqlite3 *db){
 
 #if defined(SQLITE_TEST) || defined(TCLSH)
 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
-static int test_vfslog(
+static int SQLITE_TCLAPI test_vfslog(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_quota.c b/src/test_quota.c
index e8e0b340..e87e9772 100644
--- a/src/test_quota.c
+++ b/src/test_quota.c
@@ -111,7 +111,7 @@ struct quotaFile {
 
 /*
 ** An instance of the following object represents each open connection
-** to a file that participates in quota tracking.  This object is a 
+** to a file that participates in quota tracking.  This object is a
 ** subclass of sqlite3_file.  The sqlite3_file object for the underlying
 ** VFS is appended to this structure.
 */
@@ -154,11 +154,11 @@ static struct {
   */
   sqlite3_vfs sThisVfs;
 
-  /* The sIoMethods defines the methods used by sqlite3_file objects 
+  /* The sIoMethods defines the methods used by sqlite3_file objects
   ** associated with this shim.  It is initialized at start-time and does
   ** not require a mutex.
   **
-  ** When the underlying VFS is called to open a file, it might return 
+  ** When the underlying VFS is called to open a file, it might return
   ** either a version 1 or a version 2 sqlite3_file object.  This shim
   ** has to create a wrapper sqlite3_file of the same version.  Hence
   ** there are two I/O method structures, one for version 1 and the other
@@ -190,7 +190,7 @@ static struct {
 static void quotaEnter(void){ sqlite3_mutex_enter(gQuota.pMutex); }
 static void quotaLeave(void){ sqlite3_mutex_leave(gQuota.pMutex); }
 
-/* Count the number of open files in a quotaGroup 
+/* Count the number of open files in a quotaGroup
 */
 static int quotaGroupOpenFileCount(quotaGroup *pGroup){
   int N = 0;
@@ -399,7 +399,7 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){
   return zMbcs;
 #else
   return (char*)zUtf8;  /* No-op on unix */
-#endif  
+#endif
 }
 
 /*
@@ -410,7 +410,7 @@ static void quota_mbcs_free(char *zOld){
   sqlite3_free(zOld);
 #else
   /* No-op on unix */
-#endif  
+#endif
 }
 
 /************************* VFS Method Wrappers *****************************/
@@ -428,7 +428,7 @@ static int quotaOpen(
   int flags,                  /* Flags to control the opening */
   int *pOutFlags              /* Flags showing results of opening */
 ){
-  int rc;                                    /* Result code */         
+  int rc;                                    /* Result code */
   quotaConn *pQuotaOpen;                     /* The new quota file descriptor */
   quotaFile *pFile;                          /* Corresponding quotaFile obj */
   quotaGroup *pGroup;                        /* The group file belongs to */
@@ -488,7 +488,7 @@ static int quotaDelete(
   const char *zName,          /* Name of file to be deleted */
   int syncDir                 /* Do a directory sync after deleting */
 ){
-  int rc;                                    /* Result code */         
+  int rc;                                    /* Result code */
   quotaFile *pFile;                          /* Files in the quota */
   quotaGroup *pGroup;                        /* The group file belongs to */
   sqlite3_vfs *pOrigVfs = gQuota.pOrigVfs;   /* Real VFS */
@@ -581,7 +581,7 @@ static int quotaWrite(
     szNew = pGroup->iSize - pFile->iSize + iEnd;
     if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
       if( pGroup->xCallback ){
-        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew, 
+        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew,
                           pGroup->pArg);
       }
       if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
@@ -738,7 +738,7 @@ static int quotaShmUnmap(sqlite3_file *pConn, int deleteFlag){
 /*
 ** Initialize the quota VFS shim.  Use the VFS named zOrigVfsName
 ** as the VFS that does the actual work.  Use the default if
-** zOrigVfsName==NULL.  
+** zOrigVfsName==NULL.
 **
 ** The quota VFS shim is named "quota".  It will become the default
 ** VFS if makeDefault is non-zero.
@@ -908,7 +908,7 @@ int sqlite3_quota_file(const char *zFilename){
 
   if( rc==SQLITE_OK ){
     zFull[strlen(zFull)+1] = '\0';
-    rc = quotaOpen(&gQuota.sThisVfs, zFull, fd, 
+    rc = quotaOpen(&gQuota.sThisVfs, zFull, fd,
                    SQLITE_OPEN_READONLY | SQLITE_OPEN_MAIN_DB, &outFlags);
     if( rc==SQLITE_OK ){
       fd->pMethods->xFileSize(fd, &iSize);
@@ -1016,7 +1016,7 @@ size_t sqlite3_quota_fwrite(
     szNew = pGroup->iSize - pFile->iSize + iEnd;
     if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
       if( pGroup->xCallback ){
-        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew, 
+        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew,
                           pGroup->pArg);
       }
       if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
@@ -1203,7 +1203,7 @@ sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE *p){
 sqlite3_int64 sqlite3_quota_file_size(quota_FILE *p){
   return p->pFile ? p->pFile->iSize : -1;
 }
- 
+
 /*
 ** Determine the amount of data in bytes available for reading
 ** in the given file.
@@ -1275,10 +1275,17 @@ int sqlite3_quota_remove(const char *zFilename){
   sqlite3_free(zFull);
   return rc;
 }
-  
+
 /***************************** Test Code ***********************************/
 #ifdef SQLITE_TEST
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 /*
 ** Argument passed to a TCL quota-over-limit callback.
@@ -1350,7 +1357,7 @@ static void tclCallbackDestructor(void *pObj){
 /*
 ** tclcmd: sqlite3_quota_initialize NAME MAKEDEFAULT
 */
-static int test_quota_initialize(
+static int SQLITE_TCLAPI test_quota_initialize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1379,7 +1386,7 @@ static int test_quota_initialize(
 /*
 ** tclcmd: sqlite3_quota_shutdown
 */
-static int test_quota_shutdown(
+static int SQLITE_TCLAPI test_quota_shutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1402,7 +1409,7 @@ static int test_quota_shutdown(
 /*
 ** tclcmd: sqlite3_quota_set PATTERN LIMIT SCRIPT
 */
-static int test_quota_set(
+static int SQLITE_TCLAPI test_quota_set(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1456,7 +1463,7 @@ static int test_quota_set(
 /*
 ** tclcmd: sqlite3_quota_file FILENAME
 */
-static int test_quota_file(
+static int SQLITE_TCLAPI test_quota_file(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1482,7 +1489,7 @@ static int test_quota_file(
 /*
 ** tclcmd:  sqlite3_quota_dump
 */
-static int test_quota_dump(
+static int SQLITE_TCLAPI test_quota_dump(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1530,7 +1537,7 @@ static int test_quota_dump(
 /*
 ** tclcmd: sqlite3_quota_fopen FILENAME MODE
 */
-static int test_quota_fopen(
+static int SQLITE_TCLAPI test_quota_fopen(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1560,7 +1567,7 @@ extern void *sqlite3TestTextToPtr(const char*);
 /*
 ** tclcmd: sqlite3_quota_fread HANDLE SIZE NELEM
 */
-static int test_quota_fread(
+static int SQLITE_TCLAPI test_quota_fread(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1594,7 +1601,7 @@ static int test_quota_fread(
 /*
 ** tclcmd: sqlite3_quota_fwrite HANDLE SIZE NELEM CONTENT
 */
-static int test_quota_fwrite(
+static int SQLITE_TCLAPI test_quota_fwrite(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1622,7 +1629,7 @@ static int test_quota_fwrite(
 /*
 ** tclcmd: sqlite3_quota_fclose HANDLE
 */
-static int test_quota_fclose(
+static int SQLITE_TCLAPI test_quota_fclose(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1644,7 +1651,7 @@ static int test_quota_fclose(
 /*
 ** tclcmd: sqlite3_quota_fflush HANDLE ?HARDSYNC?
 */
-static int test_quota_fflush(
+static int SQLITE_TCLAPI test_quota_fflush(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1670,7 +1677,7 @@ static int test_quota_fflush(
 /*
 ** tclcmd: sqlite3_quota_fseek HANDLE OFFSET WHENCE
 */
-static int test_quota_fseek(
+static int SQLITE_TCLAPI test_quota_fseek(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1708,7 +1715,7 @@ static int test_quota_fseek(
 /*
 ** tclcmd: sqlite3_quota_rewind HANDLE
 */
-static int test_quota_rewind(
+static int SQLITE_TCLAPI test_quota_rewind(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1727,7 +1734,7 @@ static int test_quota_rewind(
 /*
 ** tclcmd: sqlite3_quota_ftell HANDLE
 */
-static int test_quota_ftell(
+static int SQLITE_TCLAPI test_quota_ftell(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1748,7 +1755,7 @@ static int test_quota_ftell(
 /*
 ** tclcmd: sqlite3_quota_ftruncate HANDLE SIZE
 */
-static int test_quota_ftruncate(
+static int SQLITE_TCLAPI test_quota_ftruncate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1773,7 +1780,7 @@ static int test_quota_ftruncate(
 /*
 ** tclcmd: sqlite3_quota_file_size HANDLE
 */
-static int test_quota_file_size(
+static int SQLITE_TCLAPI test_quota_file_size(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1794,7 +1801,7 @@ static int test_quota_file_size(
 /*
 ** tclcmd: sqlite3_quota_file_truesize HANDLE
 */
-static int test_quota_file_truesize(
+static int SQLITE_TCLAPI test_quota_file_truesize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1815,7 +1822,7 @@ static int test_quota_file_truesize(
 /*
 ** tclcmd: sqlite3_quota_file_mtime HANDLE
 */
-static int test_quota_file_mtime(
+static int SQLITE_TCLAPI test_quota_file_mtime(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1838,7 +1845,7 @@ static int test_quota_file_mtime(
 /*
 ** tclcmd: sqlite3_quota_remove FILENAME
 */
-static int test_quota_remove(
+static int SQLITE_TCLAPI test_quota_remove(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1862,7 +1869,7 @@ static int test_quota_remove(
 ** Test the glob pattern matching.  Return 1 if TEXT matches PATTERN
 ** and return 0 if it does not.
 */
-static int test_quota_glob(
+static int SQLITE_TCLAPI test_quota_glob(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1888,7 +1895,7 @@ static int test_quota_glob(
 ** Return the number of bytes from the current file point to the end of
 ** the file.
 */
-static int test_quota_file_available(
+static int SQLITE_TCLAPI test_quota_file_available(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1911,7 +1918,7 @@ static int test_quota_file_available(
 **
 ** Return true if the file handle is in the error state.
 */
-static int test_quota_ferror(
+static int SQLITE_TCLAPI test_quota_ferror(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_rtree.c b/src/test_rtree.c
index 797ec002..0c6dbf3c 100644
--- a/src/test_rtree.c
+++ b/src/test_rtree.c
@@ -14,7 +14,11 @@
 */
 
 #include "sqlite3.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 /* Solely for the UNUSED_PARAMETER() macro. */
 #include "sqliteInt.h"
@@ -353,7 +357,11 @@ static int bfs_query_func(sqlite3_rtree_query_info *p){
 *************************************************************************/
 
 #include 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 typedef struct Cube Cube;
 struct Cube {
@@ -432,7 +440,7 @@ static int cube_geom(
 }
 #endif /* SQLITE_ENABLE_RTREE */
 
-static int register_cube_geom(
+static int SQLITE_TCLAPI register_cube_geom(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -460,7 +468,7 @@ static int register_cube_geom(
   return TCL_OK;
 }
 
-static int register_circle_geom(
+static int SQLITE_TCLAPI register_circle_geom(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_schema.c b/src/test_schema.c
index 4ee18193..cdf08579 100644
--- a/src/test_schema.c
+++ b/src/test_schema.c
@@ -35,10 +35,14 @@
 ** to be compiled into an sqlite dynamic extension.
 */
 #ifdef SQLITE_TEST
-  #include "sqliteInt.h"
-  #include "tcl.h"
+#  include "sqliteInt.h"
+#  if defined(INCLUDE_SQLITE_TCL_H)
+#    include "sqlite_tcl.h"
+#  else
+#    include "tcl.h"
+#  endif
 #else
-  #include "sqlite3ext.h"
+#  include "sqlite3ext.h"
   SQLITE_EXTENSION_INIT1
 #endif
 
@@ -302,7 +306,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the schema virtual table module.
 */
-static int register_schema_module(
+static int SQLITE_TCLAPI register_schema_module(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_superlock.c b/src/test_superlock.c
index cac78984..45d0d623 100644
--- a/src/test_superlock.c
+++ b/src/test_superlock.c
@@ -256,7 +256,14 @@ int sqlite3demo_superlock(
 
 #ifdef SQLITE_TEST
 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 struct InterpAndScript {
   Tcl_Interp *interp;
@@ -264,11 +271,11 @@ struct InterpAndScript {
 };
 typedef struct InterpAndScript InterpAndScript;
 
-static void superunlock_del(ClientData cd){
+static void SQLITE_TCLAPI superunlock_del(ClientData cd){
   sqlite3demo_superunlock((void *)cd);
 }
 
-static int superunlock_cmd(
+static int SQLITE_TCLAPI superunlock_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -300,7 +307,7 @@ static int superlock_busy(void *pCtx, int nBusy){
 /*
 ** Tclcmd: sqlite3demo_superlock CMDNAME PATH VFS BUSY-HANDLER-SCRIPT
 */
-static int superlock_cmd(
+static int SQLITE_TCLAPI superlock_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_syscall.c b/src/test_syscall.c
index 0dac2e89..947f9a9d 100644
--- a/src/test_syscall.c
+++ b/src/test_syscall.c
@@ -76,7 +76,11 @@
 
 #include "sqliteInt.h"
 #include "sqlite3.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -108,10 +112,13 @@ static int ts_ftruncate(int fd, off_t n);
 static int ts_fcntl(int fd, int cmd, ... );
 static int ts_read(int fd, void *aBuf, size_t nBuf);
 static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off);
-static int ts_pread64(int fd, void *aBuf, size_t nBuf, off_t off);
+/* Note:  pread64() and pwrite64() actually use off64_t as the type on their
+** last parameter.  But that datatype is not defined on many systems 
+** (ex: Mac, OpenBSD).  So substitute a likely equivalent: sqlite3_uint64 */
+static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off);
 static int ts_write(int fd, const void *aBuf, size_t nBuf);
 static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off);
-static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off);
+static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off);
 static int ts_fchmod(int fd, mode_t mode);
 static int ts_fallocate(int fd, off_t off, off_t len);
 static void *ts_mmap(void *, size_t, int, int, int, off_t);
@@ -155,11 +162,11 @@ struct TestSyscallArray {
 #define orig_fcntl     ((int(*)(int,int,...))aSyscall[7].xOrig)
 #define orig_read      ((ssize_t(*)(int,void*,size_t))aSyscall[8].xOrig)
 #define orig_pread     ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[9].xOrig)
-#define orig_pread64   ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].xOrig)
+#define orig_pread64   ((ssize_t(*)(int,void*,size_t,sqlite3_uint64))aSyscall[10].xOrig)
 #define orig_write     ((ssize_t(*)(int,const void*,size_t))aSyscall[11].xOrig)
 #define orig_pwrite    ((ssize_t(*)(int,const void*,size_t,off_t))\
                        aSyscall[12].xOrig)
-#define orig_pwrite64  ((ssize_t(*)(int,const void*,size_t,off_t))\
+#define orig_pwrite64  ((ssize_t(*)(int,const void*,size_t,sqlite3_uint64))\
                        aSyscall[13].xOrig)
 #define orig_fchmod    ((int(*)(int,mode_t))aSyscall[14].xOrig)
 #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig)
@@ -326,7 +333,7 @@ static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){
 /*
 ** A wrapper around pread64().
 */
-static int ts_pread64(int fd, void *aBuf, size_t nBuf, off_t off){
+static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){
   if( tsIsFailErrno("pread64") ){
     return -1;
   }
@@ -357,7 +364,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){
 /*
 ** A wrapper around pwrite64().
 */
-static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off){
+static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){
   if( tsIsFailErrno("pwrite64") ){
     return -1;
   }
@@ -415,7 +422,7 @@ static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){
   return orig_mremap(a, b, c, d, pArg);
 }
 
-static int test_syscall_install(
+static int SQLITE_TCLAPI test_syscall_install(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -451,7 +458,7 @@ static int test_syscall_install(
   return TCL_OK;
 }
 
-static int test_syscall_uninstall(
+static int SQLITE_TCLAPI test_syscall_uninstall(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -475,7 +482,7 @@ static int test_syscall_uninstall(
   return TCL_OK;
 }
 
-static int test_syscall_reset(
+static int SQLITE_TCLAPI test_syscall_reset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -513,7 +520,7 @@ static int test_syscall_reset(
   return TCL_OK;
 }
 
-static int test_syscall_exists(
+static int SQLITE_TCLAPI test_syscall_exists(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -534,7 +541,7 @@ static int test_syscall_exists(
   return TCL_OK;
 }
 
-static int test_syscall_fault(
+static int SQLITE_TCLAPI test_syscall_fault(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -563,7 +570,7 @@ static int test_syscall_fault(
   return TCL_OK;
 }
 
-static int test_syscall_errno(
+static int SQLITE_TCLAPI test_syscall_errno(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -609,7 +616,7 @@ static int test_syscall_errno(
   return TCL_OK;
 }
 
-static int test_syscall_list(
+static int SQLITE_TCLAPI test_syscall_list(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -639,7 +646,7 @@ static int test_syscall_list(
   return TCL_OK;
 }
 
-static int test_syscall_defaultvfs(
+static int SQLITE_TCLAPI test_syscall_defaultvfs(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -661,7 +668,7 @@ static int ts_getpagesize(void){
   return gSyscall.pgsz;
 }
 
-static int test_syscall_pagesize(
+static int SQLITE_TCLAPI test_syscall_pagesize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -696,7 +703,7 @@ static int test_syscall_pagesize(
   return TCL_OK;
 }
 
-static int test_syscall(
+static int SQLITE_TCLAPI test_syscall(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -719,14 +726,20 @@ static int test_syscall(
   };
   int iCmd;
   int rc;
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
 
   if( objc<2 ){
     Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ...");
     return TCL_ERROR;
   }
-  rc = Tcl_GetIndexFromObjStruct(interp, 
-      objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd
-  );
+  if( pVfs->iVersion<3 || pVfs->xSetSystemCall==0 ){
+    Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", 0);
+    rc = TCL_ERROR;
+  }else{
+    rc = Tcl_GetIndexFromObjStruct(interp, 
+        objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd
+    );
+  }
   if( rc!=TCL_OK ) return rc;
   return aCmd[iCmd].xCmd(clientData, interp, objc, objv);
 }
diff --git a/src/test_tclvar.c b/src/test_tclvar.c
index 63ed3947..8f735211 100644
--- a/src/test_tclvar.c
+++ b/src/test_tclvar.c
@@ -17,7 +17,11 @@
 ** access to TCL variables.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -407,7 +411,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the echo virtual table module.
 */
-static int register_tclvar_module(
+static int SQLITE_TCLAPI register_tclvar_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_thread.c b/src/test_thread.c
index a4d96e19..20b4cf14 100644
--- a/src/test_thread.c
+++ b/src/test_thread.c
@@ -16,7 +16,11 @@
 */
 
 #include "sqliteInt.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #if SQLITE_THREADSAFE
 
@@ -72,7 +76,7 @@ extern int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int);
 /*
 ** Handler for events of type EvalEvent.
 */
-static int tclScriptEvent(Tcl_Event *evPtr, int flags){
+static int SQLITE_TCLAPI tclScriptEvent(Tcl_Event *evPtr, int flags){
   int rc;
   EvalEvent *p = (EvalEvent *)evPtr;
   rc = Tcl_Eval(p->interp, p->zScript);
@@ -167,7 +171,7 @@ static Tcl_ThreadCreateType tclScriptThread(ClientData pSqlThread){
 **
 **     The caller can wait for the script to terminate using [vwait VARNAME].
 */
-static int sqlthread_spawn(
+static int SQLITE_TCLAPI sqlthread_spawn(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -220,7 +224,7 @@ static int sqlthread_spawn(
 **
 **     NOTE: At the moment, this doesn't work. FIXME.
 */
-static int sqlthread_parent(
+static int SQLITE_TCLAPI sqlthread_parent(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -265,7 +269,7 @@ static int xBusy(void *pArg, int nBusy){
 **     Open a database handle and return the string representation of
 **     the pointer value.
 */
-static int sqlthread_open(
+static int SQLITE_TCLAPI sqlthread_open(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -276,7 +280,7 @@ static int sqlthread_open(
   const char *zFilename;
   sqlite3 *db;
   char zBuf[100];
-  extern void Md5_Register(sqlite3*);
+  extern int Md5_Register(sqlite3*,char**,const sqlite3_api_routines*);
 
   UNUSED_PARAMETER(clientData);
   UNUSED_PARAMETER(objc);
@@ -299,7 +303,7 @@ static int sqlthread_open(
     }
   }
 #endif
-  Md5_Register(db);
+  Md5_Register(db, 0, 0);
   sqlite3_busy_handler(db, xBusy, 0);
   
   if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR;
@@ -315,7 +319,7 @@ static int sqlthread_open(
 **     Return the current thread-id (Tcl_GetCurrentThread()) cast to
 **     an integer.
 */
-static int sqlthread_id(
+static int SQLITE_TCLAPI sqlthread_id(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -333,7 +337,7 @@ static int sqlthread_id(
 /*
 ** Dispatch routine for the sub-commands of [sqlthread].
 */
-static int sqlthread_proc(
+static int SQLITE_TCLAPI sqlthread_proc(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -381,7 +385,7 @@ static int sqlthread_proc(
 ** implemented as a script in Tcl 8.5, it is not usually available to
 ** testfixture.
 */ 
-static int clock_seconds_proc(
+static int SQLITE_TCLAPI clock_seconds_proc(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -543,7 +547,7 @@ int sqlite3_blocking_prepare_v2(
 **
 ** Advance the statement to the next row.
 */
-static int blocking_step_proc(
+static int SQLITE_TCLAPI blocking_step_proc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -569,7 +573,7 @@ static int blocking_step_proc(
 ** Usage: sqlite3_blocking_prepare_v2 DB sql bytes ?tailvar?
 ** Usage: sqlite3_nonblocking_prepare_v2 DB sql bytes ?tailvar?
 */
-static int blocking_prepare_v2_proc(
+static int SQLITE_TCLAPI blocking_prepare_v2_proc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_vfs.c b/src/test_vfs.c
index a8c6ae7b..fb987a61 100644
--- a/src/test_vfs.c
+++ b/src/test_vfs.c
@@ -28,7 +28,11 @@
 
 #include "sqlite3.h"
 #include "sqliteInt.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 typedef struct Testvfs Testvfs;
 typedef struct TestvfsShm TestvfsShm;
@@ -306,7 +310,6 @@ static void tvfsExecTcl(
 ** Close an tvfs-file.
 */
 static int tvfsClose(sqlite3_file *pFile){
-  int rc;
   TestvfsFile *pTestfile = (TestvfsFile *)pFile;
   TestvfsFd *pFd = pTestfile->pFd;
   Testvfs *p = (Testvfs *)pFd->pVfs->pAppData;
@@ -324,10 +327,10 @@ static int tvfsClose(sqlite3_file *pFile){
   if( pFile->pMethods ){
     ckfree((char *)pFile->pMethods);
   }
-  rc = sqlite3OsClose(pFd->pReal);
+  sqlite3OsClose(pFd->pReal);
   ckfree((char *)pFd);
   pTestfile->pFd = 0;
-  return rc;
+  return SQLITE_OK;
 }
 
 /*
@@ -1037,7 +1040,7 @@ static int tvfsUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *p){
   return sqlite3OsUnfetch(pFd->pReal, iOfst, p);
 }
 
-static int testvfs_obj_cmd(
+static int SQLITE_TCLAPI testvfs_obj_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -1349,7 +1352,7 @@ static int testvfs_obj_cmd(
   return TCL_OK;
 }
 
-static void testvfs_obj_del(ClientData cd){
+static void SQLITE_TCLAPI testvfs_obj_del(ClientData cd){
   Testvfs *p = (Testvfs *)cd;
   if( p->pScript ) Tcl_DecrRefCount(p->pScript);
   sqlite3_vfs_unregister(p->pVfs);
@@ -1392,7 +1395,7 @@ static void testvfs_obj_del(ClientData cd){
 **
 ** where LOCK is of the form "OFFSET NBYTE lock/unlock shared/exclusive"
 */
-static int testvfs_cmd(
+static int SQLITE_TCLAPI testvfs_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_windirent.c b/src/test_windirent.c
index 11d7dc07..044bc7f4 100644
--- a/src/test_windirent.c
+++ b/src/test_windirent.c
@@ -17,6 +17,34 @@
 
 #include "test_windirent.h"
 
+/*
+** Implementation of the POSIX getenv() function using the Win32 API.
+** This function is not thread-safe.
+*/
+const char *windirent_getenv(
+  const char *name
+){
+  static char value[32768]; /* Maximum length, per MSDN */
+  DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */
+  DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */
+
+  memset(value, 0, sizeof(value));
+  dwRet = GetEnvironmentVariableA(name, value, dwSize);
+  if( dwRet==0 || dwRet>dwSize ){
+    /*
+    ** The function call to GetEnvironmentVariableA() failed -OR-
+    ** the buffer is not large enough.  Either way, return NULL.
+    */
+    return 0;
+  }else{
+    /*
+    ** The function call to GetEnvironmentVariableA() succeeded
+    ** -AND- the buffer contains the entire value.
+    */
+    return value;
+  }
+}
+
 /*
 ** Implementation of the POSIX opendir() function using the MSVCRT.
 */
@@ -32,7 +60,7 @@ LPDIR opendir(
 
   /* TODO: Remove this if Unix-style root paths are not used. */
   if( sqlite3_stricmp(dirname, "/")==0 ){
-    dirname = getenv("SystemDrive");
+    dirname = windirent_getenv("SystemDrive");
   }
 
   _snprintf(data.name, namesize, "%s\\*", dirname);
diff --git a/src/test_windirent.h b/src/test_windirent.h
index 0b8d1a7b..be454988 100644
--- a/src/test_windirent.h
+++ b/src/test_windirent.h
@@ -92,6 +92,13 @@ struct DIR {
   DIRENT d_next;     /* DIRENT constructed based on "_findnext". */
 };
 
+/*
+** Provide the function prototype for the POSIX compatiable getenv()
+** function.  This function is not thread-safe.
+*/
+
+extern const char *windirent_getenv(const char *name);
+
 /*
 ** Finally, we can provide the function prototypes for the opendir(),
 ** readdir(), readdir_r(), and closedir() POSIX functions.
diff --git a/src/threads.c b/src/threads.c
index 251b9b76..f128d69f 100644
--- a/src/threads.c
+++ b/src/threads.c
@@ -63,7 +63,7 @@ int sqlite3ThreadCreate(
 
   *ppThread = 0;
   p = sqlite3Malloc(sizeof(*p));
-  if( p==0 ) return SQLITE_NOMEM;
+  if( p==0 ) return SQLITE_NOMEM_BKPT;
   memset(p, 0, sizeof(*p));
   p->xTask = xTask;
   p->pIn = pIn;
@@ -89,7 +89,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){
   int rc;
 
   assert( ppOut!=0 );
-  if( NEVER(p==0) ) return SQLITE_NOMEM;
+  if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT;
   if( p->done ){
     *ppOut = p->pOut;
     rc = SQLITE_OK;
@@ -154,7 +154,7 @@ int sqlite3ThreadCreate(
   assert( xTask!=0 );
   *ppThread = 0;
   p = sqlite3Malloc(sizeof(*p));
-  if( p==0 ) return SQLITE_NOMEM;
+  if( p==0 ) return SQLITE_NOMEM_BKPT;
   /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a 
   ** function that returns SQLITE_ERROR when passed the argument 200, that
   ** forces worker threads to run sequentially and deterministically 
@@ -186,7 +186,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){
   BOOL bRc;
 
   assert( ppOut!=0 );
-  if( NEVER(p==0) ) return SQLITE_NOMEM;
+  if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT;
   if( p->xTask==0 ){
     /* assert( p->id==GetCurrentThreadId() ); */
     rc = WAIT_OBJECT_0;
@@ -234,7 +234,7 @@ int sqlite3ThreadCreate(
   assert( xTask!=0 );
   *ppThread = 0;
   p = sqlite3Malloc(sizeof(*p));
-  if( p==0 ) return SQLITE_NOMEM;
+  if( p==0 ) return SQLITE_NOMEM_BKPT;
   if( (SQLITE_PTR_TO_INT(p)/17)&1 ){
     p->xTask = xTask;
     p->pIn = pIn;
@@ -250,7 +250,7 @@ int sqlite3ThreadCreate(
 int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){
 
   assert( ppOut!=0 );
-  if( NEVER(p==0) ) return SQLITE_NOMEM;
+  if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT;
   if( p->xTask ){
     *ppOut = p->xTask(p->pIn);
   }else{
@@ -261,7 +261,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){
 #if defined(SQLITE_TEST)
   {
     void *pTstAlloc = sqlite3Malloc(10);
-    if (!pTstAlloc) return SQLITE_NOMEM;
+    if (!pTstAlloc) return SQLITE_NOMEM_BKPT;
     sqlite3_free(pTstAlloc);
   }
 #endif
diff --git a/src/tokenize.c b/src/tokenize.c
index 9b3444ac..9230da2a 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -285,7 +285,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
     case CC_BANG: {
       if( z[1]!='=' ){
         *tokenType = TK_ILLEGAL;
-        return 2;
+        return 1;
       }else{
         *tokenType = TK_NE;
         return 2;
@@ -435,8 +435,8 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
       *tokenType = TK_ID;
       return keywordCode((char*)z, i, tokenType);
     }
-#ifndef SQLITE_OMIT_BLOB_LITERAL
     case CC_X: {
+#ifndef SQLITE_OMIT_BLOB_LITERAL
       testcase( z[0]=='x' ); testcase( z[0]=='X' );
       if( z[1]=='\'' ){
         *tokenType = TK_BLOB;
@@ -448,10 +448,10 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
         if( z[i] ) i++;
         return i;
       }
+#endif
       /* If it is not a BLOB literal, then it must be an ID, since no
       ** SQL keywords start with the letter 'x'.  Fall through */
     }
-#endif
     case CC_ID: {
       i = 1;
       break;
@@ -495,21 +495,33 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   pEngine = sqlite3ParserAlloc(sqlite3Malloc);
   if( pEngine==0 ){
     sqlite3OomFault(db);
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   assert( pParse->pNewTable==0 );
   assert( pParse->pNewTrigger==0 );
   assert( pParse->nVar==0 );
   assert( pParse->nzVar==0 );
   assert( pParse->azVar==0 );
-  while( zSql[i]!=0 ){
+  while( 1 ){
     assert( i>=0 );
-    pParse->sLastToken.z = &zSql[i];
-    pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
-    i += pParse->sLastToken.n;
-    if( i>mxSqlLen ){
-      pParse->rc = SQLITE_TOOBIG;
-      break;
+    if( zSql[i]!=0 ){
+      pParse->sLastToken.z = &zSql[i];
+      pParse->sLastToken.n = sqlite3GetToken((u8*)&zSql[i],&tokenType);
+      i += pParse->sLastToken.n;
+      if( i>mxSqlLen ){
+        pParse->rc = SQLITE_TOOBIG;
+        break;
+      }
+    }else{
+      /* Upon reaching the end of input, call the parser two more times
+      ** with tokens TK_SEMI and 0, in that order. */
+      if( lastTokenParsed==TK_SEMI ){
+        tokenType = 0;
+      }else if( lastTokenParsed==0 ){
+        break;
+      }else{
+        tokenType = TK_SEMI;
+      }
     }
     if( tokenType>=TK_SPACE ){
       assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
@@ -523,23 +535,13 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
         break;
       }
     }else{
-      if( tokenType==TK_SEMI ) pParse->zTail = &zSql[i];
       sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
       lastTokenParsed = tokenType;
       if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
     }
   }
   assert( nErr==0 );
-  if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
-    assert( zSql[i]==0 );
-    if( lastTokenParsed!=TK_SEMI ){
-      sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
-      pParse->zTail = &zSql[i];
-    }
-    if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
-      sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
-    }
-  }
+  pParse->zTail = &zSql[i];
 #ifdef YYTRACKMAXSTACKDEPTH
   sqlite3_mutex_enter(sqlite3MallocMutex());
   sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK,
@@ -549,7 +551,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
 #endif /* YYDEBUG */
   sqlite3ParserFree(pEngine, sqlite3_free);
   if( db->mallocFailed ){
-    pParse->rc = SQLITE_NOMEM;
+    pParse->rc = SQLITE_NOMEM_BKPT;
   }
   if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
     pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc));
@@ -584,7 +586,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
     sqlite3DeleteTable(db, pParse->pNewTable);
   }
 
-  sqlite3WithDelete(db, pParse->pWithToFree);
+  if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree);
   sqlite3DeleteTrigger(db, pParse->pNewTrigger);
   for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]);
   sqlite3DbFree(db, pParse->azVar);
diff --git a/src/treeview.c b/src/treeview.c
index 29858043..d3671f2c 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -120,7 +120,7 @@ void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){
 
 
 /*
-** Generate a human-readable description of a the Select object.
+** Generate a human-readable description of a Select object.
 */
 void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
   int n = 0;
@@ -132,9 +132,10 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
     sqlite3TreeViewPush(pView, 1);
   }
   do{
-    sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x",
+    sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d",
       ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
-      ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags
+      ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags,
+      (int)p->nSelectRow
     );
     if( cnt++ ) sqlite3TreeViewPop(pView);
     if( p->pPrior ){
@@ -338,6 +339,12 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
     case TK_ISNULL:  zUniOp = "ISNULL"; break;
     case TK_NOTNULL: zUniOp = "NOTNULL"; break;
 
+    case TK_SPAN: {
+      sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken);
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
+      break;
+    }
+
     case TK_COLLATE: {
       sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken);
       sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
@@ -438,6 +445,21 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       break;
     }
 #endif
+    case TK_MATCH: {
+      sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s",
+                          pExpr->iTable, pExpr->iColumn, zFlgs);
+      sqlite3TreeViewExpr(pView, pExpr->pRight, 0);
+      break;
+    }
+    case TK_VECTOR: {
+      sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR");
+      break;
+    }
+    case TK_SELECT_COLUMN: {
+      sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn);
+      sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0);
+      break;
+    }
     default: {
       sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
       break;
@@ -454,21 +476,20 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
   sqlite3TreeViewPop(pView);
 }
 
+
 /*
 ** Generate a human-readable explanation of an expression list.
 */
-void sqlite3TreeViewExprList(
+void sqlite3TreeViewBareExprList(
   TreeView *pView,
   const ExprList *pList,
-  u8 moreToFollow,
   const char *zLabel
 ){
-  int i;
-  pView = sqlite3TreeViewPush(pView, moreToFollow);
   if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST";
   if( pList==0 ){
     sqlite3TreeViewLine(pView, "%s (empty)", zLabel);
   }else{
+    int i;
     sqlite3TreeViewLine(pView, "%s", zLabel);
     for(i=0; inExpr; i++){
       int j = pList->a[i].u.x.iOrderByCol;
@@ -480,6 +501,15 @@ void sqlite3TreeViewExprList(
       if( j ) sqlite3TreeViewPop(pView);
     }
   }
+}
+void sqlite3TreeViewExprList(
+  TreeView *pView,
+  const ExprList *pList,
+  u8 moreToFollow,
+  const char *zLabel
+){
+  pView = sqlite3TreeViewPush(pView, moreToFollow);
+  sqlite3TreeViewBareExprList(pView, pList, zLabel);
   sqlite3TreeViewPop(pView);
 }
 
diff --git a/src/trigger.c b/src/trigger.c
index 72c31eb2..7eb14bcf 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -96,7 +96,6 @@ void sqlite3BeginTrigger(
   int iDb;                /* The database to store the trigger in */
   Token *pName;           /* The unqualified db name */
   DbFixer sFix;           /* State vector for the DB fixer */
-  int iTabDb;             /* Index of the database holding pTab */
 
   assert( pName1!=0 );   /* pName1->z might be NULL, but not pName1 itself */
   assert( pName2!=0 );
@@ -209,13 +208,13 @@ void sqlite3BeginTrigger(
         " trigger on table: %S", pTableName, 0);
     goto trigger_cleanup;
   }
-  iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema);
 
 #ifndef SQLITE_OMIT_AUTHORIZATION
   {
+    int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema);
     int code = SQLITE_CREATE_TRIGGER;
-    const char *zDb = db->aDb[iTabDb].zName;
-    const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb;
+    const char *zDb = db->aDb[iTabDb].zDbSName;
+    const char *zDbTrig = isTemp ? db->aDb[1].zDbSName : zDb;
     if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER;
     if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){
       goto trigger_cleanup;
@@ -309,7 +308,7 @@ void sqlite3FinishTrigger(
     z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n);
     sqlite3NestedParse(pParse,
        "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
-       db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
+       db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), zName,
        pTrig->table, z);
     sqlite3DbFree(db, z);
     sqlite3ChangeCookie(pParse, iDb);
@@ -498,7 +497,7 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){
   assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) );
   for(i=OMIT_TEMPDB; inDb; i++){
     int j = (i<2) ? i^1 : i;  /* Search TEMP before MAIN */
-    if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue;
+    if( zDb && sqlite3StrICmp(db->aDb[j].zDbSName, zDb) ) continue;
     assert( sqlite3SchemaMutexHeld(db, j, 0) );
     pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName);
     if( pTrigger ) break;
@@ -544,7 +543,7 @@ void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){
 #ifndef SQLITE_OMIT_AUTHORIZATION
   {
     int code = SQLITE_DROP_TRIGGER;
-    const char *zDb = db->aDb[iDb].zName;
+    const char *zDb = db->aDb[iDb].zDbSName;
     const char *zTab = SCHEMA_TABLE(iDb);
     if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER;
     if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) ||
@@ -560,7 +559,7 @@ void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){
   if( (v = sqlite3GetVdbe(pParse))!=0 ){
     sqlite3NestedParse(pParse,
        "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'",
-       db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pTrigger->zName
+       db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb), pTrigger->zName
     );
     sqlite3ChangeCookie(pParse, iDb);
     sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0);
@@ -663,8 +662,10 @@ static SrcList *targetSrcList(
     pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget);
     iDb = sqlite3SchemaToIndex(db, pStep->pTrig->pSchema);
     if( iDb==0 || iDb>=2 ){
+      const char *zDb;
       assert( iDbnDb );
-      pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName);
+      zDb = db->aDb[iDb].zDbSName;
+      pSrc->a[pSrc->nSrc-1].zDatabase =  sqlite3DbStrDup(db, zDb);
     }
   }
   return pSrc;
@@ -878,7 +879,6 @@ static TriggerPrg *codeRowTrigger(
     }
     pProgram->nMem = pSubParse->nMem;
     pProgram->nCsr = pSubParse->nTab;
-    pProgram->nOnce = pSubParse->nOnce;
     pProgram->token = (void *)pTrigger;
     pPrg->aColmask[0] = pSubParse->oldmask;
     pPrg->aColmask[1] = pSubParse->newmask;
diff --git a/src/update.c b/src/update.c
index 2a436b97..15e58e34 100644
--- a/src/update.c
+++ b/src/update.c
@@ -249,7 +249,7 @@ void sqlite3Update(
       int rc;
       rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName,
                             j<0 ? "ROWID" : pTab->aCol[j].zName,
-                            db->aDb[iDb].zName);
+                            db->aDb[iDb].zDbSName);
       if( rc==SQLITE_DENY ){
         goto update_cleanup;
       }else if( rc==SQLITE_IGNORE ){
@@ -268,7 +268,7 @@ void sqlite3Update(
   ** case, set all bits of the colUsed mask (to ensure that the virtual
   ** table implementation makes all columns available).
   */
-  pTabList->a[0].colUsed = IsVirtual(pTab) ? (Bitmask)-1 : 0;
+  pTabList->a[0].colUsed = IsVirtual(pTab) ? ALLBITS : 0;
 
   hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey);
 
@@ -352,7 +352,8 @@ void sqlite3Update(
   if( HasRowid(pTab) ){
     sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
     pWInfo = sqlite3WhereBegin(
-        pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur
+        pParse, pTabList, pWhere, 0, 0,
+            WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE, iIdxCur
     );
     if( pWInfo==0 ) goto update_cleanup;
     okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
@@ -590,11 +591,30 @@ void sqlite3Update(
       VdbeCoverageNeverTaken(v);
     }
     sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1);
-  
-    /* If changing the record number, delete the old record.  */
+
+    /* If changing the rowid value, or if there are foreign key constraints
+    ** to process, delete the old record. Otherwise, add a noop OP_Delete
+    ** to invoke the pre-update hook.
+    **
+    ** That (regNew==regnewRowid+1) is true is also important for the 
+    ** pre-update hook. If the caller invokes preupdate_new(), the returned
+    ** value is copied from memory cell (regNewRowid+1+iCol), where iCol
+    ** is the column index supplied by the user.
+    */
+    assert( regNew==regNewRowid+1 );
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+    sqlite3VdbeAddOp3(v, OP_Delete, iDataCur,
+        OPFLAG_ISUPDATE | ((hasFK || chngKey || pPk!=0) ? 0 : OPFLAG_ISNOOP),
+        regNewRowid
+    );
+    if( !pParse->nested ){
+      sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE);
+    }
+#else
     if( hasFK || chngKey || pPk!=0 ){
       sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0);
     }
+#endif
     if( bReplace || chngKey ){
       sqlite3VdbeJumpHere(v, addr1);
     }
diff --git a/src/utf.c b/src/utf.c
index e42ab418..7801306c 100644
--- a/src/utf.c
+++ b/src/utf.c
@@ -231,7 +231,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
     rc = sqlite3VdbeMemMakeWriteable(pMem);
     if( rc!=SQLITE_OK ){
       assert( rc==SQLITE_NOMEM );
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }
     zIn = (u8*)pMem->z;
     zTerm = &zIn[pMem->n&~1];
@@ -273,7 +273,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){
   zTerm = &zIn[pMem->n];
   zOut = sqlite3DbMallocRaw(pMem->db, len);
   if( !zOut ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   z = zOut;
 
diff --git a/src/util.c b/src/util.c
index 37b585b2..e18b8475 100644
--- a/src/util.c
+++ b/src/util.c
@@ -109,13 +109,49 @@ int sqlite3Strlen30(const char *z){
   return 0x3fffffff & (int)strlen(z);
 }
 
+/*
+** Return the declared type of a column.  Or return zDflt if the column 
+** has no declared type.
+**
+** The column type is an extra string stored after the zero-terminator on
+** the column name if and only if the COLFLAG_HASTYPE flag is set.
+*/
+char *sqlite3ColumnType(Column *pCol, char *zDflt){
+  if( (pCol->colFlags & COLFLAG_HASTYPE)==0 ) return zDflt;
+  return pCol->zName + strlen(pCol->zName) + 1;
+}
+
+/*
+** Helper function for sqlite3Error() - called rarely.  Broken out into
+** a separate routine to avoid unnecessary register saves on entry to
+** sqlite3Error().
+*/
+static SQLITE_NOINLINE void  sqlite3ErrorFinish(sqlite3 *db, int err_code){
+  if( db->pErr ) sqlite3ValueSetNull(db->pErr);
+  sqlite3SystemError(db, err_code);
+}
+
 /*
 ** Set the current error code to err_code and clear any prior error message.
+** Also set iSysErrno (by calling sqlite3System) if the err_code indicates
+** that would be appropriate.
 */
 void sqlite3Error(sqlite3 *db, int err_code){
   assert( db!=0 );
   db->errCode = err_code;
-  if( db->pErr ) sqlite3ValueSetNull(db->pErr);
+  if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code);
+}
+
+/*
+** Load the sqlite3.iSysErrno field if that is an appropriate thing
+** to do based on the SQLite error code in rc.
+*/
+void sqlite3SystemError(sqlite3 *db, int rc){
+  if( rc==SQLITE_IOERR_NOMEM ) return;
+  rc &= 0xff;
+  if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){
+    db->iSysErrno = sqlite3OsGetLastError(db->pVfs);
+  }
 }
 
 /*
@@ -142,6 +178,7 @@ void sqlite3Error(sqlite3 *db, int err_code){
 void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){
   assert( db!=0 );
   db->errCode = err_code;
+  sqlite3SystemError(db, err_code);
   if( zFormat==0 ){
     sqlite3Error(db, err_code);
   }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){
@@ -205,18 +242,13 @@ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){
 ** brackets from around identifiers.  For example:  "[a-b-c]" becomes
 ** "a-b-c".
 */
-int sqlite3Dequote(char *z){
+void sqlite3Dequote(char *z){
   char quote;
   int i, j;
-  if( z==0 ) return -1;
+  if( z==0 ) return;
   quote = z[0];
-  switch( quote ){
-    case '\'':  break;
-    case '"':   break;
-    case '`':   break;                /* For MySQL compatibility */
-    case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
-    default:    return -1;
-  }
+  if( !sqlite3Isquote(quote) ) return;
+  if( quote=='[' ) quote = ']';
   for(i=1, j=0;; i++){
     assert( z[i] );
     if( z[i]==quote ){
@@ -231,7 +263,6 @@ int sqlite3Dequote(char *z){
     }
   }
   z[j] = 0;
-  return j;
 }
 
 /*
@@ -256,16 +287,25 @@ void sqlite3TokenInit(Token *p, char *z){
 ** independence" that SQLite uses internally when comparing identifiers.
 */
 int sqlite3_stricmp(const char *zLeft, const char *zRight){
-  register unsigned char *a, *b;
   if( zLeft==0 ){
     return zRight ? -1 : 0;
   }else if( zRight==0 ){
     return 1;
   }
+  return sqlite3StrICmp(zLeft, zRight);
+}
+int sqlite3StrICmp(const char *zLeft, const char *zRight){
+  unsigned char *a, *b;
+  int c;
   a = (unsigned char *)zLeft;
   b = (unsigned char *)zRight;
-  while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; }
-  return UpperToLower[*a] - UpperToLower[*b];
+  for(;;){
+    c = (int)UpperToLower[*a] - (int)UpperToLower[*b];
+    if( c || *a==0 ) break;
+    a++;
+    b++;
+  }
+  return c;
 }
 int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){
   register unsigned char *a, *b;
@@ -315,7 +355,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   int eValid = 1;  /* True exponent is either not used or is well-formed */
   double result;
   int nDigits = 0;
-  int nonNum = 0;
+  int nonNum = 0;  /* True if input contains UTF16 with high byte non-zero */
 
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
   *pResult = 0.0;   /* Default return value, in case of an error */
@@ -328,7 +368,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
     assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
     for(i=3-enc; i=zEnd ) goto do_atof_calc;
 
@@ -376,7 +414,12 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   if( *z=='e' || *z=='E' ){
     z+=incr;
     eValid = 0;
-    if( z>=zEnd ) goto do_atof_calc;
+
+    /* This branch is needed to avoid a (harmless) buffer overread.  The 
+    ** special comment alerts the mutation tester that the correct answer
+    ** is obtained even if the branch is omitted */
+    if( z>=zEnd ) goto do_atof_calc;              /*PREVENTS-HARMLESS-OVERREAD*/
+
     /* get sign of exponent */
     if( *z=='-' ){
       esign = -1;
@@ -393,9 +436,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   }
 
   /* skip trailing spaces */
-  if( nDigits && eValid ){
-    while( z0 ){
-      while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10;
-    }else{
-      while( !(s%10) && e>0 ) e--,s/=10;
+    /* Attempt to reduce exponent.
+    **
+    ** Branches that are not required for the correct answer but which only
+    ** help to obtain the correct answer faster are marked with special
+    ** comments, as a hint to the mutation tester.
+    */
+    while( e>0 ){                                       /*OPTIMIZATION-IF-TRUE*/
+      if( esign>0 ){
+        if( s>=(LARGEST_INT64/10) ) break;             /*OPTIMIZATION-IF-FALSE*/
+        s *= 10;
+      }else{
+        if( s%10!=0 ) break;                           /*OPTIMIZATION-IF-FALSE*/
+        s /= 10;
+      }
+      e--;
     }
 
     /* adjust the sign of significand */
     s = sign<0 ? -s : s;
 
-    /* if exponent, scale significand as appropriate
-    ** and store in result. */
-    if( e ){
+    if( e==0 ){                                         /*OPTIMIZATION-IF-TRUE*/
+      result = (double)s;
+    }else{
       LONGDOUBLE_TYPE scale = 1.0;
       /* attempt to handle extremely small/large numbers better */
-      if( e>307 && e<342 ){
-        while( e%308 ) { scale *= 1.0e+1; e -= 1; }
-        if( esign<0 ){
-          result = s / scale;
-          result /= 1.0e+308;
-        }else{
-          result = s * scale;
-          result *= 1.0e+308;
-        }
-      }else if( e>=342 ){
-        if( esign<0 ){
-          result = 0.0*s;
-        }else{
-          result = 1e308*1e308*s;  /* Infinity */
+      if( e>307 ){                                      /*OPTIMIZATION-IF-TRUE*/
+        if( e<342 ){                                    /*OPTIMIZATION-IF-TRUE*/
+          while( e%308 ) { scale *= 1.0e+1; e -= 1; }
+          if( esign<0 ){
+            result = s / scale;
+            result /= 1.0e+308;
+          }else{
+            result = s * scale;
+            result *= 1.0e+308;
+          }
+        }else{ assert( e>=342 );
+          if( esign<0 ){
+            result = 0.0*s;
+          }else{
+            result = 1e308*1e308*s;  /* Infinity */
+          }
         }
       }else{
         /* 1.0e+22 is the largest power of 10 than can be 
@@ -454,8 +505,6 @@ do_atof_calc:
           result = s * scale;
         }
       }
-    } else {
-      result = (double)s;
     }
   }
 
@@ -463,7 +512,7 @@ do_atof_calc:
   *pResult = result;
 
   /* return true if number and no extra non-whitespace chracters after */
-  return z>=zEnd && nDigits>0 && eValid && nonNum==0;
+  return z==zEnd && nDigits>0 && eValid && nonNum==0;
 #else
   return !sqlite3Atoi64(z, pResult, length, enc);
 #endif /* SQLITE_OMIT_FLOATING_POINT */
@@ -525,7 +574,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
   int neg = 0; /* assume positive */
   int i;
   int c = 0;
-  int nonNum = 0;
+  int nonNum = 0;  /* True if input contains UTF16 with high byte non-zero */
   const char *zStart;
   const char *zEnd = zNum + length;
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
@@ -536,7 +585,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
     assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
     for(i=3-enc; i19*incr || nonNum ){
+  if( &zNum[i]19*incr                  /* Too many digits */
+   || nonNum                     /* UTF16 with high-order bytes non-zero */
+  ){
     /* zNum is empty or contains non-numeric text or is longer
     ** than 19 digits (thus guaranteeing that it is too large) */
     return 1;
@@ -606,7 +658,6 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
 #ifndef SQLITE_OMIT_HEX_INTEGER
   if( z[0]=='0'
    && (z[1]=='x' || z[1]=='X')
-   && sqlite3Isxdigit(z[2])
   ){
     u64 u = 0;
     int i, k;
@@ -1076,7 +1127,7 @@ u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){
 */
 int sqlite3VarintLen(u64 v){
   int i;
-  for(i=1; (v >>= 7)!=0; i++){ assert( i<9 ); }
+  for(i=1; (v >>= 7)!=0; i++){ assert( i<10 ); }
   return i;
 }
 
@@ -1107,10 +1158,12 @@ u32 sqlite3Get4byte(const u8 *p){
 void sqlite3Put4byte(unsigned char *p, u32 v){
 #if SQLITE_BYTEORDER==4321
   memcpy(p,&v,4);
-#elif SQLITE_BYTEORDER==1234 && defined(__GNUC__) && GCC_VERSION>=4003000
+#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \
+    && defined(__GNUC__) && GCC_VERSION>=4003000
   u32 x = __builtin_bswap32(v);
   memcpy(p,&x,4);
-#elif SQLITE_BYTEORDER==1234 && defined(_MSC_VER) && _MSC_VER>=1300
+#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \
+    && defined(_MSC_VER) && _MSC_VER>=1300
   u32 x = _byteswap_ulong(v);
   memcpy(p,&x,4);
 #else
@@ -1252,36 +1305,21 @@ int sqlite3SubInt64(i64 *pA, i64 iB){
     return sqlite3AddInt64(pA, -iB);
   }
 }
-#define TWOPOWER32 (((i64)1)<<32)
-#define TWOPOWER31 (((i64)1)<<31)
 int sqlite3MulInt64(i64 *pA, i64 iB){
   i64 iA = *pA;
-  i64 iA1, iA0, iB1, iB0, r;
-
-  iA1 = iA/TWOPOWER32;
-  iA0 = iA % TWOPOWER32;
-  iB1 = iB/TWOPOWER32;
-  iB0 = iB % TWOPOWER32;
-  if( iA1==0 ){
-    if( iB1==0 ){
-      *pA *= iB;
-      return 0;
+  if( iB>0 ){
+    if( iA>LARGEST_INT64/iB ) return 1;
+    if( iA0 ){
+      if( iBLARGEST_INT64/-iB ) return 1;
     }
-    r = iA0*iB1;
-  }else if( iB1==0 ){
-    r = iA1*iB0;
-  }else{
-    /* If both iA1 and iB1 are non-zero, overflow will result */
-    return 1;
   }
-  testcase( r==(-TWOPOWER31)-1 );
-  testcase( r==(-TWOPOWER31) );
-  testcase( r==TWOPOWER31 );
-  testcase( r==TWOPOWER31-1 );
-  if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1;
-  r *= TWOPOWER32;
-  if( sqlite3AddInt64(&r, iA0*iB0) ) return 1;
-  *pA = r;
+  *pA = iA*iB;
   return 0;
 }
 
@@ -1366,7 +1404,7 @@ LogEst sqlite3LogEst(u64 x){
     if( x<2 ) return 0;
     while( x<8 ){  y -= 10; x <<= 1; }
   }else{
-    while( x>255 ){ y += 40; x >>= 4; }
+    while( x>255 ){ y += 40; x >>= 4; }  /*OPTIMIZATION-IF-TRUE*/
     while( x>15 ){  y += 10; x >>= 1; }
   }
   return a[x&7] + y - 10;
@@ -1389,18 +1427,29 @@ LogEst sqlite3LogEstFromDouble(double x){
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
+    defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \
+    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
 /*
 ** Convert a LogEst into an integer.
+**
+** Note that this routine is only used when one or more of various
+** non-standard compile-time options is enabled.
 */
 u64 sqlite3LogEstToInt(LogEst x){
   u64 n;
-  if( x<10 ) return 1;
   n = x%10;
   x /= 10;
   if( n>=5 ) n -= 2;
   else if( n>=1 ) n -= 1;
-  if( x>=3 ){
-    return x>60 ? (u64)LARGEST_INT64 : (n+8)<<(x-3);
-  }
-  return (n+8)>>(3-x);
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \
+    defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
+  if( x>60 ) return (u64)LARGEST_INT64;
+#else
+  /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input
+  ** possible to this routine is 310, resulting in a maximum x of 31 */
+  assert( x<=60 );
+#endif
+  return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x);
 }
+#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */
diff --git a/src/vacuum.c b/src/vacuum.c
index adc802e6..25b12585 100644
--- a/src/vacuum.c
+++ b/src/vacuum.c
@@ -18,57 +18,52 @@
 #include "vdbeInt.h"
 
 #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
-/*
-** Finalize a prepared statement.  If there was an error, store the
-** text of the error message in *pzErrMsg.  Return the result code.
-*/
-static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){
-  int rc;
-  rc = sqlite3VdbeFinalize((Vdbe*)pStmt);
-  if( rc ){
-    sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db));
-  }
-  return rc;
-}
 
 /*
-** Execute zSql on database db. Return an error code.
+** Execute zSql on database db.
+**
+** If zSql returns rows, then each row will have exactly one
+** column.  (This will only happen if zSql begins with "SELECT".)
+** Take each row of result and call execSql() again recursively.
+**
+** The execSqlF() routine does the same thing, except it accepts
+** a format string as its third argument
 */
 static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
-  sqlite3_stmt *pStmt;
-  VVA_ONLY( int rc; )
-  if( !zSql ){
-    return SQLITE_NOMEM;
-  }
-  if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){
-    sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db));
-    return sqlite3_errcode(db);
-  }
-  VVA_ONLY( rc = ) sqlite3_step(pStmt);
-  assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) );
-  return vacuumFinalize(db, pStmt, pzErrMsg);
-}
-
-/*
-** Execute zSql on database db. The statement returns exactly
-** one column. Execute this as SQL on the same database.
-*/
-static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
   sqlite3_stmt *pStmt;
   int rc;
 
-  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  /* printf("SQL: [%s]\n", zSql); fflush(stdout); */
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
   if( rc!=SQLITE_OK ) return rc;
-
-  while( SQLITE_ROW==sqlite3_step(pStmt) ){
-    rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0));
-    if( rc!=SQLITE_OK ){
-      vacuumFinalize(db, pStmt, pzErrMsg);
-      return rc;
+  while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+    const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0);
+    assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 );
+    if( zSubSql ){
+      assert( zSubSql[0]!='S' );
+      rc = execSql(db, pzErrMsg, zSubSql);
+      if( rc!=SQLITE_OK ) break;
     }
   }
-
-  return vacuumFinalize(db, pStmt, pzErrMsg);
+  assert( rc!=SQLITE_ROW );
+  if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+  if( rc ){
+    sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db));
+  }
+  (void)sqlite3_finalize(pStmt);
+  return rc;
+}
+static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){
+  char *z;
+  va_list ap;
+  int rc;
+  va_start(ap, zSql);
+  z = sqlite3VMPrintf(db, zSql, ap);
+  va_end(ap);
+  if( z==0 ) return SQLITE_NOMEM;
+  rc = execSql(db, pzErrMsg, z);
+  sqlite3DbFree(db, z);
+  return rc;
 }
 
 /*
@@ -101,11 +96,12 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
 ** transient would cause the database file to appear to be deleted
 ** following reboot.
 */
-void sqlite3Vacuum(Parse *pParse){
+void sqlite3Vacuum(Parse *pParse, Token *pNm){
   Vdbe *v = sqlite3GetVdbe(pParse);
-  if( v ){
-    sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0);
-    sqlite3VdbeUsesBtree(v, 0);
+  int iDb = pNm ? sqlite3TwoPartName(pParse, pNm, pNm, &pNm) : 0;
+  if( v && (iDb>=2 || iDb==0) ){
+    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
+    sqlite3VdbeUsesBtree(v, iDb);
   }
   return;
 }
@@ -113,19 +109,19 @@ void sqlite3Vacuum(Parse *pParse){
 /*
 ** This routine implements the OP_Vacuum opcode of the VDBE.
 */
-int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
+int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){
   int rc = SQLITE_OK;     /* Return code from service routines */
   Btree *pMain;           /* The database being vacuumed */
   Btree *pTemp;           /* The temporary database we vacuum into */
-  char *zSql = 0;         /* SQL statements */
   int saved_flags;        /* Saved value of the db->flags */
   int saved_nChange;      /* Saved value of db->nChange */
   int saved_nTotalChange; /* Saved value of db->nTotalChange */
-  void (*saved_xTrace)(void*,const char*);  /* Saved db->xTrace */
+  u8 saved_mTrace;        /* Saved trace settings */
   Db *pDb = 0;            /* Database to detach at end of vacuum */
   int isMemDb;            /* True if vacuuming a :memory: database */
   int nRes;               /* Bytes of reserved space at the end of each page */
   int nDb;                /* Number of attached databases */
+  const char *zDbMain;    /* Schema name of database to vacuum */
 
   if( !db->autoCommit ){
     sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
@@ -142,12 +138,14 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   saved_flags = db->flags;
   saved_nChange = db->nChange;
   saved_nTotalChange = db->nTotalChange;
-  saved_xTrace = db->xTrace;
-  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin;
-  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder);
-  db->xTrace = 0;
+  saved_mTrace = db->mTrace;
+  db->flags |= (SQLITE_WriteSchema | SQLITE_IgnoreChecks
+                 | SQLITE_PreferBuiltin | SQLITE_Vacuum);
+  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_CountRows);
+  db->mTrace = 0;
 
-  pMain = db->aDb[0].pBt;
+  zDbMain = db->aDb[iDb].zDbSName;
+  pMain = db->aDb[iDb].pBt;
   isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));
 
   /* Attach the temporary database as 'vacuum_db'. The synchronous pragma
@@ -165,18 +163,12 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   ** to write the journal header file.
   */
   nDb = db->nDb;
-  if( sqlite3TempInMemory(db) ){
-    zSql = "ATTACH ':memory:' AS vacuum_db;";
-  }else{
-    zSql = "ATTACH '' AS vacuum_db;";
-  }
-  rc = execSql(db, pzErrMsg, zSql);
-  if( db->nDb>nDb ){
-    pDb = &db->aDb[db->nDb-1];
-    assert( strcmp(pDb->zName,"vacuum_db")==0 );
-  }
+  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  pTemp = db->aDb[db->nDb-1].pBt;
+  assert( (db->nDb-1)==nDb );
+  pDb = &db->aDb[nDb];
+  assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
+  pTemp = pDb->pBt;
 
   /* The call to execSql() to attach the temp database has left the file
   ** locked (as there was more than one active statement when the transaction
@@ -197,14 +189,15 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   }
 #endif
 
-  rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF");
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
+  sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size);
+  sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0));
+  sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL);
 
   /* Begin a transaction and take an exclusive lock on the main database
   ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
   ** to ensure that we do not try to change the page-size on a WAL database.
   */
-  rc = execSql(db, pzErrMsg, "BEGIN;");
+  rc = execSql(db, pzErrMsg, "BEGIN");
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
   rc = sqlite3BtreeBeginTrans(pMain, 2);
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
@@ -219,7 +212,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
    || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0))
    || NEVER(db->mallocFailed)
   ){
-    rc = SQLITE_NOMEM;
+    rc = SQLITE_NOMEM_BKPT;
     goto end_of_vacuum;
   }
 
@@ -231,64 +224,48 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   /* Query the schema of the main database. Create a mirror schema
   ** in the temporary database.
   */
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) "
-      "  FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'"
-      "   AND coalesce(rootpage,1)>0"
+  db->init.iDb = nDb; /* force new CREATE statements into vacuum_db */
+  rc = execSqlF(db, pzErrMsg,
+      "SELECT sql FROM \"%w\".sqlite_master"
+      " WHERE type='table'AND name<>'sqlite_sequence'"
+      " AND coalesce(rootpage,1)>0",
+      zDbMain
   );
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)"
-      "  FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' ");
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) "
-      "  FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'");
+  rc = execSqlF(db, pzErrMsg,
+      "SELECT sql FROM \"%w\".sqlite_master"
+      " WHERE type='index' AND length(sql)>10",
+      zDbMain
+  );
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
+  db->init.iDb = 0;
 
   /* Loop through the tables in the main database. For each, do
   ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
   ** the contents to the temporary database.
   */
-  assert( (db->flags & SQLITE_Vacuum)==0 );
-  db->flags |= SQLITE_Vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'INSERT INTO vacuum_db.' || quote(name) "
-      "|| ' SELECT * FROM main.' || quote(name) || ';'"
-      "FROM main.sqlite_master "
-      "WHERE type = 'table' AND name!='sqlite_sequence' "
-      "  AND coalesce(rootpage,1)>0"
+  rc = execSqlF(db, pzErrMsg,
+      "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+      "||' SELECT*FROM\"%w\".'||quote(name)"
+      "FROM vacuum_db.sqlite_master "
+      "WHERE type='table'AND coalesce(rootpage,1)>0",
+      zDbMain
   );
   assert( (db->flags & SQLITE_Vacuum)!=0 );
   db->flags &= ~SQLITE_Vacuum;
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
 
-  /* Copy over the sequence table
-  */
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' "
-      "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' "
-  );
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'INSERT INTO vacuum_db.' || quote(name) "
-      "|| ' SELECT * FROM main.' || quote(name) || ';' "
-      "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';"
-  );
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
-
-
   /* Copy the triggers, views, and virtual tables from the main database
   ** over to the temporary database.  None of these objects has any
   ** associated storage, so all we have to do is copy their entries
   ** from the SQLITE_MASTER table.
   */
-  rc = execSql(db, pzErrMsg,
-      "INSERT INTO vacuum_db.sqlite_master "
-      "  SELECT type, name, tbl_name, rootpage, sql"
-      "    FROM main.sqlite_master"
-      "   WHERE type='view' OR type='trigger'"
-      "      OR (type='table' AND rootpage=0)"
+  rc = execSqlF(db, pzErrMsg,
+      "INSERT INTO vacuum_db.sqlite_master"
+      " SELECT*FROM \"%w\".sqlite_master"
+      " WHERE type IN('view','trigger')"
+      " OR(type='table'AND rootpage=0)",
+      zDbMain
   );
   if( rc ) goto end_of_vacuum;
 
@@ -342,10 +319,11 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
 
 end_of_vacuum:
   /* Restore the original value of db->flags */
+  db->init.iDb = 0;
   db->flags = saved_flags;
   db->nChange = saved_nChange;
   db->nTotalChange = saved_nTotalChange;
-  db->xTrace = saved_xTrace;
+  db->mTrace = saved_mTrace;
   sqlite3BtreeSetPageSize(pMain, -1, -1, 1);
 
   /* Currently there is an SQL level transaction open on the vacuum
diff --git a/src/vdbe.c b/src/vdbe.c
index e7c3e48a..54df5c53 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -86,6 +86,16 @@ static void updateMaxBlobsize(Mem *p){
 }
 #endif
 
+/*
+** This macro evaluates to true if either the update hook or the preupdate
+** hook are enabled for database connect DB.
+*/
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback)
+#else
+# define HAS_UPDATE_HOOK(DB) ((DB)->xUpdateCallback)
+#endif
+
 /*
 ** The next global variable is incremented each time the OP_Found opcode
 ** is executed. This is used to test whether or not the foreign key
@@ -192,11 +202,11 @@ static VdbeCursor *allocateCursor(
   **     be freed lazily via the sqlite3_release_memory() API. This
   **     minimizes the number of malloc calls made by the system.
   **
-  ** Memory cells for cursors are allocated at the top of the address
-  ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for
-  ** cursor 1 is managed by memory cell (p->nMem-1), etc.
+  ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from
+  ** the top of the register space.  Cursor 1 is at Mem[p->nMem-1].
+  ** Cursor 2 is at Mem[p->nMem-2]. And so forth.
   */
-  Mem *pMem = &p->aMem[p->nMem-iCur];
+  Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem;
 
   int nByte;
   VdbeCursor *pCx = 0;
@@ -204,8 +214,8 @@ static VdbeCursor *allocateCursor(
       ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + 
       (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0);
 
-  assert( iCurnCursor );
-  if( p->apCsr[iCur] ){
+  assert( iCur>=0 && iCurnCursor );
+  if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/
     sqlite3VdbeFreeCursor(p, p->apCsr[iCur]);
     p->apCsr[iCur] = 0;
   }
@@ -282,7 +292,7 @@ static void applyAffinity(
   if( affinity>=SQLITE_AFF_NUMERIC ){
     assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL
              || affinity==SQLITE_AFF_NUMERIC );
-    if( (pRec->flags & MEM_Int)==0 ){
+    if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/
       if( (pRec->flags & MEM_Real)==0 ){
         if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1);
       }else{
@@ -292,10 +302,13 @@ static void applyAffinity(
   }else if( affinity==SQLITE_AFF_TEXT ){
     /* Only attempt the conversion to TEXT if there is an integer or real
     ** representation (blob and NULL do not get converted) but no string
-    ** representation.
-    */
-    if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){
-      sqlite3VdbeMemStringify(pRec, enc, 1);
+    ** representation.  It would be harmless to repeat the conversion if 
+    ** there is already a string rep, but it is pointless to waste those
+    ** CPU cycles. */
+    if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/
+      if( (pRec->flags&(MEM_Real|MEM_Int)) ){
+        sqlite3VdbeMemStringify(pRec, enc, 1);
+      }
     }
     pRec->flags &= ~(MEM_Real|MEM_Int);
   }
@@ -529,10 +542,10 @@ static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){
 static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){
   Mem *pOut;
   assert( pOp->p2>0 );
-  assert( pOp->p2<=(p->nMem-p->nCursor) );
+  assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
   pOut = &p->aMem[pOp->p2];
   memAboutToChange(p, pOut);
-  if( VdbeMemDynamic(pOut) ){
+  if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/
     return out2PrereleaseWithClear(pOut);
   }else{
     pOut->flags = MEM_Int;
@@ -560,7 +573,7 @@ int sqlite3VdbeExec(
   sqlite3 *db = p->db;       /* The database */
   u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */
   u8 encoding = ENC(db);     /* The database encoding */
-  int iCompare = 0;          /* Result of last OP_Compare operation */
+  int iCompare = 0;          /* Result of last comparison */
   unsigned nVmStep = 0;      /* Number of virtual machine steps */
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   unsigned nProgressLimit = 0;/* Invoke xProgress() when nVmStep reaches this */
@@ -627,7 +640,11 @@ int sqlite3VdbeExec(
   }
   sqlite3EndBenignMalloc();
 #endif
-  for(pOp=&aOp[p->pc]; rc==SQLITE_OK; pOp++){
+  for(pOp=&aOp[p->pc]; 1; pOp++){
+    /* Errors are detected by individual opcodes, with an immediate
+    ** jumps to abort_due_to_error. */
+    assert( rc==SQLITE_OK );
+
     assert( pOp>=aOp && pOp<&aOp[p->nOp]);
 #ifdef VDBE_PROFILE
     start = sqlite3Hwtime();
@@ -660,37 +677,39 @@ int sqlite3VdbeExec(
 
     /* Sanity checking on other operands */
 #ifdef SQLITE_DEBUG
-    assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] );
-    if( (pOp->opflags & OPFLG_IN1)!=0 ){
-      assert( pOp->p1>0 );
-      assert( pOp->p1<=(p->nMem-p->nCursor) );
-      assert( memIsValid(&aMem[pOp->p1]) );
-      assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) );
-      REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
-    }
-    if( (pOp->opflags & OPFLG_IN2)!=0 ){
-      assert( pOp->p2>0 );
-      assert( pOp->p2<=(p->nMem-p->nCursor) );
-      assert( memIsValid(&aMem[pOp->p2]) );
-      assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) );
-      REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
-    }
-    if( (pOp->opflags & OPFLG_IN3)!=0 ){
-      assert( pOp->p3>0 );
-      assert( pOp->p3<=(p->nMem-p->nCursor) );
-      assert( memIsValid(&aMem[pOp->p3]) );
-      assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) );
-      REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
-    }
-    if( (pOp->opflags & OPFLG_OUT2)!=0 ){
-      assert( pOp->p2>0 );
-      assert( pOp->p2<=(p->nMem-p->nCursor) );
-      memAboutToChange(p, &aMem[pOp->p2]);
-    }
-    if( (pOp->opflags & OPFLG_OUT3)!=0 ){
-      assert( pOp->p3>0 );
-      assert( pOp->p3<=(p->nMem-p->nCursor) );
-      memAboutToChange(p, &aMem[pOp->p3]);
+    {
+      u8 opProperty = sqlite3OpcodeProperty[pOp->opcode];
+      if( (opProperty & OPFLG_IN1)!=0 ){
+        assert( pOp->p1>0 );
+        assert( pOp->p1<=(p->nMem+1 - p->nCursor) );
+        assert( memIsValid(&aMem[pOp->p1]) );
+        assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) );
+        REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
+      }
+      if( (opProperty & OPFLG_IN2)!=0 ){
+        assert( pOp->p2>0 );
+        assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
+        assert( memIsValid(&aMem[pOp->p2]) );
+        assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) );
+        REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
+      }
+      if( (opProperty & OPFLG_IN3)!=0 ){
+        assert( pOp->p3>0 );
+        assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
+        assert( memIsValid(&aMem[pOp->p3]) );
+        assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) );
+        REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
+      }
+      if( (opProperty & OPFLG_OUT2)!=0 ){
+        assert( pOp->p2>0 );
+        assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
+        memAboutToChange(p, &aMem[pOp->p2]);
+      }
+      if( (opProperty & OPFLG_OUT3)!=0 ){
+        assert( pOp->p3>0 );
+        assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
+        memAboutToChange(p, &aMem[pOp->p3]);
+      }
     }
 #endif
 #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
@@ -774,7 +793,7 @@ check_for_interrupt:
     nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps);
     if( db->xProgress(db->pProgressArg) ){
       rc = SQLITE_INTERRUPT;
-      goto vdbe_error_halt;
+      goto abort_due_to_error;
     }
   }
 #endif
@@ -788,7 +807,7 @@ check_for_interrupt:
 ** and then jump to address P2.
 */
 case OP_Gosub: {            /* jump */
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
   pIn1 = &aMem[pOp->p1];
   assert( VdbeMemDynamic(pIn1)==0 );
   memAboutToChange(p, pIn1);
@@ -828,7 +847,7 @@ case OP_Return: {           /* in1 */
 ** See also: EndCoroutine
 */
 case OP_InitCoroutine: {     /* jump */
-  assert( pOp->p1>0 &&  pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 &&  pOp->p1<=(p->nMem+1 - p->nCursor) );
   assert( pOp->p2>=0 && pOp->p2nOp );
   assert( pOp->p3>=0 && pOp->p3nOp );
   pOut = &aMem[pOp->p1];
@@ -886,7 +905,7 @@ case OP_Yield: {            /* in1, jump */
 }
 
 /* Opcode:  HaltIfNull  P1 P2 P3 P4 P5
-** Synopsis:  if r[P3]=null halt
+** Synopsis: if r[P3]=null halt
 **
 ** Check the value in register P3.  If it is NULL then Halt using
 ** parameter P1, P2, and P4 as if this were a Halt instruction.  If the
@@ -930,8 +949,6 @@ case OP_HaltIfNull: {      /* in3 */
 ** is the same as executing Halt.
 */
 case OP_Halt: {
-  const char *zType;
-  const char *zLogFmt;
   VdbeFrame *pFrame;
   int pcx;
 
@@ -960,34 +977,28 @@ case OP_Halt: {
   p->rc = pOp->p1;
   p->errorAction = (u8)pOp->p2;
   p->pc = pcx;
+  assert( pOp->p5>=0 && pOp->p5<=4 );
   if( p->rc ){
     if( pOp->p5 ){
       static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK",
                                              "FOREIGN KEY" };
-      assert( pOp->p5>=1 && pOp->p5<=4 );
       testcase( pOp->p5==1 );
       testcase( pOp->p5==2 );
       testcase( pOp->p5==3 );
       testcase( pOp->p5==4 );
-      zType = azType[pOp->p5-1];
+      sqlite3VdbeError(p, "%s constraint failed", azType[pOp->p5-1]);
+      if( pOp->p4.z ){
+        p->zErrMsg = sqlite3MPrintf(db, "%z: %s", p->zErrMsg, pOp->p4.z);
+      }
     }else{
-      zType = 0;
-    }
-    assert( zType!=0 || pOp->p4.z!=0 );
-    zLogFmt = "abort at %d in [%s]: %s";
-    if( zType && pOp->p4.z ){
-      sqlite3VdbeError(p, "%s constraint failed: %s", zType, pOp->p4.z);
-    }else if( pOp->p4.z ){
       sqlite3VdbeError(p, "%s", pOp->p4.z);
-    }else{
-      sqlite3VdbeError(p, "%s constraint failed", zType);
     }
-    sqlite3_log(pOp->p1, zLogFmt, pcx, p->zSql, p->zErrMsg);
+    sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg);
   }
   rc = sqlite3VdbeHalt(p);
   assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR );
   if( rc==SQLITE_BUSY ){
-    p->rc = rc = SQLITE_BUSY;
+    p->rc = SQLITE_BUSY;
   }else{
     assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT );
     assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 );
@@ -1053,7 +1064,7 @@ case OP_String8: {         /* same as TK_STRING, out2 */
 #ifndef SQLITE_OMIT_UTF16
   if( encoding!=SQLITE_UTF8 ){
     rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC);
-    if( rc==SQLITE_TOOBIG ) goto too_big;
+    assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG );
     if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem;
     assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z );
     assert( VdbeMemDynamic(pOut)==0 );
@@ -1066,10 +1077,12 @@ case OP_String8: {         /* same as TK_STRING, out2 */
     pOp->p4.z = pOut->z;
     pOp->p1 = pOut->n;
   }
+  testcase( rc==SQLITE_TOOBIG );
 #endif
   if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
     goto too_big;
   }
+  assert( rc==SQLITE_OK );
   /* Fall through to the next case, OP_String */
 }
   
@@ -1078,10 +1091,12 @@ case OP_String8: {         /* same as TK_STRING, out2 */
 **
 ** The string value P4 of length P1 (bytes) is stored in register P2.
 **
-** If P5!=0 and the content of register P3 is greater than zero, then
+** If P3 is not zero and the content of register P3 is equal to P5, then
 ** the datatype of the register P2 is converted to BLOB.  The content is
 ** the same sequence of bytes, it is merely interpreted as a BLOB instead
-** of a string, as if it had been CAST.
+** of a string, as if it had been CAST.  In other words:
+**
+** if( P3!=0 and reg[P3]==P5 ) reg[P2] := CAST(reg[P2] as BLOB)
 */
 case OP_String: {          /* out2 */
   assert( pOp->p4.z!=0 );
@@ -1092,19 +1107,18 @@ case OP_String: {          /* out2 */
   pOut->enc = encoding;
   UPDATE_MAX_BLOBSIZE(pOut);
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
-  if( pOp->p5 ){
-    assert( pOp->p3>0 );
-    assert( pOp->p3<=(p->nMem-p->nCursor) );
+  if( pOp->p3>0 ){
+    assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
     pIn3 = &aMem[pOp->p3];
     assert( pIn3->flags & MEM_Int );
-    if( pIn3->u.i ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term;
+    if( pIn3->u.i==pOp->p5 ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term;
   }
 #endif
   break;
 }
 
 /* Opcode: Null P1 P2 P3 * *
-** Synopsis:  r[P2..P3]=NULL
+** Synopsis: r[P2..P3]=NULL
 **
 ** Write a NULL into registers P2.  If P3 greater than P2, then also write
 ** NULL into register P3 and every register in between P2 and P3.  If P3
@@ -1120,20 +1134,22 @@ case OP_Null: {           /* out2 */
   u16 nullFlag;
   pOut = out2Prerelease(p, pOp);
   cnt = pOp->p3-pOp->p2;
-  assert( pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
   pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null;
+  pOut->n = 0;
   while( cnt>0 ){
     pOut++;
     memAboutToChange(p, pOut);
     sqlite3VdbeMemSetNull(pOut);
     pOut->flags = nullFlag;
+    pOut->n = 0;
     cnt--;
   }
   break;
 }
 
 /* Opcode: SoftNull P1 * * * *
-** Synopsis:  r[P1]=NULL
+** Synopsis: r[P1]=NULL
 **
 ** Set register P1 to have the value NULL as seen by the OP_MakeRecord
 ** instruction, but do not free any string or blob memory associated with
@@ -1141,7 +1157,7 @@ case OP_Null: {           /* out2 */
 ** previously copied using OP_SCopy, the copies will continue to be valid.
 */
 case OP_SoftNull: {
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
   pOut = &aMem[pOp->p1];
   pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined;
   break;
@@ -1186,7 +1202,7 @@ case OP_Variable: {            /* out2 */
 }
 
 /* Opcode: Move P1 P2 P3 * *
-** Synopsis:  r[P2@P3]=r[P1@P3]
+** Synopsis: r[P2@P3]=r[P1@P3]
 **
 ** Move the P3 values in register P1..P1+P3-1 over into
 ** registers P2..P2+P3-1.  Registers P1..P1+P3-1 are
@@ -1208,8 +1224,8 @@ case OP_Move: {
   pIn1 = &aMem[p1];
   pOut = &aMem[p2];
   do{
-    assert( pOut<=&aMem[(p->nMem-p->nCursor)] );
-    assert( pIn1<=&aMem[(p->nMem-p->nCursor)] );
+    assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] );
+    assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] );
     assert( memIsValid(pIn1) );
     memAboutToChange(p, pOut);
     sqlite3VdbeMemMove(pOut, pIn1);
@@ -1296,7 +1312,7 @@ case OP_IntCopy: {            /* out2 */
 }
 
 /* Opcode: ResultRow P1 P2 * * *
-** Synopsis:  output=r[P1@P2]
+** Synopsis: output=r[P1@P2]
 **
 ** The registers P1 through P1+P2-1 contain a single row of
 ** results. This opcode causes the sqlite3_step() call to terminate
@@ -1309,7 +1325,7 @@ case OP_ResultRow: {
   int i;
   assert( p->nResColumn==pOp->p2 );
   assert( pOp->p1>0 );
-  assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 );
+  assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 );
 
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   /* Run the progress counter just before returning.
@@ -1319,7 +1335,7 @@ case OP_ResultRow: {
    && db->xProgress(db->pProgressArg)!=0
   ){
     rc = SQLITE_INTERRUPT;
-    goto vdbe_error_halt;
+    goto abort_due_to_error;
   }
 #endif
 
@@ -1329,7 +1345,7 @@ case OP_ResultRow: {
   if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){
     assert( db->flags&SQLITE_CountRows );
     assert( p->usesStmtJournal );
-    break;
+    goto abort_due_to_error;
   }
 
   /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then 
@@ -1349,9 +1365,7 @@ case OP_ResultRow: {
   */
   assert( p->iStatement==0 || db->flags&SQLITE_CountRows );
   rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE);
-  if( NEVER(rc!=SQLITE_OK) ){
-    break;
-  }
+  assert( rc==SQLITE_OK );
 
   /* Invalidate all ephemeral cursor row caches */
   p->cacheCtr = (p->cacheCtr + 2)|1;
@@ -1371,6 +1385,10 @@ case OP_ResultRow: {
   }
   if( db->mallocFailed ) goto no_mem;
 
+  if( db->mTrace & SQLITE_TRACE_ROW ){
+    db->xTrace(SQLITE_TRACE_ROW, db->pTraceArg, p, 0);
+  }
+
   /* Return SQLITE_ROW
   */
   p->pc = (int)(pOp - aOp) + 1;
@@ -1427,14 +1445,14 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 }
 
 /* Opcode: Add P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]+r[P2]
+** Synopsis: r[P3]=r[P1]+r[P2]
 **
 ** Add the value in register P1 to the value in register P2
 ** and store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: Multiply P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]*r[P2]
+** Synopsis: r[P3]=r[P1]*r[P2]
 **
 **
 ** Multiply the value in register P1 by the value in register P2
@@ -1442,14 +1460,14 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: Subtract P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]-r[P1]
+** Synopsis: r[P3]=r[P2]-r[P1]
 **
 ** Subtract the value in register P1 from the value in register P2
 ** and store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: Divide P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]/r[P1]
+** Synopsis: r[P3]=r[P2]/r[P1]
 **
 ** Divide the value in register P1 by the value in register P2
 ** and store the result in register P3 (P3=P2/P1). If the value in 
@@ -1457,7 +1475,7 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 ** NULL, the result is NULL.
 */
 /* Opcode: Remainder P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]%r[P1]
+** Synopsis: r[P3]=r[P2]%r[P1]
 **
 ** Compute the remainder after integer register P2 is divided by 
 ** register P1 and store the result in register P3. 
@@ -1623,8 +1641,8 @@ case OP_Function0: {
 
   assert( pOp->p4type==P4_FUNCDEF );
   n = pOp->p5;
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
-  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
+  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) );
   assert( pOp->p3p2 || pOp->p3>=pOp->p2+n );
   pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*));
   if( pCtx==0 ) goto no_mem;
@@ -1674,7 +1692,8 @@ case OP_Function: {
       sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut));
       rc = pCtx->isError;
     }
-    sqlite3VdbeDeleteAuxData(p, pCtx->iOp, pOp->p1);
+    sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1);
+    if( rc ) goto abort_due_to_error;
   }
 
   /* Copy the result of the function into register P3 */
@@ -1689,21 +1708,21 @@ case OP_Function: {
 }
 
 /* Opcode: BitAnd P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]&r[P2]
+** Synopsis: r[P3]=r[P1]&r[P2]
 **
 ** Take the bit-wise AND of the values in register P1 and P2 and
 ** store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: BitOr P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]|r[P2]
+** Synopsis: r[P3]=r[P1]|r[P2]
 **
 ** Take the bit-wise OR of the values in register P1 and P2 and
 ** store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: ShiftLeft P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]<>r[P1]
+** Synopsis: r[P3]=r[P2]>>r[P1]
 **
 ** Shift the integer value in register P2 to the right by the
 ** number of bits specified by the integer in register P1.
@@ -1771,7 +1790,7 @@ case OP_ShiftRight: {           /* same as TK_RSHIFT, in1, in2, out3 */
 }
 
 /* Opcode: AddImm  P1 P2 * * *
-** Synopsis:  r[P1]=r[P1]+P2
+** Synopsis: r[P1]=r[P1]+P2
 ** 
 ** Add the constant P2 to the value in register P1.
 ** The result is always an integer.
@@ -1858,18 +1877,66 @@ case OP_Cast: {                  /* in1 */
   rc = ExpandBlob(pIn1);
   sqlite3VdbeMemCast(pIn1, pOp->p2, encoding);
   UPDATE_MAX_BLOBSIZE(pIn1);
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif /* SQLITE_OMIT_CAST */
 
+/* Opcode: Eq P1 P2 P3 P4 P5
+** Synopsis: IF r[P3]==r[P1]
+**
+** Compare the values in register P1 and P3.  If reg(P3)==reg(P1) then
+** jump to address P2.  Or if the SQLITE_STOREP2 flag is set in P5, then
+** store the result of comparison in register P2.
+**
+** The SQLITE_AFF_MASK portion of P5 must be an affinity character -
+** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made 
+** to coerce both inputs according to this affinity before the
+** comparison is made. If the SQLITE_AFF_MASK is 0x00, then numeric
+** affinity is used. Note that the affinity conversions are stored
+** back into the input registers P1 and P3.  So this opcode can cause
+** persistent changes to registers P1 and P3.
+**
+** Once any conversions have taken place, and neither value is NULL, 
+** the values are compared. If both values are blobs then memcmp() is
+** used to determine the results of the comparison.  If both values
+** are text, then the appropriate collating function specified in
+** P4 is used to do the comparison.  If P4 is not specified then
+** memcmp() is used to compare text string.  If both values are
+** numeric, then a numeric comparison is used. If the two values
+** are of different types, then numbers are considered less than
+** strings and strings are considered less than blobs.
+**
+** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either
+** true or false and is never NULL.  If both operands are NULL then the result
+** of comparison is true.  If either operand is NULL then the result is false.
+** If neither operand is NULL the result is the same as it would be if
+** the SQLITE_NULLEQ flag were omitted from P5.
+**
+** If both SQLITE_STOREP2 and SQLITE_KEEPNULL flags are set then the
+** content of r[P2] is only changed if the new value is NULL or 0 (false).
+** In other words, a prior r[P2] value will not be overwritten by 1 (true).
+*/
+/* Opcode: Ne P1 P2 P3 P4 P5
+** Synopsis: IF r[P3]!=r[P1]
+**
+** This works just like the Eq opcode except that the jump is taken if
+** the operands in registers P1 and P3 are not equal.  See the Eq opcode for
+** additional information.
+**
+** If both SQLITE_STOREP2 and SQLITE_KEEPNULL flags are set then the
+** content of r[P2] is only changed if the new value is NULL or 1 (true).
+** In other words, a prior r[P2] value will not be overwritten by 0 (false).
+*/
 /* Opcode: Lt P1 P2 P3 P4 P5
-** Synopsis: if r[P1]r[P3] goto P2
+** Synopsis: IF r[P3]>r[P1]
 **
 ** This works just like the Lt opcode except that the jump is taken if
 ** the content of register P3 is greater than the content of
 ** register P1.  See the Lt opcode for additional information.
 */
 /* Opcode: Ge P1 P2 P3 P4 P5
-** Synopsis: if r[P1]>=r[P3] goto P2
+** Synopsis: IF r[P3]>=r[P1]
 **
 ** This works just like the Lt opcode except that the jump is taken if
 ** the content of register P3 is greater than or equal to the content of
@@ -1950,7 +1984,7 @@ case OP_Lt:               /* same as TK_LT, jump, in1, in3 */
 case OP_Le:               /* same as TK_LE, jump, in1, in3 */
 case OP_Gt:               /* same as TK_GT, jump, in1, in3 */
 case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
-  int res;            /* Result of the comparison of pIn1 against pIn3 */
+  int res, res2;      /* Result of the comparison of pIn1 against pIn3 */
   char affinity;      /* Affinity to use for comparison */
   u16 flags1;         /* Copy of initial value of pIn1->flags */
   u16 flags3;         /* Copy of initial value of pIn3->flags */
@@ -1973,9 +2007,9 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
        && (flags3&MEM_Null)!=0
        && (flags3&MEM_Cleared)==0
       ){
-        res = 0;  /* Results are equal */
+        res = 0;  /* Operands are equal */
       }else{
-        res = 1;  /* Results are not equal */
+        res = 1;  /* Operands are not equal */
       }
     }else{
       /* SQLITE_NULLEQ is clear and at least one operand is NULL,
@@ -1984,6 +2018,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       */
       if( pOp->p5 & SQLITE_STOREP2 ){
         pOut = &aMem[pOp->p2];
+        iCompare = 1;    /* Operands are not equal */
         memAboutToChange(p, pOut);
         MemSetTypeFlag(pOut, MEM_Null);
         REGISTER_TRACE(pOp->p2, pOut);
@@ -1999,11 +2034,23 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
     /* Neither operand is NULL.  Do a comparison. */
     affinity = pOp->p5 & SQLITE_AFF_MASK;
     if( affinity>=SQLITE_AFF_NUMERIC ){
-      if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
-        applyNumericAffinity(pIn1,0);
+      if( (flags1 | flags3)&MEM_Str ){
+        if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
+          applyNumericAffinity(pIn1,0);
+          testcase( flags3!=pIn3->flags ); /* Possible if pIn1==pIn3 */
+          flags3 = pIn3->flags;
+        }
+        if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
+          applyNumericAffinity(pIn3,0);
+        }
       }
-      if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
-        applyNumericAffinity(pIn3,0);
+      /* Handle the common case of integer comparison here, as an
+      ** optimization, to avoid a call to sqlite3MemCompare() */
+      if( (pIn1->flags & pIn3->flags & MEM_Int)!=0 ){
+        if( pIn3->u.i > pIn1->u.i ){ res = +1; goto compare_op; }
+        if( pIn3->u.i < pIn1->u.i ){ res = -1; goto compare_op; }
+        res = 0;
+        goto compare_op;
       }
     }else if( affinity==SQLITE_AFF_TEXT ){
       if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){
@@ -2012,6 +2059,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
         sqlite3VdbeMemStringify(pIn1, encoding, 1);
         testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) );
         flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
+        assert( pIn1!=pIn3 );
       }
       if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){
         testcase( pIn3->flags & MEM_Int );
@@ -2022,23 +2070,16 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       }
     }
     assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 );
-    if( flags1 & MEM_Zero ){
-      sqlite3VdbeMemExpandBlob(pIn1);
-      flags1 &= ~MEM_Zero;
-    }
-    if( flags3 & MEM_Zero ){
-      sqlite3VdbeMemExpandBlob(pIn3);
-      flags3 &= ~MEM_Zero;
-    }
     res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl);
   }
+compare_op:
   switch( pOp->opcode ){
-    case OP_Eq:    res = res==0;     break;
-    case OP_Ne:    res = res!=0;     break;
-    case OP_Lt:    res = res<0;      break;
-    case OP_Le:    res = res<=0;     break;
-    case OP_Gt:    res = res>0;      break;
-    default:       res = res>=0;     break;
+    case OP_Eq:    res2 = res==0;     break;
+    case OP_Ne:    res2 = res;        break;
+    case OP_Lt:    res2 = res<0;      break;
+    case OP_Le:    res2 = res<=0;     break;
+    case OP_Gt:    res2 = res>0;      break;
+    default:       res2 = res>=0;     break;
   }
 
   /* Undo any changes made by applyAffinity() to the input registers. */
@@ -2049,19 +2090,55 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
 
   if( pOp->p5 & SQLITE_STOREP2 ){
     pOut = &aMem[pOp->p2];
+    iCompare = res;
+    res2 = res2!=0;  /* For this path res2 must be exactly 0 or 1 */
+    if( (pOp->p5 & SQLITE_KEEPNULL)!=0 ){
+      /* The KEEPNULL flag prevents OP_Eq from overwriting a NULL with 1
+      ** and prevents OP_Ne from overwriting NULL with 0.  This flag
+      ** is only used in contexts where either:
+      **   (1) op==OP_Eq && (r[P2]==NULL || r[P2]==0)
+      **   (2) op==OP_Ne && (r[P2]==NULL || r[P2]==1)
+      ** Therefore it is not necessary to check the content of r[P2] for
+      ** NULL. */
+      assert( pOp->opcode==OP_Ne || pOp->opcode==OP_Eq );
+      assert( res2==0 || res2==1 );
+      testcase( res2==0 && pOp->opcode==OP_Eq );
+      testcase( res2==1 && pOp->opcode==OP_Eq );
+      testcase( res2==0 && pOp->opcode==OP_Ne );
+      testcase( res2==1 && pOp->opcode==OP_Ne );
+      if( (pOp->opcode==OP_Eq)==res2 ) break;
+    }
     memAboutToChange(p, pOut);
     MemSetTypeFlag(pOut, MEM_Int);
-    pOut->u.i = res;
+    pOut->u.i = res2;
     REGISTER_TRACE(pOp->p2, pOut);
   }else{
     VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3);
-    if( res ){
+    if( res2 ){
       goto jump_to_p2;
     }
   }
   break;
 }
 
+/* Opcode: ElseNotEq * P2 * * *
+**
+** This opcode must immediately follow an OP_Lt or OP_Gt comparison operator.
+** If result of an OP_Eq comparison on the same two operands
+** would have be NULL or false (0), then then jump to P2. 
+** If the result of an OP_Eq comparison on the two previous operands
+** would have been true (1), then fall through.
+*/
+case OP_ElseNotEq: {       /* same as TK_ESCAPE, jump */
+  assert( pOp>aOp );
+  assert( pOp[-1].opcode==OP_Lt || pOp[-1].opcode==OP_Gt );
+  assert( pOp[-1].p5 & SQLITE_STOREP2 );
+  VdbeBranchTaken(iCompare!=0, 2);
+  if( iCompare!=0 ) goto jump_to_p2;
+  break;
+}
+
+
 /* Opcode: Permutation * * * P4 *
 **
 ** Set the permutation used by the OP_Compare operator to be the array
@@ -2122,11 +2199,11 @@ case OP_Compare: {
   if( aPermute ){
     int k, mx = 0;
     for(k=0; kmx ) mx = aPermute[k];
-    assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 );
-    assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 );
+    assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 );
+    assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 );
   }else{
-    assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 );
-    assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 );
+    assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 );
+    assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 );
   }
 #endif /* SQLITE_DEBUG */
   for(i=0; ip1nOnceFlag );
-  VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2);
-  if( p->aOnceFlag[pOp->p1] ){
+  assert( p->aOp[0].opcode==OP_Init );
+  VdbeBranchTaken(p->aOp[0].p1==pOp->p1, 2);
+  if( p->aOp[0].p1==pOp->p1 ){
     goto jump_to_p2;
   }else{
-    p->aOnceFlag[pOp->p1] = 1;
+    pOp->p1 = p->aOp[0].p1;
   }
   break;
 }
@@ -2311,7 +2384,7 @@ case OP_IfNot: {            /* jump, in1 */
 }
 
 /* Opcode: IsNull P1 P2 * * *
-** Synopsis:  if r[P1]==NULL goto P2
+** Synopsis: if r[P1]==NULL goto P2
 **
 ** Jump to P2 if the value in register P1 is NULL.
 */
@@ -2339,7 +2412,7 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 }
 
 /* Opcode: Column P1 P2 P3 P4 P5
-** Synopsis:  r[P3]=PX
+** Synopsis: r[P3]=PX
 **
 ** Interpret the data that cursor P1 points to as a structure built using
 ** the MakeRecord instruction.  (See the MakeRecord opcode for additional
@@ -2364,7 +2437,6 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 ** skipped for length() and all content loading can be skipped for typeof().
 */
 case OP_Column: {
-  i64 payloadSize64; /* Number of bytes in the record */
   int p2;            /* column number to retrieve */
   VdbeCursor *pC;    /* The VDBE cursor */
   BtCursor *pCrsr;   /* The BTree cursor */
@@ -2387,8 +2459,9 @@ case OP_Column: {
 
   /* If the cursor cache is stale, bring it up-to-date */
   rc = sqlite3VdbeCursorMoveto(&pC, &p2);
+  if( rc ) goto abort_due_to_error;
 
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pDest = &aMem[pOp->p3];
   memAboutToChange(p, pDest);
   assert( pOp->p1>=0 && pOp->p1nCursor );
@@ -2400,8 +2473,7 @@ case OP_Column: {
   assert( pC->eCurType!=CURTYPE_SORTER );
   pCrsr = pC->uc.pCursor;
 
-  if( rc ) goto abort_due_to_error;
-  if( pC->cacheStatus!=p->cacheCtr ){
+  if( pC->cacheStatus!=p->cacheCtr ){                /*OPTIMIZATION-IF-FALSE*/
     if( pC->nullRow ){
       if( pC->eCurType==CURTYPE_PSEUDO ){
         assert( pC->uc.pseudoTableReg>0 );
@@ -2417,22 +2489,9 @@ case OP_Column: {
     }else{
       assert( pC->eCurType==CURTYPE_BTREE );
       assert( pCrsr );
-      if( pC->isTable==0 ){
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64);
-        assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */
-        /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the
-        ** payload size, so it is impossible for payloadSize64 to be
-        ** larger than 32 bits. */
-        assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 );
-        pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail);
-        pC->payloadSize = (u32)payloadSize64;
-      }else{
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize);
-        assert( rc==SQLITE_OK );   /* DataSize() cannot fail */
-        pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail);
-      }
+      assert( sqlite3BtreeCursorIsValid(pCrsr) );
+      pC->payloadSize = sqlite3BtreePayloadSize(pCrsr);
+      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail);
       assert( avail<=65536 );  /* Maximum page size is 64KiB */
       if( pC->payloadSize <= (u32)avail ){
         pC->szRow = pC->payloadSize;
@@ -2448,7 +2507,7 @@ case OP_Column: {
     aOffset[0] = offset;
 
 
-    if( availaRow does not have to hold the entire row, but it does at least
       ** need to cover the header of the record.  If pC->aRow does not contain
       ** the complete header, then set it to zero, forcing the header to be
@@ -2467,16 +2526,17 @@ case OP_Column: {
       */
       if( offset > 98307 || offset > pC->payloadSize ){
         rc = SQLITE_CORRUPT_BKPT;
-        goto op_column_error;
+        goto abort_due_to_error;
       }
+    }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/
+      /* The following goto is an optimization.  It can be omitted and
+      ** everything will still work.  But OP_Column is measurably faster
+      ** by skipping the subsequent conditional, which is always true.
+      */
+      zData = pC->aRow;
+      assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
+      goto op_column_read_header;
     }
-
-    /* The following goto is an optimization.  It can be omitted and
-    ** everything will still work.  But OP_Column is measurably faster
-    ** by skipping the subsequent conditional, which is always true.
-    */
-    assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
-    goto op_column_read_header;
   }
 
   /* Make sure at least the first p2+1 entries of the header have been
@@ -2486,24 +2546,23 @@ case OP_Column: {
     /* If there is more header available for parsing in the record, try
     ** to extract additional fields up through the p2+1-th field 
     */
-    op_column_read_header:
     if( pC->iHdrOffsetaRow==0 ){
         memset(&sMem, 0, sizeof(sMem));
         rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem);
-        if( rc!=SQLITE_OK ) goto op_column_error;
+        if( rc!=SQLITE_OK ) goto abort_due_to_error;
         zData = (u8*)sMem.z;
       }else{
         zData = pC->aRow;
       }
   
       /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */
+    op_column_read_header:
       i = pC->nHdrParsed;
       offset64 = aOffset[i];
       zHdr = zData + pC->iHdrOffset;
       zEndHdr = zData + aOffset[0];
-      assert( i<=p2 && zHdraType[i++] = t;
         aOffset[i] = (u32)(offset64 & 0xffffffff);
       }while( i<=p2 && zHdrnHdrParsed = i;
-      pC->iHdrOffset = (u32)(zHdr - zData);
-      if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
-  
+
       /* The record is corrupt if any of the following are true:
       ** (1) the bytes of the header extend past the declared header size
       ** (2) the entire header was used but not all data was used
@@ -2527,9 +2583,14 @@ case OP_Column: {
       if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize))
        || (offset64 > pC->payloadSize)
       ){
+        if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
         rc = SQLITE_CORRUPT_BKPT;
-        goto op_column_error;
+        goto abort_due_to_error;
       }
+
+      pC->nHdrParsed = i;
+      pC->iHdrOffset = (u32)(zHdr - zData);
+      if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
     }else{
       t = 0;
     }
@@ -2557,9 +2618,10 @@ case OP_Column: {
   assert( p2nHdrParsed );
   assert( rc==SQLITE_OK );
   assert( sqlite3VdbeCheckMemInvariants(pDest) );
-  if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest);
+  if( VdbeMemDynamic(pDest) ){
+    sqlite3VdbeMemSetNull(pDest);
+  }
   assert( t==pC->aType[p2] );
-  pDest->enc = encoding;
   if( pC->szRow>=aOffset[p2+1] ){
     /* This is the common case where the desired content fits on the original
     ** page - where the content is not on an overflow page */
@@ -2573,6 +2635,7 @@ case OP_Column: {
       */
       static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term };
       pDest->n = len = (t-12)/2;
+      pDest->enc = encoding;
       if( pDest->szMalloc < len+2 ){
         pDest->flags = MEM_Null;
         if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem;
@@ -2585,6 +2648,7 @@ case OP_Column: {
       pDest->flags = aFlag[t&1];
     }
   }else{
+    pDest->enc = encoding;
     /* This branch happens only when content is on overflow pages */
     if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
           && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
@@ -2601,15 +2665,13 @@ case OP_Column: {
     }else{
       rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable,
                                    pDest);
-      if( rc==SQLITE_OK ){
-        sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
-        pDest->flags &= ~MEM_Ephem;
-      }
+      if( rc!=SQLITE_OK ) goto abort_due_to_error;
+      sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
+      pDest->flags &= ~MEM_Ephem;
     }
   }
 
 op_column_out:
-op_column_error:
   UPDATE_MAX_BLOBSIZE(pDest);
   REGISTER_TRACE(pOp->p3, pDest);
   break;
@@ -2633,7 +2695,7 @@ case OP_Affinity: {
   assert( zAffinity[pOp->p2]==0 );
   pIn1 = &aMem[pOp->p1];
   while( (cAff = *(zAffinity++))!=0 ){
-    assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] );
+    assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] );
     assert( memIsValid(pIn1) );
     applyAffinity(pIn1, cAff, encoding);
     pIn1++;
@@ -2695,7 +2757,7 @@ case OP_MakeRecord: {
   nZero = 0;         /* Number of zero bytes at the end of the record */
   nField = pOp->p1;
   zAffinity = pOp->p4.z;
-  assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 );
+  assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 );
   pData0 = &aMem[nField];
   nField = pOp->p2;
   pLast = &pData0[nField-1];
@@ -2736,7 +2798,9 @@ case OP_MakeRecord: {
     testcase( serial_type==127 );
     testcase( serial_type==128 );
     nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type);
-  }while( (--pRec)>=pData0 );
+    if( pRec==pData0 ) break;
+    pRec--;
+  }while(1);
 
   /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint
   ** which determines the total number of bytes in the header. The varint
@@ -2785,7 +2849,7 @@ case OP_MakeRecord: {
   assert( i==nHdr );
   assert( j==nByte );
 
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pOut->n = (int)nByte;
   pOut->flags = MEM_Blob;
   if( nZero ){
@@ -2814,6 +2878,7 @@ case OP_Count: {         /* out2 */
   assert( pCrsr );
   nEntry = 0;  /* Not needed.  Only used to silence a warning. */
   rc = sqlite3BtreeCount(pCrsr, &nEntry);
+  if( rc ) goto abort_due_to_error;
   pOut = out2Prerelease(p, pOp);
   pOut->u.i = nEntry;
   break;
@@ -2883,7 +2948,7 @@ case OP_Savepoint: {
         }else{
           db->nSavepoint++;
         }
-    
+
         /* Link the new savepoint into the database handle's list. */
         pNew->pNext = db->pSavepoint;
         db->pSavepoint = pNew;
@@ -2991,6 +3056,7 @@ case OP_Savepoint: {
       }
     }
   }
+  if( rc ) goto abort_due_to_error;
 
   break;
 }
@@ -3027,7 +3093,7 @@ case OP_AutoCommit: {
       sqlite3VdbeError(p, "cannot commit transaction - "
                           "SQL statements in progress");
       rc = SQLITE_BUSY;
-      break;
+      goto abort_due_to_error;
     }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){
       goto vdbe_return;
     }else{
@@ -3054,6 +3120,7 @@ case OP_AutoCommit: {
                    "cannot commit - no transaction is active"));
          
     rc = SQLITE_ERROR;
+    goto abort_due_to_error;
   }
   break;
 }
@@ -3111,12 +3178,12 @@ case OP_Transaction: {
     rc = sqlite3BtreeBeginTrans(pBt, pOp->p2);
     testcase( rc==SQLITE_BUSY_SNAPSHOT );
     testcase( rc==SQLITE_BUSY_RECOVERY );
-    if( (rc&0xff)==SQLITE_BUSY ){
-      p->pc = (int)(pOp - aOp);
-      p->rc = rc;
-      goto vdbe_return;
-    }
     if( rc!=SQLITE_OK ){
+      if( (rc&0xff)==SQLITE_BUSY ){
+        p->pc = (int)(pOp - aOp);
+        p->rc = rc;
+        goto vdbe_return;
+      }
       goto abort_due_to_error;
     }
 
@@ -3143,10 +3210,9 @@ case OP_Transaction: {
     }
 
     /* Gather the schema version number for checking:
-    ** IMPLEMENTATION-OF: R-32195-19465 The schema version is used by SQLite
-    ** each time a query is executed to ensure that the internal cache of the
-    ** schema used when compiling the SQL query matches the schema of the
-    ** database against which the compiled query is actually executed.
+    ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema
+    ** version is checked to ensure that the schema has not changed since the
+    ** SQL statement was prepared.
     */
     sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta);
     iGen = db->aDb[pOp->p1].pSchema->iGeneration;
@@ -3176,6 +3242,7 @@ case OP_Transaction: {
     p->expired = 1;
     rc = SQLITE_SCHEMA;
   }
+  if( rc ) goto abort_due_to_error;
   break;
 }
 
@@ -3245,6 +3312,7 @@ case OP_SetCookie: {
     sqlite3ExpirePreparedStatements(db);
     p->expired = 0;
   }
+  if( rc ) goto abort_due_to_error;
   break;
 }
 
@@ -3342,7 +3410,7 @@ case OP_OpenWrite:
 
   if( p->expired ){
     rc = SQLITE_ABORT_ROLLBACK;
-    break;
+    goto abort_due_to_error;
   }
 
   nField = 0;
@@ -3366,7 +3434,7 @@ case OP_OpenWrite:
   }
   if( pOp->p5 & OPFLAG_P2ISREG ){
     assert( p2>0 );
-    assert( p2<=(p->nMem-p->nCursor) );
+    assert( p2<=(p->nMem+1 - p->nCursor) );
     pIn2 = &aMem[p2];
     assert( memIsValid(pIn2) );
     assert( (pIn2->flags & MEM_Int)!=0 );
@@ -3376,10 +3444,7 @@ case OP_OpenWrite:
     ** that opcode will always set the p2 value to 2 or more or else fail.
     ** If there were a failure, the prepared statement would have halted
     ** before reaching this instruction. */
-    if( NEVER(p2<2) ) {
-      rc = SQLITE_CORRUPT_BKPT;
-      goto abort_due_to_error;
-    }
+    assert( p2>=2 );
   }
   if( pOp->p4type==P4_KEYINFO ){
     pKeyInfo = pOp->p4.pKeyInfo;
@@ -3417,6 +3482,7 @@ open_cursor_set_hints:
 #endif
   sqlite3BtreeCursorHintFlags(pCur->uc.pCursor,
                                (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ)));
+  if( rc ) goto abort_due_to_error;
   break;
 }
 
@@ -3493,6 +3559,7 @@ case OP_OpenEphemeral: {
       pCx->isTable = 1;
     }
   }
+  if( rc ) goto abort_due_to_error;
   pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED);
   break;
 }
@@ -3518,6 +3585,7 @@ case OP_SorterOpen: {
   assert( pCx->pKeyInfo->db==db );
   assert( pCx->pKeyInfo->enc==ENC(db) );
   rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx);
+  if( rc ) goto abort_due_to_error;
   break;
 }
 
@@ -3805,7 +3873,6 @@ case OP_SeekGT: {       /* jump, in3 */
 #ifdef SQLITE_DEBUG
     { int i; for(i=0; iuc.pCursor, &r, 0, 0, &res);
     if( rc!=SQLITE_OK ){
@@ -3853,7 +3920,6 @@ seek_not_found:
   }
   break;
 }
-  
 
 /* Opcode: Found P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
@@ -3947,13 +4013,13 @@ case OP_Found: {        /* jump, in3 */
     r.pKeyInfo = pC->pKeyInfo;
     r.nField = (u16)pOp->p4.i;
     r.aMem = pIn3;
+#ifdef SQLITE_DEBUG
     for(ii=0; iip3+ii, &r.aMem[ii]);
-#endif
     }
+#endif
     pIdxKey = &r;
   }else{
     pIdxKey = sqlite3VdbeAllocUnpackedRecord(
@@ -3961,7 +4027,7 @@ case OP_Found: {        /* jump, in3 */
     );
     if( pIdxKey==0 ) goto no_mem;
     assert( pIn3->flags & MEM_Blob );
-    ExpandBlob(pIn3);
+    (void)ExpandBlob(pIn3);
     sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey);
   }
   pIdxKey->default_rc = 0;
@@ -3980,7 +4046,7 @@ case OP_Found: {        /* jump, in3 */
   rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res);
   sqlite3DbFree(db, pFree);
   if( rc!=SQLITE_OK ){
-    break;
+    goto abort_due_to_error;
   }
   pC->seekResult = res;
   alreadyExists = (res==0);
@@ -3997,6 +4063,30 @@ case OP_Found: {        /* jump, in3 */
   break;
 }
 
+/* Opcode: SeekRowid P1 P2 P3 * *
+** Synopsis: intkey=r[P3]
+**
+** P1 is the index of a cursor open on an SQL table btree (with integer
+** keys).  If register P3 does not contain an integer or if P1 does not
+** contain a record with rowid P3 then jump immediately to P2.  
+** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain
+** a record with rowid P3 then 
+** leave the cursor pointing at that record and fall through to the next
+** instruction.
+**
+** The OP_NotExists opcode performs the same operation, but with OP_NotExists
+** the P3 register must be guaranteed to contain an integer value.  With this
+** opcode, register P3 might not contain an integer.
+**
+** The OP_NotFound opcode performs the same operation on index btrees
+** (with arbitrary multi-value keys).
+**
+** This opcode leaves the cursor in a state where it cannot be advanced
+** in either direction.  In other words, the Next and Prev opcodes will
+** not work following this opcode.
+**
+** See also: Found, NotFound, NoConflict, SeekRowid
+*/
 /* Opcode: NotExists P1 P2 P3 * *
 ** Synopsis: intkey=r[P3]
 **
@@ -4007,6 +4097,10 @@ case OP_Found: {        /* jump, in3 */
 ** leave the cursor pointing at that record and fall through to the next
 ** instruction.
 **
+** The OP_SeekRowid opcode performs the same operation but also allows the
+** P3 register to contain a non-integer value, in which case the jump is
+** always taken.  This opcode requires that P3 always contain an integer.
+**
 ** The OP_NotFound opcode performs the same operation on index btrees
 ** (with arbitrary multi-value keys).
 **
@@ -4014,14 +4108,21 @@ case OP_Found: {        /* jump, in3 */
 ** in either direction.  In other words, the Next and Prev opcodes will
 ** not work following this opcode.
 **
-** See also: Found, NotFound, NoConflict
+** See also: Found, NotFound, NoConflict, SeekRowid
 */
-case OP_NotExists: {        /* jump, in3 */
+case OP_SeekRowid: {        /* jump, in3 */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
   u64 iKey;
 
+  pIn3 = &aMem[pOp->p3];
+  if( (pIn3->flags & MEM_Int)==0 ){
+    applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding);
+    if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2;
+  }
+  /* Fall through into OP_NotExists */
+case OP_NotExists:          /* jump, in3 */
   pIn3 = &aMem[pOp->p3];
   assert( pIn3->flags & MEM_Int );
   assert( pOp->p1>=0 && pOp->p1nCursor );
@@ -4052,6 +4153,7 @@ case OP_NotExists: {        /* jump, in3 */
       goto jump_to_p2;
     }
   }
+  if( rc ) goto abort_due_to_error;
   break;
 }
 
@@ -4139,8 +4241,7 @@ case OP_NewRowid: {           /* out2 */
         v = 1;   /* IMP: R-61914-48074 */
       }else{
         assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) );
-        rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-        assert( rc==SQLITE_OK );   /* Cannot fail following BtreeLast() */
+        v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
         if( v>=MAX_ROWID ){
           pC->useRandomRowid = 1;
         }else{
@@ -4160,7 +4261,7 @@ case OP_NewRowid: {           /* out2 */
         pMem = &pFrame->aMem[pOp->p3];
       }else{
         /* Assert that P3 is a valid memory cell. */
-        assert( pOp->p3<=(p->nMem-p->nCursor) );
+        assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
         pMem = &aMem[pOp->p3];
         memAboutToChange(p, pMem);
       }
@@ -4194,7 +4295,8 @@ case OP_NewRowid: {           /* out2 */
                                                  0, &res))==SQLITE_OK)
             && (res==0)
             && (++cnt<100));
-      if( rc==SQLITE_OK && res==0 ){
+      if( rc ) goto abort_due_to_error;
+      if( res==0 ){
         rc = SQLITE_FULL;   /* IMP: R-38219-53002 */
         goto abort_due_to_error;
       }
@@ -4222,10 +4324,12 @@ case OP_NewRowid: {           /* out2 */
 ** sqlite3_last_insert_rowid() function (otherwise it is unmodified).
 **
 ** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of
-** the last seek operation (OP_NotExists) was a success, then this
+** the last seek operation (OP_NotExists or OP_SeekRowid) was a success,
+** then this
 ** operation will not attempt to find the appropriate row before doing
 ** the insert but will instead overwrite the row that the cursor is
-** currently pointing to.  Presumably, the prior OP_NotExists opcode
+** currently pointing to.  Presumably, the prior OP_NotExists or
+** OP_SeekRowid opcode
 ** has already positioned the cursor correctly.  This is an optimization
 ** that boosts performance by avoiding redundant seeks.
 **
@@ -4234,9 +4338,9 @@ case OP_NewRowid: {           /* out2 */
 ** is part of an INSERT operation.  The difference is only important to
 ** the update hook.
 **
-** Parameter P4 may point to a string containing the table-name, or
-** may be NULL. If it is not NULL, then the update-hook 
-** (sqlite3.xUpdateCallback) is invoked following a successful insert.
+** Parameter P4 may point to a Table structure, or may be NULL. If it is 
+** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked 
+** following a successful insert.
 **
 ** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically
 ** allocated, then ownership of P2 is transferred to the pseudo-cursor
@@ -4248,7 +4352,7 @@ case OP_NewRowid: {           /* out2 */
 ** for indices is OP_IdxInsert.
 */
 /* Opcode: InsertInt P1 P2 P3 P4 P5
-** Synopsis:  intkey=P3 data=r[P2]
+** Synopsis: intkey=P3 data=r[P2]
 **
 ** This works exactly like OP_Insert except that the key is the
 ** integer value P3, not the value of the integer stored in register P3.
@@ -4257,14 +4361,14 @@ case OP_Insert:
 case OP_InsertInt: {
   Mem *pData;       /* MEM cell holding data for the record to be inserted */
   Mem *pKey;        /* MEM cell holding key  for the record */
-  i64 iKey;         /* The integer ROWID or key for the record to be inserted */
   VdbeCursor *pC;   /* Cursor to table into which insert is written */
-  int nZero;        /* Number of zero-bytes to append */
   int seekResult;   /* Result of prior seek or 0 if no USESEEKRESULT flag */
   const char *zDb;  /* database name - used by the update hook */
-  const char *zTbl; /* Table name - used by the opdate hook */
+  Table *pTab;      /* Table structure - used by update and pre-update hooks */
   int op;           /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
+  BtreePayload x;   /* Payload to be inserted */
 
+  op = 0;
   pData = &aMem[pOp->p2];
   assert( pOp->p1>=0 && pOp->p1nCursor );
   assert( memIsValid(pData) );
@@ -4273,6 +4377,7 @@ case OP_InsertInt: {
   assert( pC->eCurType==CURTYPE_BTREE );
   assert( pC->uc.pCursor!=0 );
   assert( pC->isTable );
+  assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC );
   REGISTER_TRACE(pOp->p2, pData);
 
   if( pOp->opcode==OP_Insert ){
@@ -4280,46 +4385,66 @@ case OP_InsertInt: {
     assert( pKey->flags & MEM_Int );
     assert( memIsValid(pKey) );
     REGISTER_TRACE(pOp->p3, pKey);
-    iKey = pKey->u.i;
+    x.nKey = pKey->u.i;
   }else{
     assert( pOp->opcode==OP_InsertInt );
-    iKey = pOp->p3;
+    x.nKey = pOp->p3;
   }
 
+  if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
+    assert( pC->isTable );
+    assert( pC->iDb>=0 );
+    zDb = db->aDb[pC->iDb].zDbSName;
+    pTab = pOp->p4.pTab;
+    assert( HasRowid(pTab) );
+    op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT);
+  }else{
+    pTab = 0; /* Not needed.  Silence a comiler warning. */
+    zDb = 0;  /* Not needed.  Silence a compiler warning. */
+  }
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  /* Invoke the pre-update hook, if any */
+  if( db->xPreUpdateCallback 
+   && pOp->p4type==P4_TABLE
+   && !(pOp->p5 & OPFLAG_ISUPDATE)
+  ){
+    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2);
+  }
+#endif
+
   if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
-  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey;
+  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = x.nKey;
   if( pData->flags & MEM_Null ){
-    pData->z = 0;
-    pData->n = 0;
+    x.pData = 0;
+    x.nData = 0;
   }else{
     assert( pData->flags & (MEM_Blob|MEM_Str) );
+    x.pData = pData->z;
+    x.nData = pData->n;
   }
   seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0);
   if( pData->flags & MEM_Zero ){
-    nZero = pData->u.nZero;
+    x.nZero = pData->u.nZero;
   }else{
-    nZero = 0;
+    x.nZero = 0;
   }
-  rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey,
-                          pData->z, pData->n, nZero,
+  x.pKey = 0;
+  rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
                           (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
   );
   pC->deferredMoveto = 0;
   pC->cacheStatus = CACHE_STALE;
 
   /* Invoke the update-hook if required. */
-  if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z ){
-    zDb = db->aDb[pC->iDb].zName;
-    zTbl = pOp->p4.z;
-    op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT);
-    assert( pC->isTable );
-    db->xUpdateCallback(db->pUpdateArg, op, zDb, zTbl, iKey);
-    assert( pC->iDb>=0 );
+  if( rc ) goto abort_due_to_error;
+  if( db->xUpdateCallback && op ){
+    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
   }
   break;
 }
 
-/* Opcode: Delete P1 P2 * P4 P5
+/* Opcode: Delete P1 P2 P3 P4 P5
 **
 ** Delete the record at which the P1 cursor is currently pointing.
 **
@@ -4343,15 +4468,24 @@ case OP_InsertInt: {
 ** P1 must not be pseudo-table.  It has to be a real table with
 ** multiple rows.
 **
-** If P4 is not NULL, then it is the name of the table that P1 is
-** pointing to.  The update hook will be invoked, if it exists.
-** If P4 is not NULL then the P1 cursor must have been positioned
-** using OP_NotFound prior to invoking this opcode.
+** If P4 is not NULL then it points to a Table object. In this case either 
+** the update or pre-update hook, or both, may be invoked. The P1 cursor must
+** have been positioned using OP_NotFound prior to invoking this opcode in 
+** this case. Specifically, if one is configured, the pre-update hook is 
+** invoked if P4 is not NULL. The update-hook is invoked if one is configured, 
+** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2.
+**
+** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address
+** of the memory cell that contains the value that the rowid of the row will
+** be set to by the update.
 */
 case OP_Delete: {
   VdbeCursor *pC;
-  u8 hasUpdateCallback;
+  const char *zDb;
+  Table *pTab;
+  int opflags;
 
+  opflags = pOp->p2;
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
@@ -4359,22 +4493,47 @@ case OP_Delete: {
   assert( pC->uc.pCursor!=0 );
   assert( pC->deferredMoveto==0 );
 
-  hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable;
-  if( pOp->p5 && hasUpdateCallback ){
-    sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget);
-  }
-
 #ifdef SQLITE_DEBUG
-  /* The seek operation that positioned the cursor prior to OP_Delete will
-  ** have also set the pC->movetoTarget field to the rowid of the row that
-  ** is being deleted */
-  if( pOp->p4.z && pC->isTable && pOp->p5==0 ){
-    i64 iKey = 0;
-    sqlite3BtreeKeySize(pC->uc.pCursor, &iKey);
-    assert( pC->movetoTarget==iKey ); 
+  if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){
+    /* If p5 is zero, the seek operation that positioned the cursor prior to
+    ** OP_Delete will have also set the pC->movetoTarget field to the rowid of
+    ** the row that is being deleted */
+    i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor);
+    assert( pC->movetoTarget==iKey );
   }
 #endif
 
+  /* If the update-hook or pre-update-hook will be invoked, set zDb to
+  ** the name of the db to pass as to it. Also set local pTab to a copy
+  ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was
+  ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set 
+  ** VdbeCursor.movetoTarget to the current rowid.  */
+  if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
+    assert( pC->iDb>=0 );
+    assert( pOp->p4.pTab!=0 );
+    zDb = db->aDb[pC->iDb].zDbSName;
+    pTab = pOp->p4.pTab;
+    if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){
+      pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor);
+    }
+  }else{
+    zDb = 0;   /* Not needed.  Silence a compiler warning. */
+    pTab = 0;  /* Not needed.  Silence a compiler warning. */
+  }
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  /* Invoke the pre-update-hook if required. */
+  if( db->xPreUpdateCallback && pOp->p4.pTab && HasRowid(pTab) ){
+    assert( !(opflags & OPFLAG_ISUPDATE) || (aMem[pOp->p3].flags & MEM_Int) );
+    sqlite3VdbePreUpdateHook(p, pC,
+        (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, 
+        zDb, pTab, pC->movetoTarget,
+        pOp->p3
+    );
+  }
+  if( opflags & OPFLAG_ISNOOP ) break;
+#endif
+ 
   /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ 
   assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 );
   assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION );
@@ -4396,14 +4555,18 @@ case OP_Delete: {
 
   rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5);
   pC->cacheStatus = CACHE_STALE;
+  if( rc ) goto abort_due_to_error;
 
   /* Invoke the update-hook if required. */
-  if( rc==SQLITE_OK && hasUpdateCallback ){
-    db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE,
-                        db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget);
-    assert( pC->iDb>=0 );
+  if( opflags & OPFLAG_NCHANGE ){
+    p->nChange++;
+    if( db->xUpdateCallback && HasRowid(pTab) ){
+      db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName,
+          pC->movetoTarget);
+      assert( pC->iDb>=0 );
+    }
   }
-  if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++;
+
   break;
 }
 /* Opcode: ResetCount * * * * *
@@ -4420,7 +4583,7 @@ case OP_ResetCount: {
 }
 
 /* Opcode: SorterCompare P1 P2 P3 P4
-** Synopsis:  if key(P1)!=trim(r[P3],P4) goto P2
+** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2
 **
 ** P1 is a sorter cursor. This instruction compares a prefix of the
 ** record blob in register P3 against a prefix of the entry that 
@@ -4447,6 +4610,7 @@ case OP_SorterCompare: {
   res = 0;
   rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res);
   VdbeBranchTaken(res!=0,2);
+  if( rc ) goto abort_due_to_error;
   if( res ) goto jump_to_p2;
   break;
 };
@@ -4472,6 +4636,7 @@ case OP_SorterData: {
   rc = sqlite3VdbeSorterRowkey(pC, pOut);
   assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) );
   assert( pOp->p1>=0 && pOp->p1nCursor );
+  if( rc ) goto abort_due_to_error;
   p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE;
   break;
 }
@@ -4503,7 +4668,6 @@ case OP_RowData: {
   VdbeCursor *pC;
   BtCursor *pCrsr;
   u32 n;
-  i64 n64;
 
   pOut = &aMem[pOp->p2];
   memAboutToChange(p, pOut);
@@ -4521,8 +4685,9 @@ case OP_RowData: {
   pCrsr = pC->uc.pCursor;
 
   /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or
-  ** OP_Rewind/Op_Next with no intervening instructions that might invalidate
-  ** the cursor.  If this where not the case, on of the following assert()s
+  ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions
+  ** that might invalidate the cursor.
+  ** If this where not the case, on of the following assert()s
   ** would fail.  Should this ever change (because of changes in the code
   ** generator) then the fix would be to insert a call to
   ** sqlite3VdbeCursorMoveto().
@@ -4534,20 +4699,9 @@ case OP_RowData: {
   if( rc!=SQLITE_OK ) goto abort_due_to_error;
 #endif
 
-  if( pC->isTable==0 ){
-    assert( !pC->isTable );
-    VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64);
-    assert( rc==SQLITE_OK );    /* True because of CursorMoveto() call above */
-    if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
-    n = (u32)n64;
-  }else{
-    VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n);
-    assert( rc==SQLITE_OK );    /* DataSize() cannot fail */
-    if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
+  n = sqlite3BtreePayloadSize(pCrsr);
+  if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+    goto too_big;
   }
   testcase( n==0 );
   if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){
@@ -4560,6 +4714,7 @@ case OP_RowData: {
   }else{
     rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z);
   }
+  if( rc ) goto abort_due_to_error;
   pOut->enc = SQLITE_UTF8;  /* In case the blob is ever cast to text */
   UPDATE_MAX_BLOBSIZE(pOut);
   REGISTER_TRACE(pOp->p2, pOut);
@@ -4600,6 +4755,7 @@ case OP_Rowid: {                 /* out2 */
     assert( pModule->xRowid );
     rc = pModule->xRowid(pC->uc.pVCur, &v);
     sqlite3VtabImportErrmsg(p, pVtab);
+    if( rc ) goto abort_due_to_error;
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
   }else{
     assert( pC->eCurType==CURTYPE_BTREE );
@@ -4610,8 +4766,7 @@ case OP_Rowid: {                 /* out2 */
       pOut->flags = MEM_Null;
       break;
     }
-    rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
+    v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
   }
   pOut->u.i = v;
   break;
@@ -4670,6 +4825,7 @@ case OP_Last: {        /* jump */
 #ifdef SQLITE_DEBUG
   pC->seekOp = OP_Last;
 #endif
+  if( rc ) goto abort_due_to_error;
   if( pOp->p2>0 ){
     VdbeBranchTaken(res!=0,2);
     if( res ) goto jump_to_p2;
@@ -4734,6 +4890,7 @@ case OP_Rewind: {        /* jump */
     pC->deferredMoveto = 0;
     pC->cacheStatus = CACHE_STALE;
   }
+  if( rc ) goto abort_due_to_error;
   pC->nullRow = (u8)res;
   assert( pOp->p2>0 && pOp->p2nOp );
   VdbeBranchTaken(res!=0,2);
@@ -4846,6 +5003,7 @@ case OP_Next:          /* jump */
 next_tail:
   pC->cacheStatus = CACHE_STALE;
   VdbeBranchTaken(res==0,2);
+  if( rc ) goto abort_due_to_error;
   if( res==0 ){
     pC->nullRow = 0;
     p->aCounter[pOp->p5]++;
@@ -4883,8 +5041,7 @@ next_tail:
 case OP_SorterInsert:       /* in2 */
 case OP_IdxInsert: {        /* in2 */
   VdbeCursor *pC;
-  int nKey;
-  const char *zKey;
+  BtreePayload x;
 
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
@@ -4896,19 +5053,19 @@ case OP_IdxInsert: {        /* in2 */
   assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert );
   assert( pC->isTable==0 );
   rc = ExpandBlob(pIn2);
-  if( rc==SQLITE_OK ){
-    if( pOp->opcode==OP_SorterInsert ){
-      rc = sqlite3VdbeSorterWrite(pC, pIn2);
-    }else{
-      nKey = pIn2->n;
-      zKey = pIn2->z;
-      rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, 
-          ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
-          );
-      assert( pC->deferredMoveto==0 );
-      pC->cacheStatus = CACHE_STALE;
-    }
+  if( rc ) goto abort_due_to_error;
+  if( pOp->opcode==OP_SorterInsert ){
+    rc = sqlite3VdbeSorterWrite(pC, pIn2);
+  }else{
+    x.nKey = pIn2->n;
+    x.pKey = pIn2->z;
+    rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, pOp->p3, 
+        ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
+        );
+    assert( pC->deferredMoveto==0 );
+    pC->cacheStatus = CACHE_STALE;
   }
+  if( rc) goto abort_due_to_error;
   break;
 }
 
@@ -4926,7 +5083,7 @@ case OP_IdxDelete: {
   UnpackedRecord r;
 
   assert( pOp->p3>0 );
-  assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 );
+  assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 );
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
@@ -4939,8 +5096,10 @@ case OP_IdxDelete: {
   r.default_rc = 0;
   r.aMem = &aMem[pOp->p2];
   rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res);
-  if( rc==SQLITE_OK && res==0 ){
+  if( rc ) goto abort_due_to_error;
+  if( res==0 ){
     rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE);
+    if( rc ) goto abort_due_to_error;
   }
   assert( pC->deferredMoveto==0 );
   pC->cacheStatus = CACHE_STALE;
@@ -4948,7 +5107,7 @@ case OP_IdxDelete: {
 }
 
 /* Opcode: Seek P1 * P3 P4 *
-** Synopsis:  Move P3 to P1.rowid
+** Synopsis: Move P3 to P1.rowid
 **
 ** P1 is an open index cursor and P3 is a cursor on the corresponding
 ** table.  This opcode does a deferred seek of the P3 table cursor
@@ -5115,6 +5274,7 @@ case OP_IdxGE:  {       /* jump */
     res++;
   }
   VdbeBranchTaken(res>0,2);
+  if( rc ) goto abort_due_to_error;
   if( res>0 ) goto jump_to_p2;
   break;
 }
@@ -5150,6 +5310,7 @@ case OP_Destroy: {     /* out2 */
   if( db->nVdbeRead > db->nVDestroy+1 ){
     rc = SQLITE_LOCKED;
     p->errorAction = OE_Abort;
+    goto abort_due_to_error;
   }else{
     iDb = pOp->p3;
     assert( DbMaskTest(p->btreeMask, iDb) );
@@ -5157,8 +5318,9 @@ case OP_Destroy: {     /* out2 */
     rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved);
     pOut->flags = MEM_Int;
     pOut->u.i = iMoved;
+    if( rc ) goto abort_due_to_error;
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    if( rc==SQLITE_OK && iMoved!=0 ){
+    if( iMoved!=0 ){
       sqlite3RootPageMoved(db, iDb, iMoved, pOp->p1);
       /* All OP_Destroy operations occur on the same btree */
       assert( resetSchemaOnFault==0 || resetSchemaOnFault==iDb+1 );
@@ -5204,6 +5366,7 @@ case OP_Clear: {
       aMem[pOp->p3].u.i += nChange;
     }
   }
+  if( rc ) goto abort_due_to_error;
   break;
 }
 
@@ -5227,6 +5390,7 @@ case OP_ResetSorter: {
     assert( pC->eCurType==CURTYPE_BTREE );
     assert( pC->isEphemeral );
     rc = sqlite3BtreeClearTableOfCursor(pC->uc.pCursor);
+    if( rc ) goto abort_due_to_error;
   }
   break;
 }
@@ -5275,6 +5439,7 @@ case OP_CreateTable: {          /* out2 */
     flags = BTREE_BLOBKEY;
   }
   rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags);
+  if( rc ) goto abort_due_to_error;
   pOut->u.i = pgno;
   break;
 }
@@ -5313,9 +5478,9 @@ case OP_ParseSchema: {
     initData.pzErrMsg = &p->zErrMsg;
     zSql = sqlite3MPrintf(db,
        "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid",
-       db->aDb[iDb].zName, zMaster, pOp->p4.z);
+       db->aDb[iDb].zDbSName, zMaster, pOp->p4.z);
     if( zSql==0 ){
-      rc = SQLITE_NOMEM;
+      rc = SQLITE_NOMEM_BKPT;
     }else{
       assert( db->init.busy==0 );
       db->init.busy = 1;
@@ -5327,9 +5492,12 @@ case OP_ParseSchema: {
       db->init.busy = 0;
     }
   }
-  if( rc ) sqlite3ResetAllSchemasOfConnection(db);
-  if( rc==SQLITE_NOMEM ){
-    goto no_mem;
+  if( rc ){
+    sqlite3ResetAllSchemasOfConnection(db);
+    if( rc==SQLITE_NOMEM ){
+      goto no_mem;
+    }
+    goto abort_due_to_error;
   }
   break;  
 }
@@ -5344,6 +5512,7 @@ case OP_ParseSchema: {
 case OP_LoadAnalysis: {
   assert( pOp->p1>=0 && pOp->p1nDb );
   rc = sqlite3AnalysisLoad(db, pOp->p1);
+  if( rc ) goto abort_due_to_error;
   break;  
 }
 #endif /* !defined(SQLITE_OMIT_ANALYZE) */
@@ -5389,7 +5558,7 @@ case OP_DropTrigger: {
 
 
 #ifndef SQLITE_OMIT_INTEGRITY_CHECK
-/* Opcode: IntegrityCk P1 P2 P3 * P5
+/* Opcode: IntegrityCk P1 P2 P3 P4 P5
 **
 ** Do an analysis of the currently open database.  Store in
 ** register P1 the text of an error message describing any problems.
@@ -5400,9 +5569,8 @@ case OP_DropTrigger: {
 ** In other words, the analysis stops as soon as reg(P1) errors are 
 ** seen.  Reg(P1) is updated with the number of errors remaining.
 **
-** The root page numbers of all tables in the database are integer
-** stored in reg(P1), reg(P1+1), reg(P1+2), ....  There are P2 tables
-** total.
+** The root page numbers of all tables in the database are integers
+** stored in P4_INTARRAY argument.
 **
 ** If P5 is not zero, the check is done on the auxiliary database
 ** file, not the main database file.
@@ -5412,30 +5580,24 @@ case OP_DropTrigger: {
 case OP_IntegrityCk: {
   int nRoot;      /* Number of tables to check.  (Number of root pages.) */
   int *aRoot;     /* Array of rootpage numbers for tables to be checked */
-  int j;          /* Loop counter */
   int nErr;       /* Number of errors reported */
   char *z;        /* Text of the error report */
   Mem *pnErr;     /* Register keeping track of errors remaining */
 
   assert( p->bIsReader );
   nRoot = pOp->p2;
+  aRoot = pOp->p4.ai;
   assert( nRoot>0 );
-  aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(nRoot+1) );
-  if( aRoot==0 ) goto no_mem;
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( aRoot[nRoot]==0 );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pnErr = &aMem[pOp->p3];
   assert( (pnErr->flags & MEM_Int)!=0 );
   assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 );
   pIn1 = &aMem[pOp->p1];
-  for(j=0; jp5nDb );
   assert( DbMaskTest(p->btreeMask, pOp->p5) );
   z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot,
                                  (int)pnErr->u.i, &nErr);
-  sqlite3DbFree(db, aRoot);
   pnErr->u.i -= nErr;
   sqlite3VdbeMemSetNull(pIn1);
   if( nErr==0 ){
@@ -5452,7 +5614,7 @@ case OP_IntegrityCk: {
 #endif /* SQLITE_OMIT_INTEGRITY_CHECK */
 
 /* Opcode: RowSetAdd P1 P2 * * *
-** Synopsis:  rowset(P1)=r[P2]
+** Synopsis: rowset(P1)=r[P2]
 **
 ** Insert the integer value held by register P2 into a boolean index
 ** held in register P1.
@@ -5472,7 +5634,7 @@ case OP_RowSetAdd: {       /* in1, in2 */
 }
 
 /* Opcode: RowSetRead P1 P2 P3 * *
-** Synopsis:  r[P3]=rowset(P1)
+** Synopsis: r[P3]=rowset(P1)
 **
 ** Extract the smallest value from boolean index P1 and put that value into
 ** register P3.  Or, if boolean index P1 is initially empty, leave P3
@@ -5603,7 +5765,7 @@ case OP_Program: {        /* jump */
   if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){
     rc = SQLITE_ERROR;
     sqlite3VdbeError(p, "too many levels of trigger recursion");
-    break;
+    goto abort_due_to_error;
   }
 
   /* Register pRt is used to store the memory required to save the state
@@ -5617,10 +5779,11 @@ case OP_Program: {        /* jump */
     ** variable nMem (and later, VdbeFrame.nChildMem) to this value.
     */
     nMem = pProgram->nMem + pProgram->nCsr;
+    assert( nMem>0 );
+    if( pProgram->nCsr==0 ) nMem++;
     nByte = ROUND8(sizeof(VdbeFrame))
               + nMem * sizeof(Mem)
-              + pProgram->nCsr * sizeof(VdbeCursor *)
-              + pProgram->nOnce * sizeof(u8);
+              + pProgram->nCsr * sizeof(VdbeCursor *);
     pFrame = sqlite3DbMallocZero(db, nByte);
     if( !pFrame ){
       goto no_mem;
@@ -5640,8 +5803,6 @@ case OP_Program: {        /* jump */
     pFrame->aOp = p->aOp;
     pFrame->nOp = p->nOp;
     pFrame->token = pProgram->token;
-    pFrame->aOnceFlag = p->aOnceFlag;
-    pFrame->nOnceFlag = p->nOnceFlag;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
     pFrame->anExec = p->anExec;
 #endif
@@ -5653,7 +5814,8 @@ case OP_Program: {        /* jump */
     }
   }else{
     pFrame = pRt->u.pFrame;
-    assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem );
+    assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem 
+        || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) );
     assert( pProgram->nCsr==pFrame->nChildCsr );
     assert( (int)(pOp - aOp)==pFrame->pc );
   }
@@ -5663,21 +5825,21 @@ case OP_Program: {        /* jump */
   pFrame->lastRowid = lastRowid;
   pFrame->nChange = p->nChange;
   pFrame->nDbChange = p->db->nChange;
+  assert( pFrame->pAuxData==0 );
+  pFrame->pAuxData = p->pAuxData;
+  p->pAuxData = 0;
   p->nChange = 0;
   p->pFrame = pFrame;
-  p->aMem = aMem = &VdbeFrameMem(pFrame)[-1];
+  p->aMem = aMem = VdbeFrameMem(pFrame);
   p->nMem = pFrame->nChildMem;
   p->nCursor = (u16)pFrame->nChildCsr;
-  p->apCsr = (VdbeCursor **)&aMem[p->nMem+1];
+  p->apCsr = (VdbeCursor **)&aMem[p->nMem];
   p->aOp = aOp = pProgram->aOp;
   p->nOp = pProgram->nOp;
-  p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor];
-  p->nOnceFlag = pProgram->nOnce;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   p->anExec = 0;
 #endif
   pOp = &aOp[-1];
-  memset(p->aOnceFlag, 0, p->nOnceFlag);
 
   break;
 }
@@ -5865,21 +6027,6 @@ case OP_DecrJumpZero: {      /* jump, in1 */
 }
 
 
-/* Opcode: JumpZeroIncr P1 P2 * * *
-** Synopsis: if (r[P1]++)==0 ) goto P2
-**
-** The register P1 must contain an integer.  If register P1 is initially
-** zero, then jump to P2.  Increment register P1 regardless of whether or
-** not the jump is taken.
-*/
-case OP_JumpZeroIncr: {        /* jump, in1 */
-  pIn1 = &aMem[pOp->p1];
-  assert( pIn1->flags&MEM_Int );
-  VdbeBranchTaken(pIn1->u.i==0, 2);
-  if( (pIn1->u.i++)==0 ) goto jump_to_p2;
-  break;
-}
-
 /* Opcode: AggStep0 * P2 P3 P4 P5
 ** Synopsis: accum=r[P3] step(r[P2@P5])
 **
@@ -5914,8 +6061,8 @@ case OP_AggStep0: {
 
   assert( pOp->p4type==P4_FUNCDEF );
   n = pOp->p5;
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
-  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
+  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) );
   assert( pOp->p3p2 || pOp->p3>=pOp->p2+n );
   pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*));
   if( pCtx==0 ) goto no_mem;
@@ -5967,6 +6114,7 @@ case OP_AggStep: {
       rc = pCtx->isError;
     }
     sqlite3VdbeMemRelease(&t);
+    if( rc ) goto abort_due_to_error;
   }else{
     assert( t.flags==MEM_Null );
   }
@@ -5993,12 +6141,13 @@ case OP_AggStep: {
 */
 case OP_AggFinal: {
   Mem *pMem;
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
   pMem = &aMem[pOp->p1];
   assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
   rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc);
   if( rc ){
     sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem));
+    goto abort_due_to_error;
   }
   sqlite3VdbeChangeEncoding(pMem, encoding);
   UPDATE_MAX_BLOBSIZE(pMem);
@@ -6034,7 +6183,8 @@ case OP_Checkpoint: {
        || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE
   );
   rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]);
-  if( rc==SQLITE_BUSY ){
+  if( rc ){
+    if( rc!=SQLITE_BUSY ) goto abort_due_to_error;
     rc = SQLITE_OK;
     aRes[0] = 1;
   }
@@ -6107,7 +6257,7 @@ case OP_JournalMode: {    /* out2 */
           "cannot change %s wal mode from within a transaction",
           (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of")
       );
-      break;
+      goto abort_due_to_error;
     }else{
  
       if( eOld==PAGER_JOURNALMODE_WAL ){
@@ -6137,9 +6287,7 @@ case OP_JournalMode: {    /* out2 */
   }
 #endif /* ifndef SQLITE_OMIT_WAL */
 
-  if( rc ){
-    eNew = eOld;
-  }
+  if( rc ) eNew = eOld;
   eNew = sqlite3PagerSetJournalMode(pPager, eNew);
 
   pOut->flags = MEM_Str|MEM_Static|MEM_Term;
@@ -6147,20 +6295,21 @@ case OP_JournalMode: {    /* out2 */
   pOut->n = sqlite3Strlen30(pOut->z);
   pOut->enc = SQLITE_UTF8;
   sqlite3VdbeChangeEncoding(pOut, encoding);
+  if( rc ) goto abort_due_to_error;
   break;
 };
 #endif /* SQLITE_OMIT_PRAGMA */
 
 #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
-/* Opcode: Vacuum * * * * *
+/* Opcode: Vacuum P1 * * * *
 **
-** Vacuum the entire database.  This opcode will cause other virtual
-** machines to be created and run.  It may not be called from within
-** a transaction.
+** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
+** for an attached database.  The "temp" database may not be vacuumed.
 */
 case OP_Vacuum: {
   assert( p->readOnly==0 );
-  rc = sqlite3RunVacuum(&p->zErrMsg, db);
+  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif
@@ -6181,7 +6330,8 @@ case OP_IncrVacuum: {        /* jump */
   pBt = db->aDb[pOp->p1].pBt;
   rc = sqlite3BtreeIncrVacuum(pBt);
   VdbeBranchTaken(rc==SQLITE_DONE,2);
-  if( rc==SQLITE_DONE ){
+  if( rc ){
+    if( rc!=SQLITE_DONE ) goto abort_due_to_error;
     rc = SQLITE_OK;
     goto jump_to_p2;
   }
@@ -6232,9 +6382,12 @@ case OP_TableLock: {
     assert( DbMaskTest(p->btreeMask, p1) );
     assert( isWriteLock==0 || isWriteLock==1 );
     rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock);
-    if( (rc&0xFF)==SQLITE_LOCKED ){
-      const char *z = pOp->p4.z;
-      sqlite3VdbeError(p, "database table is locked: %s", z);
+    if( rc ){
+      if( (rc&0xFF)==SQLITE_LOCKED ){
+        const char *z = pOp->p4.z;
+        sqlite3VdbeError(p, "database table is locked: %s", z);
+      }
+      goto abort_due_to_error;
     }
   }
   break;
@@ -6256,6 +6409,7 @@ case OP_VBegin: {
   pVTab = pOp->p4.pVtab;
   rc = sqlite3VtabBegin(db, pVTab);
   if( pVTab ) sqlite3VtabImportErrmsg(p, pVTab->pVtab);
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -6284,6 +6438,7 @@ case OP_VCreate: {
     rc = sqlite3VtabCallCreate(db, pOp->p1, zTab, &p->zErrMsg);
   }
   sqlite3VdbeMemRelease(&sMem);
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -6298,6 +6453,7 @@ case OP_VDestroy: {
   db->nVDestroy++;
   rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z);
   db->nVDestroy--;
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -6321,25 +6477,25 @@ case OP_VOpen: {
   pVtab = pOp->p4.pVtab->pVtab;
   if( pVtab==0 || NEVER(pVtab->pModule==0) ){
     rc = SQLITE_LOCKED;
-    break;
+    goto abort_due_to_error;
   }
   pModule = pVtab->pModule;
   rc = pModule->xOpen(pVtab, &pVCur);
   sqlite3VtabImportErrmsg(p, pVtab);
-  if( SQLITE_OK==rc ){
-    /* Initialize sqlite3_vtab_cursor base class */
-    pVCur->pVtab = pVtab;
+  if( rc ) goto abort_due_to_error;
 
-    /* Initialize vdbe cursor object */
-    pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB);
-    if( pCur ){
-      pCur->uc.pVCur = pVCur;
-      pVtab->nRef++;
-    }else{
-      assert( db->mallocFailed );
-      pModule->xClose(pVCur);
-      goto no_mem;
-    }
+  /* Initialize sqlite3_vtab_cursor base class */
+  pVCur->pVtab = pVtab;
+
+  /* Initialize vdbe cursor object */
+  pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB);
+  if( pCur ){
+    pCur->uc.pVCur = pVCur;
+    pVtab->nRef++;
+  }else{
+    assert( db->mallocFailed );
+    pModule->xClose(pVCur);
+    goto no_mem;
   }
   break;
 }
@@ -6401,9 +6557,8 @@ case OP_VFilter: {   /* jump */
   }
   rc = pModule->xFilter(pVCur, iQuery, pOp->p4.z, nArg, apArg);
   sqlite3VtabImportErrmsg(p, pVtab);
-  if( rc==SQLITE_OK ){
-    res = pModule->xEof(pVCur);
-  }
+  if( rc ) goto abort_due_to_error;
+  res = pModule->xEof(pVCur);
   pCur->nullRow = 0;
   VdbeBranchTaken(res!=0,2);
   if( res ) goto jump_to_p2;
@@ -6427,7 +6582,7 @@ case OP_VColumn: {
 
   VdbeCursor *pCur = p->apCsr[pOp->p1];
   assert( pCur->eCurType==CURTYPE_VTAB );
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pDest = &aMem[pOp->p3];
   memAboutToChange(p, pDest);
   if( pCur->nullRow ){
@@ -6452,6 +6607,7 @@ case OP_VColumn: {
   if( sqlite3VdbeMemTooBig(pDest) ){
     goto too_big;
   }
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -6487,9 +6643,8 @@ case OP_VNext: {   /* jump */
   */
   rc = pModule->xNext(pCur->uc.pVCur);
   sqlite3VtabImportErrmsg(p, pVtab);
-  if( rc==SQLITE_OK ){
-    res = pModule->xEof(pCur->uc.pVCur);
-  }
+  if( rc ) goto abort_due_to_error;
+  res = pModule->xEof(pCur->uc.pVCur);
   VdbeBranchTaken(!res,2);
   if( !res ){
     /* If there is data, jump to P2 */
@@ -6521,11 +6676,11 @@ case OP_VRename: {
   testcase( pName->enc==SQLITE_UTF16BE );
   testcase( pName->enc==SQLITE_UTF16LE );
   rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8);
-  if( rc==SQLITE_OK ){
-    rc = pVtab->pModule->xRename(pVtab, pName->z);
-    sqlite3VtabImportErrmsg(p, pVtab);
-    p->expired = 0;
-  }
+  if( rc ) goto abort_due_to_error;
+  rc = pVtab->pModule->xRename(pVtab, pName->z);
+  sqlite3VtabImportErrmsg(p, pVtab);
+  p->expired = 0;
+  if( rc ) goto abort_due_to_error;
   break;
 }
 #endif
@@ -6574,7 +6729,7 @@ case OP_VUpdate: {
   pVtab = pOp->p4.pVtab->pVtab;
   if( pVtab==0 || NEVER(pVtab->pModule==0) ){
     rc = SQLITE_LOCKED;
-    break;
+    goto abort_due_to_error;
   }
   pModule = pVtab->pModule;
   nArg = pOp->p2;
@@ -6606,6 +6761,7 @@ case OP_VUpdate: {
     }else{
       p->nChange++;
     }
+    if( rc ) goto abort_due_to_error;
   }
   break;
 }
@@ -6650,8 +6806,8 @@ case OP_MaxPgcnt: {            /* out2 */
 #endif
 
 
-/* Opcode: Init * P2 * P4 *
-** Synopsis:  Start at P2
+/* Opcode: Init P1 P2 * P4 *
+** Synopsis: Start at P2
 **
 ** Programs contain a single instance of this opcode as the very first
 ** opcode.
@@ -6661,27 +6817,50 @@ case OP_MaxPgcnt: {            /* out2 */
 ** Or if P4 is blank, use the string returned by sqlite3_sql().
 **
 ** If P2 is not zero, jump to instruction P2.
+**
+** Increment the value of P1 so that OP_Once opcodes will jump the
+** first time they are evaluated for this run.
 */
 case OP_Init: {          /* jump */
   char *zTrace;
-  char *z;
+  int i;
+
+  /* If the P4 argument is not NULL, then it must be an SQL comment string.
+  ** The "--" string is broken up to prevent false-positives with srcck1.c.
+  **
+  ** This assert() provides evidence for:
+  ** EVIDENCE-OF: R-50676-09860 The callback can compute the same text that
+  ** would have been returned by the legacy sqlite3_trace() interface by
+  ** using the X argument when X begins with "--" and invoking
+  ** sqlite3_expanded_sql(P) otherwise.
+  */
+  assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 );
+  assert( pOp==p->aOp );  /* Always instruction 0 */
 
 #ifndef SQLITE_OMIT_TRACE
-  if( db->xTrace
+  if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0
    && !p->doingRerun
    && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0
   ){
-    z = sqlite3VdbeExpandSql(p, zTrace);
-    db->xTrace(db->pTraceArg, z);
-    sqlite3DbFree(db, z);
+#ifndef SQLITE_OMIT_DEPRECATED
+    if( db->mTrace & SQLITE_TRACE_LEGACY ){
+      void (*x)(void*,const char*) = (void(*)(void*,const char*))db->xTrace;
+      char *z = sqlite3VdbeExpandSql(p, zTrace);
+      x(db->pTraceArg, z);
+      sqlite3_free(z);
+    }else
+#endif
+    {
+      (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace);
+    }
   }
 #ifdef SQLITE_USE_FCNTL_TRACE
   zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
   if( zTrace ){
-    int i;
-    for(i=0; inDb; i++){
-      if( DbMaskTest(p->btreeMask, i)==0 ) continue;
-      sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace);
+    int j;
+    for(j=0; jnDb; j++){
+      if( DbMaskTest(p->btreeMask, j)==0 ) continue;
+      sqlite3_file_control(db, db->aDb[j].zDbSName, SQLITE_FCNTL_TRACE, zTrace);
     }
   }
 #endif /* SQLITE_USE_FCNTL_TRACE */
@@ -6693,8 +6872,15 @@ case OP_Init: {          /* jump */
   }
 #endif /* SQLITE_DEBUG */
 #endif /* SQLITE_OMIT_TRACE */
-  if( pOp->p2 ) goto jump_to_p2;
-  break;
+  assert( pOp->p2>0 );
+  if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){
+    for(i=1; inOp; i++){
+      if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0;
+    }
+    pOp->p1 = 0;
+  }
+  pOp->p1++;
+  goto jump_to_p2;
 }
 
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
@@ -6762,11 +6948,12 @@ default: {          /* This is really OP_Noop and OP_Explain */
 
 #ifdef SQLITE_DEBUG
     if( db->flags & SQLITE_VdbeTrace ){
+      u8 opProperty = sqlite3OpcodeProperty[pOrigOp->opcode];
       if( rc!=0 ) printf("rc=%d\n",rc);
-      if( pOrigOp->opflags & (OPFLG_OUT2) ){
+      if( opProperty & (OPFLG_OUT2) ){
         registerTrace(pOrigOp->p2, &aMem[pOrigOp->p2]);
       }
-      if( pOrigOp->opflags & OPFLG_OUT3 ){
+      if( opProperty & OPFLG_OUT3 ){
         registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]);
       }
     }
@@ -6777,9 +6964,14 @@ default: {          /* This is really OP_Noop and OP_Explain */
   /* If we reach this point, it means that execution is finished with
   ** an error of some kind.
   */
-vdbe_error_halt:
+abort_due_to_error:
+  if( db->mallocFailed ) rc = SQLITE_NOMEM_BKPT;
   assert( rc );
+  if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){
+    sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc));
+  }
   p->rc = rc;
+  sqlite3SystemError(db, rc);
   testcase( sqlite3GlobalConfig.xLog!=0 );
   sqlite3_log(rc, "statement aborts at %d: [%s] %s", 
                    (int)(pOp - aOp), p->zSql, p->zErrMsg);
@@ -6809,34 +7001,23 @@ vdbe_return:
 too_big:
   sqlite3VdbeError(p, "string or blob too big");
   rc = SQLITE_TOOBIG;
-  goto vdbe_error_halt;
+  goto abort_due_to_error;
 
   /* Jump to here if a malloc() fails.
   */
 no_mem:
   sqlite3OomFault(db);
   sqlite3VdbeError(p, "out of memory");
-  rc = SQLITE_NOMEM;
-  goto vdbe_error_halt;
-
-  /* Jump to here for any other kind of fatal error.  The "rc" variable
-  ** should hold the error number.
-  */
-abort_due_to_error:
-  assert( p->zErrMsg==0 );
-  if( db->mallocFailed ) rc = SQLITE_NOMEM;
-  if( rc!=SQLITE_IOERR_NOMEM ){
-    sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc));
-  }
-  goto vdbe_error_halt;
+  rc = SQLITE_NOMEM_BKPT;
+  goto abort_due_to_error;
 
   /* Jump to here if the sqlite3_interrupt() API sets the interrupt
   ** flag.
   */
 abort_due_to_interrupt:
   assert( db->u1.isInterrupted );
-  rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_INTERRUPT;
+  rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT;
   p->rc = rc;
   sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc));
-  goto vdbe_error_halt;
+  goto abort_due_to_error;
 }
diff --git a/src/vdbe.h b/src/vdbe.h
index 4c02f584..ecb2b0e3 100644
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -15,8 +15,8 @@
 ** or VDBE.  The VDBE implements an abstract machine that runs a
 ** simple program to access and modify the underlying database.
 */
-#ifndef _SQLITE_VDBE_H_
-#define _SQLITE_VDBE_H_
+#ifndef SQLITE_VDBE_H
+#define SQLITE_VDBE_H
 #include 
 
 /*
@@ -41,7 +41,7 @@ typedef struct SubProgram SubProgram;
 struct VdbeOp {
   u8 opcode;          /* What operation to perform */
   signed char p4type; /* One of the P4_xxx constants for p4 */
-  u8 opflags;         /* Mask of the OPFLG_* flags in opcodes.h */
+  u8 notUsed1;
   u8 p5;              /* Fifth parameter is an unsigned character */
   int p1;             /* First operand */
   int p2;             /* Second parameter (often the jump destination) */
@@ -60,6 +60,7 @@ struct VdbeOp {
     KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
     int *ai;               /* Used when p4type is P4_INTARRAY */
     SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
+    Table *pTab;           /* Used when p4type is P4_TABLE */
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
     Expr *pExpr;           /* Used when p4type is P4_EXPR */
 #endif
@@ -87,7 +88,6 @@ struct SubProgram {
   int nOp;                      /* Elements in aOp[] */
   int nMem;                     /* Number of memory cells required */
   int nCsr;                     /* Number of cursors required */
-  int nOnce;                    /* Number of OP_Once instructions */
   void *token;                  /* id that may be used to recursive triggers */
   SubProgram *pNext;            /* Next sub-program already visited */
 };
@@ -124,7 +124,8 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */
 #define P4_SUBPROGRAM  (-18) /* P4 is a pointer to a SubProgram structure */
 #define P4_ADVANCE  (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */
-#define P4_FUNCCTX  (-20) /* P4 is a pointer to an sqlite3_context object */
+#define P4_TABLE    (-20) /* P4 is a pointer to a Table structure */
+#define P4_FUNCCTX  (-21) /* P4 is a pointer to an sqlite3_context object */
 
 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1
@@ -202,6 +203,7 @@ void sqlite3VdbeUsesBtree(Vdbe*, int);
 VdbeOp *sqlite3VdbeGetOp(Vdbe*, int);
 int sqlite3VdbeMakeLabel(Vdbe*);
 void sqlite3VdbeRunOnlyOnce(Vdbe*);
+void sqlite3VdbeReusable(Vdbe*);
 void sqlite3VdbeDelete(Vdbe*);
 void sqlite3VdbeClearObject(sqlite3*,Vdbe*);
 void sqlite3VdbeMakeReady(Vdbe*,Parse*);
@@ -306,4 +308,4 @@ void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*);
 # define sqlite3VdbeScanStatus(a,b,c,d,e)
 #endif
 
-#endif
+#endif /* SQLITE_VDBE_H */
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 839c2039..86445cef 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -15,8 +15,8 @@
 ** 6000 lines long) it was split up into several smaller files and
 ** this header information was factored out.
 */
-#ifndef _VDBEINT_H_
-#define _VDBEINT_H_
+#ifndef SQLITE_VDBEINT_H
+#define SQLITE_VDBEINT_H
 
 /*
 ** The maximum number of times that a statement will try to reparse
@@ -52,9 +52,6 @@ typedef unsigned Bool;
 /* Opaque type used by code in vdbesort.c */
 typedef struct VdbeSorter VdbeSorter;
 
-/* Opaque type used by the explainer */
-typedef struct Explain Explain;
-
 /* Elements of the linked list at Vdbe.pAuxData */
 typedef struct AuxData AuxData;
 
@@ -87,7 +84,7 @@ struct VdbeCursor {
 #endif
   Bool isEphemeral:1;   /* True for an ephemeral table */
   Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */
-  Bool isOrdered:1;     /* True if the underlying table is BTREE_UNORDERED */
+  Bool isOrdered:1;     /* True if the table is not BTREE_UNORDERED */
   Pgno pgnoRoot;        /* Root page of the open btree cursor */
   i16 nField;           /* Number of fields in the header */
   u16 nHdrParsed;       /* Number of header fields parsed so far */
@@ -129,6 +126,12 @@ struct VdbeCursor {
   ** aType[] and nField+1 array slots for aOffset[] */
 };
 
+
+/*
+** A value for VdbeCursor.cacheStatus that means the cache is always invalid.
+*/
+#define CACHE_STALE 0
+
 /*
 ** When a sub-program is executed (OP_Program), a structure of this type
 ** is allocated to store the current value of the program counter, as
@@ -157,15 +160,14 @@ struct VdbeFrame {
   Op *aOp;                /* Program instructions for parent frame */
   i64 *anExec;            /* Event counters from parent frame */
   Mem *aMem;              /* Array of memory cells for parent frame */
-  u8 *aOnceFlag;          /* Array of OP_Once flags for parent frame */
   VdbeCursor **apCsr;     /* Array of Vdbe cursors for parent frame */
   void *token;            /* Copy of SubProgram.token */
   i64 lastRowid;          /* Last insert rowid (sqlite3.lastRowid) */
+  AuxData *pAuxData;      /* Linked list of auxdata allocations */
   int nCursor;            /* Number of entries in apCsr */
   int pc;                 /* Program Counter in parent (calling) frame */
   int nOp;                /* Size of aOp array */
   int nMem;               /* Number of entries in aMem */
-  int nOnceFlag;          /* Number of entries in aOnceFlag */
   int nChildMem;          /* Number of memory cells for child frame */
   int nChildCsr;          /* Number of cursors for child frame */
   int nChange;            /* Statement changes (Vdbe.nChange)     */
@@ -174,11 +176,6 @@ struct VdbeFrame {
 
 #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
 
-/*
-** A value for VdbeCursor.cacheValid that means the cache is always invalid.
-*/
-#define CACHE_STALE 0
-
 /*
 ** Internally, the vdbe manipulates nearly all SQL values as Mem
 ** structures. Each Mem struct may cache multiple representations (string,
@@ -319,18 +316,6 @@ struct sqlite3_context {
   sqlite3_value *argv[1]; /* Argument set */
 };
 
-/*
-** An Explain object accumulates indented output which is helpful
-** in describing recursive data structures.
-*/
-struct Explain {
-  Vdbe *pVdbe;       /* Attach the explanation to this Vdbe */
-  StrAccum str;      /* The string being accumulated */
-  int nIndent;       /* Number of elements in aIndent */
-  u16 aIndent[100];  /* Levels of indentation */
-  char zBase[100];   /* Initial space */
-};
-
 /* A bitfield type for use inside of structures.  Always follow with :N where
 ** N is the number of bits.
 */
@@ -355,53 +340,57 @@ struct ScanStatus {
 */
 struct Vdbe {
   sqlite3 *db;            /* The database connection that owns this statement */
+  Vdbe *pPrev,*pNext;     /* Linked list of VDBEs with the same Vdbe.db */
+  Parse *pParse;          /* Parsing context used to create this Vdbe */
+  ynVar nVar;             /* Number of entries in aVar[] */
+  ynVar nzVar;            /* Number of entries in azVar[] */
+  u32 magic;              /* Magic number for sanity checking */
+  int nMem;               /* Number of memory locations currently allocated */
+  int nCursor;            /* Number of slots in apCsr[] */
+  u32 cacheCtr;           /* VdbeCursor row cache generation counter */
+  int pc;                 /* The program counter */
+  int rc;                 /* Value to return */
+  int nChange;            /* Number of db changes made since last reset */
+  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
+  i64 iCurrentTime;       /* Value of julianday('now') for this statement */
+  i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
+  i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
+  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
+
+  /* When allocating a new Vdbe object, all of the fields below should be
+  ** initialized to zero or NULL */
+
   Op *aOp;                /* Space to hold the virtual machine's program */
   Mem *aMem;              /* The memory locations */
   Mem **apArg;            /* Arguments to currently executing user function */
   Mem *aColName;          /* Column names to return */
   Mem *pResultSet;        /* Pointer to an array of results */
-  Parse *pParse;          /* Parsing context used to create this Vdbe */
-  int nMem;               /* Number of memory locations currently allocated */
-  int nOp;                /* Number of instructions in the program */
-  int nCursor;            /* Number of slots in apCsr[] */
-  u32 magic;              /* Magic number for sanity checking */
   char *zErrMsg;          /* Error message written here */
-  Vdbe *pPrev,*pNext;     /* Linked list of VDBEs with the same Vdbe.db */
   VdbeCursor **apCsr;     /* One element of this array for each open cursor */
   Mem *aVar;              /* Values for the OP_Variable opcode. */
   char **azVar;           /* Name of variables */
-  ynVar nVar;             /* Number of entries in aVar[] */
-  ynVar nzVar;            /* Number of entries in azVar[] */
-  u32 cacheCtr;           /* VdbeCursor row cache generation counter */
-  int pc;                 /* The program counter */
-  int rc;                 /* Value to return */
+#ifndef SQLITE_OMIT_TRACE
+  i64 startTime;          /* Time when query started - used for profiling */
+#endif
+  int nOp;                /* Number of instructions in the program */
 #ifdef SQLITE_DEBUG
   int rcApp;              /* errcode set by sqlite3_result_error_code() */
 #endif
   u16 nResColumn;         /* Number of columns in one row of the result set */
   u8 errorAction;         /* Recovery action to do in case of an error */
   u8 minWriteFileFormat;  /* Minimum file format for writable database files */
+  bft expired:1;          /* True if the VM needs to be recompiled */
+  bft doingRerun:1;       /* True if rerunning after an auto-reprepare */
   bft explain:2;          /* True if EXPLAIN present on SQL command */
   bft changeCntOn:1;      /* True to update the change-counter */
-  bft expired:1;          /* True if the VM needs to be recompiled */
   bft runOnlyOnce:1;      /* Automatically expire on reset */
   bft usesStmtJournal:1;  /* True if uses a statement journal */
   bft readOnly:1;         /* True for statements that do not write */
   bft bIsReader:1;        /* True for statements that read */
   bft isPrepareV2:1;      /* True if prepared with prepare_v2() */
-  bft doingRerun:1;       /* True if rerunning after an auto-reprepare */
-  int nChange;            /* Number of db changes made since last reset */
   yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
-  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
   u32 aCounter[5];        /* Counters used by sqlite3_stmt_status() */
-#ifndef SQLITE_OMIT_TRACE
-  i64 startTime;          /* Time when query started - used for profiling */
-#endif
-  i64 iCurrentTime;       /* Value of julianday('now') for this statement */
-  i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
-  i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
-  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   char *zSql;             /* Text of the SQL statement that generated this */
   void *pFree;            /* Free this when deleting the vdbe */
   VdbeFrame *pFrame;      /* Parent frame */
@@ -409,8 +398,6 @@ struct Vdbe {
   int nFrame;             /* Number of frames in pFrame list */
   u32 expmask;            /* Binding to these vars invalidates VM */
   SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */
-  int nOnceFlag;          /* Size of array aOnceFlag[] */
-  u8 *aOnceFlag;          /* Flags for OP_Once */
   AuxData *pAuxData;      /* Linked list of auxdata allocations */
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   i64 *anExec;            /* Number of times each op has been executed */
@@ -422,10 +409,30 @@ struct Vdbe {
 /*
 ** The following are allowed values for Vdbe.magic
 */
-#define VDBE_MAGIC_INIT     0x26bceaa5    /* Building a VDBE program */
-#define VDBE_MAGIC_RUN      0xbdf20da3    /* VDBE is ready to execute */
-#define VDBE_MAGIC_HALT     0x519c2973    /* VDBE has completed execution */
-#define VDBE_MAGIC_DEAD     0xb606c3c8    /* The VDBE has been deallocated */
+#define VDBE_MAGIC_INIT     0x16bceaa5    /* Building a VDBE program */
+#define VDBE_MAGIC_RUN      0x2df20da3    /* VDBE is ready to execute */
+#define VDBE_MAGIC_HALT     0x319c2973    /* VDBE has completed execution */
+#define VDBE_MAGIC_RESET    0x48fa9f76    /* Reset and ready to run again */
+#define VDBE_MAGIC_DEAD     0x5606c3c8    /* The VDBE has been deallocated */
+
+/*
+** Structure used to store the context required by the 
+** sqlite3_preupdate_*() API functions.
+*/
+struct PreUpdate {
+  Vdbe *v;
+  VdbeCursor *pCsr;               /* Cursor to read old values from */
+  int op;                         /* One of SQLITE_INSERT, UPDATE, DELETE */
+  u8 *aRecord;                    /* old.* database record */
+  KeyInfo keyinfo;
+  UnpackedRecord *pUnpacked;      /* Unpacked version of aRecord[] */
+  UnpackedRecord *pNewUnpacked;   /* Unpacked version of new.* record */
+  int iNewReg;                    /* Register for new.* values */
+  i64 iKey1;                      /* First key value passed to hook */
+  i64 iKey2;                      /* Second key value passed to hook */
+  Mem *aNew;                      /* Array of new.* values */
+  Table *pTab;                    /* Schema object being upated */          
+};
 
 /*
 ** Function prototypes
@@ -443,7 +450,7 @@ u8 sqlite3VdbeOneByteSerialTypeLen(u8);
 u32 sqlite3VdbeSerialType(Mem*, int, u32*);
 u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32);
 u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*);
-void sqlite3VdbeDeleteAuxData(Vdbe*, int, int);
+void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int);
 
 int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *);
 int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*);
@@ -486,6 +493,9 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int n);
 int sqlite3VdbeCloseStatement(Vdbe *, int);
 void sqlite3VdbeFrameDelete(VdbeFrame*);
 int sqlite3VdbeFrameRestore(VdbeFrame *);
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int);
+#endif
 int sqlite3VdbeTransferError(Vdbe *p);
 
 int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *);
@@ -535,4 +545,4 @@ int sqlite3VdbeMemHandleBom(Mem *pMem);
   #define ExpandBlob(P) SQLITE_OK
 #endif
 
-#endif /* !defined(_VDBEINT_H_) */
+#endif /* !defined(SQLITE_VDBEINT_H) */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index fce43630..645ef1c8 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -60,12 +60,19 @@ static int vdbeSafetyNotNull(Vdbe *p){
 */
 static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
   sqlite3_int64 iNow;
+  sqlite3_int64 iElapse;
   assert( p->startTime>0 );
-  assert( db->xProfile!=0 );
+  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
   assert( db->init.busy==0 );
   assert( p->zSql!=0 );
   sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
-  db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000);
+  iElapse = (iNow - p->startTime)*1000000;
+  if( db->xProfile ){
+    db->xProfile(db->pProfileArg, p->zSql, iElapse);
+  }
+  if( db->mTrace & SQLITE_TRACE_PROFILE ){
+    db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
+  }
   p->startTime = 0;
 }
 /*
@@ -162,7 +169,7 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){
 const void *sqlite3_value_blob(sqlite3_value *pVal){
   Mem *p = (Mem*)pVal;
   if( p->flags & (MEM_Blob|MEM_Str) ){
-    if( sqlite3VdbeMemExpandBlob(p)!=SQLITE_OK ){
+    if( ExpandBlob(p)!=SQLITE_OK ){
       assert( p->flags==MEM_Null && p->z==0 );
       return 0;
     }
@@ -471,7 +478,7 @@ void sqlite3_result_error_toobig(sqlite3_context *pCtx){
 void sqlite3_result_error_nomem(sqlite3_context *pCtx){
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
   sqlite3VdbeMemSetNull(pCtx->pOut);
-  pCtx->isError = SQLITE_NOMEM;
+  pCtx->isError = SQLITE_NOMEM_BKPT;
   pCtx->fErrorOrAux = 1;
   sqlite3OomFault(pCtx->pOut->db);
 }
@@ -492,7 +499,7 @@ static int doWalCallbacks(sqlite3 *db){
       nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt));
       sqlite3BtreeLeave(pBt);
       if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){
-        rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry);
+        rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry);
       }
     }
   }
@@ -547,7 +554,7 @@ static int sqlite3Step(Vdbe *p){
   db = p->db;
   if( db->mallocFailed ){
     p->rc = SQLITE_NOMEM;
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
 
   if( p->pc<=0 && p->expired ){
@@ -569,7 +576,8 @@ static int sqlite3Step(Vdbe *p){
     );
 
 #ifndef SQLITE_OMIT_TRACE
-    if( db->xProfile && !db->init.busy && p->zSql ){
+    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
+        && !db->init.busy && p->zSql ){
       sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
     }else{
       assert( p->startTime==0 );
@@ -610,7 +618,7 @@ static int sqlite3Step(Vdbe *p){
 
   db->errCode = rc;
   if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){
-    p->rc = SQLITE_NOMEM;
+    p->rc = SQLITE_NOMEM_BKPT;
   }
 end_of_step:
   /* At this point local variable rc holds the value that should be 
@@ -677,7 +685,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){
       v->rc = rc2;
     } else {
       v->zErrMsg = 0;
-      v->rc = rc = SQLITE_NOMEM;
+      v->rc = rc = SQLITE_NOMEM_BKPT;
     }
   }
   rc = sqlite3ApiExit(db, rc);
@@ -944,14 +952,13 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){
   Mem *pOut;
 
   pVm = (Vdbe *)pStmt;
-  if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){
-    sqlite3_mutex_enter(pVm->db->mutex);
+  if( pVm==0 ) return (Mem*)columnNullValue();
+  assert( pVm->db );
+  sqlite3_mutex_enter(pVm->db->mutex);
+  if( pVm->pResultSet!=0 && inResColumn && i>=0 ){
     pOut = &pVm->pResultSet[i];
   }else{
-    if( pVm && ALWAYS(pVm->db) ){
-      sqlite3_mutex_enter(pVm->db->mutex);
-      sqlite3Error(pVm->db, SQLITE_RANGE);
-    }
+    sqlite3Error(pVm->db, SQLITE_RANGE);
     pOut = (Mem*)columnNullValue();
   }
   return pOut;
@@ -984,6 +991,8 @@ static void columnMallocFailure(sqlite3_stmt *pStmt)
   */
   Vdbe *p = (Vdbe *)pStmt;
   if( p ){
+    assert( p->db!=0 );
+    assert( sqlite3_mutex_held(p->db->mutex) );
     p->rc = sqlite3ApiExit(p->db, p->rc);
     sqlite3_mutex_leave(p->db->mutex);
   }
@@ -1301,6 +1310,9 @@ int sqlite3_bind_blob(
   int nData, 
   void (*xDel)(void*)
 ){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( nData<0 ) return SQLITE_MISUSE_BKPT;
+#endif
   return bindText(pStmt, i, zData, nData, xDel, 0);
 }
 int sqlite3_bind_blob64(
@@ -1557,7 +1569,7 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){
 */
 int sqlite3_stmt_busy(sqlite3_stmt *pStmt){
   Vdbe *v = (Vdbe*)pStmt;
-  return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN;
+  return v!=0 && v->magic==VDBE_MAGIC_RUN && v->pc>=0;
 }
 
 /*
@@ -1601,6 +1613,224 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){
   return (int)v;
 }
 
+/*
+** Return the SQL associated with a prepared statement
+*/
+const char *sqlite3_sql(sqlite3_stmt *pStmt){
+  Vdbe *p = (Vdbe *)pStmt;
+  return p ? p->zSql : 0;
+}
+
+/*
+** Return the SQL associated with a prepared statement with
+** bound parameters expanded.  Space to hold the returned string is
+** obtained from sqlite3_malloc().  The caller is responsible for
+** freeing the returned string by passing it to sqlite3_free().
+**
+** The SQLITE_TRACE_SIZE_LIMIT puts an upper bound on the size of
+** expanded bound parameters.
+*/
+char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){
+#ifdef SQLITE_OMIT_TRACE
+  return 0;
+#else
+  char *z = 0;
+  const char *zSql = sqlite3_sql(pStmt);
+  if( zSql ){
+    Vdbe *p = (Vdbe *)pStmt;
+    sqlite3_mutex_enter(p->db->mutex);
+    z = sqlite3VdbeExpandSql(p, zSql);
+    sqlite3_mutex_leave(p->db->mutex);
+  }
+  return z;
+#endif
+}
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** Allocate and populate an UnpackedRecord structure based on the serialized
+** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure
+** if successful, or a NULL pointer if an OOM error is encountered.
+*/
+static UnpackedRecord *vdbeUnpackRecord(
+  KeyInfo *pKeyInfo, 
+  int nKey, 
+  const void *pKey
+){
+  char *dummy;                    /* Dummy argument for AllocUnpackedRecord() */
+  UnpackedRecord *pRet;           /* Return value */
+
+  pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo, 0, 0, &dummy);
+  if( pRet ){
+    memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nField+1));
+    sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet);
+  }
+  return pRet;
+}
+
+/*
+** This function is called from within a pre-update callback to retrieve
+** a field of the row currently being updated or deleted.
+*/
+int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
+  PreUpdate *p = db->pPreUpdate;
+  int rc = SQLITE_OK;
+
+  /* Test that this call is being made from within an SQLITE_DELETE or
+  ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */
+  if( !p || p->op==SQLITE_INSERT ){
+    rc = SQLITE_MISUSE_BKPT;
+    goto preupdate_old_out;
+  }
+  if( iIdx>=p->pCsr->nField || iIdx<0 ){
+    rc = SQLITE_RANGE;
+    goto preupdate_old_out;
+  }
+
+  /* If the old.* record has not yet been loaded into memory, do so now. */
+  if( p->pUnpacked==0 ){
+    u32 nRec;
+    u8 *aRec;
+
+    nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
+    aRec = sqlite3DbMallocRaw(db, nRec);
+    if( !aRec ) goto preupdate_old_out;
+    rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec);
+    if( rc==SQLITE_OK ){
+      p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec);
+      if( !p->pUnpacked ) rc = SQLITE_NOMEM;
+    }
+    if( rc!=SQLITE_OK ){
+      sqlite3DbFree(db, aRec);
+      goto preupdate_old_out;
+    }
+    p->aRecord = aRec;
+  }
+
+  if( iIdx>=p->pUnpacked->nField ){
+    *ppValue = (sqlite3_value *)columnNullValue();
+  }else{
+    Mem *pMem = *ppValue = &p->pUnpacked->aMem[iIdx];
+    *ppValue = &p->pUnpacked->aMem[iIdx];
+    if( iIdx==p->pTab->iPKey ){
+      sqlite3VdbeMemSetInt64(pMem, p->iKey1);
+    }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
+      if( pMem->flags & MEM_Int ){
+        sqlite3VdbeMemRealify(pMem);
+      }
+    }
+  }
+
+ preupdate_old_out:
+  sqlite3Error(db, rc);
+  return sqlite3ApiExit(db, rc);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** This function is called from within a pre-update callback to retrieve
+** the number of columns in the row being updated, deleted or inserted.
+*/
+int sqlite3_preupdate_count(sqlite3 *db){
+  PreUpdate *p = db->pPreUpdate;
+  return (p ? p->keyinfo.nField : 0);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** This function is designed to be called from within a pre-update callback
+** only. It returns zero if the change that caused the callback was made
+** immediately by a user SQL statement. Or, if the change was made by a
+** trigger program, it returns the number of trigger programs currently
+** on the stack (1 for a top-level trigger, 2 for a trigger fired by a 
+** top-level trigger etc.).
+**
+** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL
+** or SET DEFAULT action is considered a trigger.
+*/
+int sqlite3_preupdate_depth(sqlite3 *db){
+  PreUpdate *p = db->pPreUpdate;
+  return (p ? p->v->nFrame : 0);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** This function is called from within a pre-update callback to retrieve
+** a field of the row currently being updated or inserted.
+*/
+int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
+  PreUpdate *p = db->pPreUpdate;
+  int rc = SQLITE_OK;
+  Mem *pMem;
+
+  if( !p || p->op==SQLITE_DELETE ){
+    rc = SQLITE_MISUSE_BKPT;
+    goto preupdate_new_out;
+  }
+  if( iIdx>=p->pCsr->nField || iIdx<0 ){
+    rc = SQLITE_RANGE;
+    goto preupdate_new_out;
+  }
+
+  if( p->op==SQLITE_INSERT ){
+    /* For an INSERT, memory cell p->iNewReg contains the serialized record
+    ** that is being inserted. Deserialize it. */
+    UnpackedRecord *pUnpack = p->pNewUnpacked;
+    if( !pUnpack ){
+      Mem *pData = &p->v->aMem[p->iNewReg];
+      rc = ExpandBlob(pData);
+      if( rc!=SQLITE_OK ) goto preupdate_new_out;
+      pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z);
+      if( !pUnpack ){
+        rc = SQLITE_NOMEM;
+        goto preupdate_new_out;
+      }
+      p->pNewUnpacked = pUnpack;
+    }
+    if( iIdx>=pUnpack->nField ){
+      pMem = (sqlite3_value *)columnNullValue();
+    }else{
+      pMem = &pUnpack->aMem[iIdx];
+      if( iIdx==p->pTab->iPKey ){
+        sqlite3VdbeMemSetInt64(pMem, p->iKey2);
+      }
+    }
+  }else{
+    /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required
+    ** value. Make a copy of the cell contents and return a pointer to it.
+    ** It is not safe to return a pointer to the memory cell itself as the
+    ** caller may modify the value text encoding.
+    */
+    assert( p->op==SQLITE_UPDATE );
+    if( !p->aNew ){
+      p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField);
+      if( !p->aNew ){
+        rc = SQLITE_NOMEM;
+        goto preupdate_new_out;
+      }
+    }
+    assert( iIdx>=0 && iIdxpCsr->nField );
+    pMem = &p->aNew[iIdx];
+    if( pMem->flags==0 ){
+      if( iIdx==p->pTab->iPKey ){
+        sqlite3VdbeMemSetInt64(pMem, p->iKey2);
+      }else{
+        rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]);
+        if( rc!=SQLITE_OK ) goto preupdate_new_out;
+      }
+    }
+  }
+  *ppValue = pMem;
+
+ preupdate_new_out:
+  sqlite3Error(db, rc);
+  return sqlite3ApiExit(db, rc);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
 /*
 ** Return status data for a single loop within query pStmt.
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index ce98edd0..596ab30c 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -21,8 +21,9 @@
 Vdbe *sqlite3VdbeCreate(Parse *pParse){
   sqlite3 *db = pParse->db;
   Vdbe *p;
-  p = sqlite3DbMallocZero(db, sizeof(Vdbe) );
+  p = sqlite3DbMallocRawNN(db, sizeof(Vdbe) );
   if( p==0 ) return 0;
+  memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp));
   p->db = db;
   if( db->pVdbe ){
     db->pVdbe->pPrev = p;
@@ -64,20 +65,13 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){
   p->isPrepareV2 = (u8)isPrepareV2;
 }
 
-/*
-** Return the SQL associated with a prepared statement
-*/
-const char *sqlite3_sql(sqlite3_stmt *pStmt){
-  Vdbe *p = (Vdbe *)pStmt;
-  return p ? p->zSql : 0;
-}
-
 /*
 ** Swap all content between two VDBE structures.
 */
 void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
   Vdbe tmp, *pTmp;
   char *zTmp;
+  assert( pA->db==pB->db );
   tmp = *pA;
   *pA = *pB;
   *pB = tmp;
@@ -129,7 +123,7 @@ static int growOpArray(Vdbe *v, int nOp){
     p->nOpAlloc = p->szOpAlloc/sizeof(Op);
     v->aOp = pNew;
   }
-  return (pNew ? SQLITE_OK : SQLITE_NOMEM);
+  return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT);
 }
 
 #ifdef SQLITE_DEBUG
@@ -191,9 +185,8 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
   if( p->db->flags & SQLITE_VdbeAddopTrace ){
     int jj, kk;
     Parse *pParse = p->pParse;
-    for(jj=kk=0; jjnColCache; jj++){
       struct yColCache *x = pParse->aColCache + jj;
-      if( x->iLevel>pParse->iCacheLevel || x->iReg==0 ) continue;
       printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn);
       kk++;
     }
@@ -381,7 +374,6 @@ void sqlite3VdbeResolveLabel(Vdbe *v, int x){
   if( p->aLabel ){
     p->aLabel[j] = v->nOp;
   }
-  p->iFixedOp = v->nOp - 1;
 }
 
 /*
@@ -391,6 +383,13 @@ void sqlite3VdbeRunOnlyOnce(Vdbe *p){
   p->runOnlyOnce = 1;
 }
 
+/*
+** Mark the VDBE as one that can only be run multiple times.
+*/
+void sqlite3VdbeReusable(Vdbe *p){
+  p->runOnlyOnce = 0;
+}
+
 #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */
 
 /*
@@ -537,73 +536,84 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
 ** (4) Initialize the p4.xAdvance pointer on opcodes that use it.
 **
 ** (5) Reclaim the memory allocated for storing labels.
+**
+** This routine will only function correctly if the mkopcodeh.tcl generator
+** script numbers the opcodes correctly.  Changes to this routine must be
+** coordinated with changes to mkopcodeh.tcl.
 */
 static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
-  int i;
   int nMaxArgs = *pMaxFuncArgs;
   Op *pOp;
   Parse *pParse = p->pParse;
   int *aLabel = pParse->aLabel;
   p->readOnly = 1;
   p->bIsReader = 0;
-  for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
-    u8 opcode = pOp->opcode;
+  pOp = &p->aOp[p->nOp-1];
+  while(1){
 
-    /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing
-    ** cases from this switch! */
-    switch( opcode ){
-      case OP_Transaction: {
-        if( pOp->p2!=0 ) p->readOnly = 0;
-        /* fall thru */
-      }
-      case OP_AutoCommit:
-      case OP_Savepoint: {
-        p->bIsReader = 1;
-        break;
-      }
+    /* Only JUMP opcodes and the short list of special opcodes in the switch
+    ** below need to be considered.  The mkopcodeh.tcl generator script groups
+    ** all these opcodes together near the front of the opcode list.  Skip
+    ** any opcode that does not need processing by virtual of the fact that
+    ** it is larger than SQLITE_MX_JUMP_OPCODE, as a performance optimization.
+    */
+    if( pOp->opcode<=SQLITE_MX_JUMP_OPCODE ){
+      /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing
+      ** cases from this switch! */
+      switch( pOp->opcode ){
+        case OP_Transaction: {
+          if( pOp->p2!=0 ) p->readOnly = 0;
+          /* fall thru */
+        }
+        case OP_AutoCommit:
+        case OP_Savepoint: {
+          p->bIsReader = 1;
+          break;
+        }
 #ifndef SQLITE_OMIT_WAL
-      case OP_Checkpoint:
+        case OP_Checkpoint:
 #endif
-      case OP_Vacuum:
-      case OP_JournalMode: {
-        p->readOnly = 0;
-        p->bIsReader = 1;
-        break;
-      }
+        case OP_Vacuum:
+        case OP_JournalMode: {
+          p->readOnly = 0;
+          p->bIsReader = 1;
+          break;
+        }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-      case OP_VUpdate: {
-        if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
-        break;
-      }
-      case OP_VFilter: {
-        int n;
-        assert( p->nOp - i >= 3 );
-        assert( pOp[-1].opcode==OP_Integer );
-        n = pOp[-1].p1;
-        if( n>nMaxArgs ) nMaxArgs = n;
-        break;
-      }
+        case OP_VUpdate: {
+          if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
+          break;
+        }
+        case OP_VFilter: {
+          int n;
+          assert( (pOp - p->aOp) >= 3 );
+          assert( pOp[-1].opcode==OP_Integer );
+          n = pOp[-1].p1;
+          if( n>nMaxArgs ) nMaxArgs = n;
+          break;
+        }
 #endif
-      case OP_Next:
-      case OP_NextIfOpen:
-      case OP_SorterNext: {
-        pOp->p4.xAdvance = sqlite3BtreeNext;
-        pOp->p4type = P4_ADVANCE;
-        break;
+        case OP_Next:
+        case OP_NextIfOpen:
+        case OP_SorterNext: {
+          pOp->p4.xAdvance = sqlite3BtreeNext;
+          pOp->p4type = P4_ADVANCE;
+          break;
+        }
+        case OP_Prev:
+        case OP_PrevIfOpen: {
+          pOp->p4.xAdvance = sqlite3BtreePrevious;
+          pOp->p4type = P4_ADVANCE;
+          break;
+        }
       }
-      case OP_Prev:
-      case OP_PrevIfOpen: {
-        pOp->p4.xAdvance = sqlite3BtreePrevious;
-        pOp->p4type = P4_ADVANCE;
-        break;
+      if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 && pOp->p2<0 ){
+        assert( ADDR(pOp->p2)nLabel );
+        pOp->p2 = aLabel[ADDR(pOp->p2)];
       }
     }
-
-    pOp->opflags = sqlite3OpcodeProperty[opcode];
-    if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
-      assert( ADDR(pOp->p2)nLabel );
-      pOp->p2 = aLabel[ADDR(pOp->p2)];
-    }
+    if( pOp==p->aOp ) break;
+    pOp--;
   }
   sqlite3DbFree(p->db, pParse->aLabel);
   pParse->aLabel = 0;
@@ -754,7 +764,8 @@ void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){
   sqlite3VdbeGetOp(p,addr)->p3 = val;
 }
 void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
-  if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5;
+  assert( p->nOp>0 || p->db->mallocFailed );
+  if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5;
 }
 
 /*
@@ -762,7 +773,6 @@ void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
 ** the address of the next instruction to be coded.
 */
 void sqlite3VdbeJumpHere(Vdbe *p, int addr){
-  p->pParse->iFixedOp = p->nOp - 1;
   sqlite3VdbeChangeP2(p, addr, p->nOp);
 }
 
@@ -772,7 +782,7 @@ void sqlite3VdbeJumpHere(Vdbe *p, int addr){
 ** the FuncDef is not ephermal, then do nothing.
 */
 static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
-  if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
+  if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
     sqlite3DbFree(db, pDef);
   }
 }
@@ -782,53 +792,57 @@ static void vdbeFreeOpArray(sqlite3 *, Op *, int);
 /*
 ** Delete a P4 value if necessary.
 */
+static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){
+  if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
+  sqlite3DbFree(db, p);
+}
+static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){
+  freeEphemeralFunction(db, p->pFunc);
+  sqlite3DbFree(db, p);
+}
 static void freeP4(sqlite3 *db, int p4type, void *p4){
-  if( p4 ){
-    assert( db );
-    switch( p4type ){
-      case P4_FUNCCTX: {
-        freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc);
-        /* Fall through into the next case */
-      }
-      case P4_REAL:
-      case P4_INT64:
-      case P4_DYNAMIC:
-      case P4_INTARRAY: {
-        sqlite3DbFree(db, p4);
-        break;
-      }
-      case P4_KEYINFO: {
-        if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4);
-        break;
-      }
+  assert( db );
+  switch( p4type ){
+    case P4_FUNCCTX: {
+      freeP4FuncCtx(db, (sqlite3_context*)p4);
+      break;
+    }
+    case P4_REAL:
+    case P4_INT64:
+    case P4_DYNAMIC:
+    case P4_INTARRAY: {
+      sqlite3DbFree(db, p4);
+      break;
+    }
+    case P4_KEYINFO: {
+      if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4);
+      break;
+    }
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
-      case P4_EXPR: {
-        sqlite3ExprDelete(db, (Expr*)p4);
-        break;
-      }
+    case P4_EXPR: {
+      sqlite3ExprDelete(db, (Expr*)p4);
+      break;
+    }
 #endif
-      case P4_MPRINTF: {
-        if( db->pnBytesFreed==0 ) sqlite3_free(p4);
-        break;
-      }
-      case P4_FUNCDEF: {
-        freeEphemeralFunction(db, (FuncDef*)p4);
-        break;
-      }
-      case P4_MEM: {
-        if( db->pnBytesFreed==0 ){
-          sqlite3ValueFree((sqlite3_value*)p4);
-        }else{
-          Mem *p = (Mem*)p4;
-          if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
-          sqlite3DbFree(db, p);
-        }
-        break;
-      }
-      case P4_VTAB : {
-        if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
-        break;
+    case P4_MPRINTF: {
+      if( db->pnBytesFreed==0 ) sqlite3_free(p4);
+      break;
+    }
+    case P4_FUNCDEF: {
+      freeEphemeralFunction(db, (FuncDef*)p4);
+      break;
+    }
+    case P4_MEM: {
+      if( db->pnBytesFreed==0 ){
+        sqlite3ValueFree((sqlite3_value*)p4);
+      }else{
+        freeP4Mem(db, (Mem*)p4);
       }
+      break;
+    }
+    case P4_VTAB : {
+      if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
+      break;
     }
   }
 }
@@ -881,7 +895,7 @@ int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
 ** then remove it.  Return true if and only if an opcode was removed.
 */
 int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){
-  if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){
+  if( p->nOp>0 && p->aOp[p->nOp-1].opcode==op ){
     return sqlite3VdbeChangeToNoop(p, p->nOp-1);
   }else{
     return 0;
@@ -1079,12 +1093,21 @@ static int displayComment(
   const char *zSynopsis;
   int nOpName;
   int ii, jj;
+  char zAlt[50];
   zOpName = sqlite3OpcodeName(pOp->opcode);
   nOpName = sqlite3Strlen30(zOpName);
   if( zOpName[nOpName+1] ){
     int seenCom = 0;
     char c;
     zSynopsis = zOpName += nOpName + 1;
+    if( strncmp(zSynopsis,"IF ",3)==0 ){
+      if( pOp->p5 & SQLITE_STOREP2 ){
+        sqlite3_snprintf(sizeof(zAlt), zAlt, "r[P2] = (%s)", zSynopsis+3);
+      }else{
+        sqlite3_snprintf(sizeof(zAlt), zAlt, "if %s goto P2", zSynopsis+3);
+      }
+      zSynopsis = zAlt;
+    }
     for(ii=jj=0; jjp4.pTab->zName);
+      break;
+    }
     default: {
       zP4 = pOp->p4.z;
       if( zP4==0 ){
@@ -1431,6 +1458,21 @@ void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){
 }
 #endif
 
+/*
+** Initialize an array of N Mem element.
+*/
+static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){
+  while( (N--)>0 ){
+    p->db = db;
+    p->flags = flags;
+    p->szMalloc = 0;
+#ifdef SQLITE_DEBUG
+    p->pScopyFrom = 0;
+#endif
+    p++;
+  }
+}
+
 /*
 ** Release an array of N Mem elements
 */
@@ -1488,6 +1530,7 @@ void sqlite3VdbeFrameDelete(VdbeFrame *p){
     sqlite3VdbeFreeCursor(p->v, apCsr[i]);
   }
   releaseMemArray(aMem, p->nChildMem);
+  sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0);
   sqlite3DbFree(p->v->db, p);
 }
 
@@ -1530,7 +1573,7 @@ int sqlite3VdbeList(
   releaseMemArray(pMem, 8);
   p->pResultSet = 0;
 
-  if( p->rc==SQLITE_NOMEM ){
+  if( p->rc==SQLITE_NOMEM_BKPT ){
     /* This happens if a malloc() inside a call to sqlite3_column_text() or
     ** sqlite3_column_text16() failed.  */
     sqlite3OomFault(db);
@@ -1641,6 +1684,7 @@ int sqlite3VdbeList(
     pMem->flags = MEM_Str|MEM_Term;
     zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
     if( zP4!=pMem->z ){
+      pMem->n = 0;
       sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
     }else{
       assert( pMem->z!=0 );
@@ -1783,7 +1827,7 @@ void sqlite3VdbeRewind(Vdbe *p){
   int i;
 #endif
   assert( p!=0 );
-  assert( p->magic==VDBE_MAGIC_INIT );
+  assert( p->magic==VDBE_MAGIC_INIT || p->magic==VDBE_MAGIC_RESET );
 
   /* There should be at least one opcode.
   */
@@ -1793,7 +1837,7 @@ void sqlite3VdbeRewind(Vdbe *p){
   p->magic = VDBE_MAGIC_RUN;
 
 #ifdef SQLITE_DEBUG
-  for(i=1; inMem; i++){
+  for(i=0; inMem; i++){
     assert( p->aMem[i].db==p->db );
   }
 #endif
@@ -1840,7 +1884,6 @@ void sqlite3VdbeMakeReady(
   int nMem;                      /* Number of VM memory registers */
   int nCursor;                   /* Number of cursors required */
   int nArg;                      /* Number of arguments in subprograms */
-  int nOnce;                     /* Number of OP_Once instructions */
   int n;                         /* Loop counter */
   struct ReusableSpace x;        /* Reusable bulk memory */
 
@@ -1855,19 +1898,14 @@ void sqlite3VdbeMakeReady(
   nMem = pParse->nMem;
   nCursor = pParse->nTab;
   nArg = pParse->nMaxArg;
-  nOnce = pParse->nOnce;
-  if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */
   
-  /* For each cursor required, also allocate a memory cell. Memory
-  ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by
-  ** the vdbe program. Instead they are used to allocate memory for
-  ** VdbeCursor/BtCursor structures. The blob of memory associated with 
-  ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1)
-  ** stores the blob of memory associated with cursor 1, etc.
-  **
+  /* Each cursor uses a memory cell.  The first cursor (cursor 0) can
+  ** use aMem[0] which is not otherwise used by the VDBE program.  Allocate
+  ** space at the end of aMem[] for cursors 1 and greater.
   ** See also: allocateCursor().
   */
   nMem += nCursor;
+  if( nCursor==0 && nMem>0 ) nMem++;  /* Space for aMem[0] even if not used */
 
   /* Figure out how much reusable memory is available at the end of the
   ** opcode array.  This extra memory will be reallocated for other elements
@@ -1878,10 +1916,7 @@ void sqlite3VdbeMakeReady(
   assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) );
   x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n);  /* Bytes of unused memory */
   assert( x.nFree>=0 );
-  if( x.nFree>0 ){
-    memset(x.pSpace, 0, x.nFree);
-    assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) );
-  }
+  assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) );
 
   resolveP2Values(p, &nArg);
   p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
@@ -1906,37 +1941,34 @@ void sqlite3VdbeMakeReady(
     p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem));
     p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*));
     p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*));
-    p->aOnceFlag = allocSpace(&x, p->aOnceFlag, nOnce);
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
     p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64));
 #endif
     if( x.nNeeded==0 ) break;
-    x.pSpace = p->pFree = sqlite3DbMallocZero(db, x.nNeeded);
+    x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded);
     x.nFree = x.nNeeded;
   }while( !db->mallocFailed );
 
-  p->nCursor = nCursor;
-  p->nOnceFlag = nOnce;
-  if( p->aVar ){
-    p->nVar = (ynVar)nVar;
-    for(n=0; naVar[n].flags = MEM_Null;
-      p->aVar[n].db = db;
-    }
-  }
   p->nzVar = pParse->nzVar;
   p->azVar = pParse->azVar;
   pParse->nzVar =  0;
   pParse->azVar = 0;
-  if( p->aMem ){
-    p->aMem--;                      /* aMem[] goes from 1..nMem */
-    p->nMem = nMem;                 /*       not from 0..nMem-1 */
-    for(n=1; n<=nMem; n++){
-      p->aMem[n].flags = MEM_Undefined;
-      p->aMem[n].db = db;
-    }
-  }
   p->explain = pParse->explain;
+  if( db->mallocFailed ){
+    p->nVar = 0;
+    p->nCursor = 0;
+    p->nMem = 0;
+  }else{
+    p->nCursor = nCursor;
+    p->nVar = (ynVar)nVar;
+    initMemArray(p->aVar, nVar, db, MEM_Null);
+    p->nMem = nMem;
+    initMemArray(p->aMem, nMem, db, MEM_Undefined);
+    memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*));
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+    memset(p->anExec, 0, p->nOp*sizeof(i64));
+#endif
+  }
   sqlite3VdbeRewind(p);
 }
 
@@ -2005,8 +2037,6 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   v->anExec = pFrame->anExec;
 #endif
-  v->aOnceFlag = pFrame->aOnceFlag;
-  v->nOnceFlag = pFrame->nOnceFlag;
   v->aOp = pFrame->aOp;
   v->nOp = pFrame->nOp;
   v->aMem = pFrame->aMem;
@@ -2016,6 +2046,9 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){
   v->db->lastRowid = pFrame->lastRowid;
   v->nChange = pFrame->nChange;
   v->db->nChange = pFrame->nDbChange;
+  sqlite3VdbeDeleteAuxData(v->db, &v->pAuxData, -1, 0);
+  v->pAuxData = pFrame->pAuxData;
+  pFrame->pAuxData = 0;
   return pFrame->pc;
 }
 
@@ -2038,7 +2071,7 @@ static void closeAllCursors(Vdbe *p){
   assert( p->nFrame==0 );
   closeCursorsInFrame(p);
   if( p->aMem ){
-    releaseMemArray(&p->aMem[1], p->nMem);
+    releaseMemArray(p->aMem, p->nMem);
   }
   while( p->pDelFrame ){
     VdbeFrame *pDel = p->pDelFrame;
@@ -2047,7 +2080,7 @@ static void closeAllCursors(Vdbe *p){
   }
 
   /* Delete any auxdata allocations made by the VM */
-  if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0);
+  if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p->db, &p->pAuxData, -1, 0);
   assert( p->pAuxData==0 );
 }
 
@@ -2063,7 +2096,7 @@ static void Cleanup(Vdbe *p){
   int i;
   if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 );
   if( p->aMem ){
-    for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined );
+    for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined );
   }
 #endif
 
@@ -2087,13 +2120,9 @@ void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){
   sqlite3DbFree(db, p->aColName);
   n = nResColumn*COLNAME_N;
   p->nResColumn = (u16)nResColumn;
-  p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n );
+  p->aColName = pColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n );
   if( p->aColName==0 ) return;
-  while( n-- > 0 ){
-    pColName->flags = MEM_Null;
-    pColName->db = p->db;
-    pColName++;
-  }
+  initMemArray(p->aColName, n, p->db, MEM_Null);
 }
 
 /*
@@ -2119,7 +2148,7 @@ int sqlite3VdbeSetColName(
   assert( vardb->mallocFailed ){
     assert( !zName || xDel!=SQLITE_DYNAMIC );
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   assert( p->aColName!=0 );
   pColName = &(p->aColName[idx+var*p->nResColumn]);
@@ -2136,7 +2165,9 @@ int sqlite3VdbeSetColName(
 */
 static int vdbeCommit(sqlite3 *db, Vdbe *p){
   int i;
-  int nTrans = 0;  /* Number of databases with an active write-transaction */
+  int nTrans = 0;  /* Number of databases with an active write-transaction
+                   ** that are candidates for a two-phase commit using a
+                   ** master-journal */
   int rc = SQLITE_OK;
   int needXcommit = 0;
 
@@ -2164,10 +2195,28 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
   for(i=0; rc==SQLITE_OK && inDb; i++){ 
     Btree *pBt = db->aDb[i].pBt;
     if( sqlite3BtreeIsInTrans(pBt) ){
+      /* Whether or not a database might need a master journal depends upon
+      ** its journal mode (among other things).  This matrix determines which
+      ** journal modes use a master journal and which do not */
+      static const u8 aMJNeeded[] = {
+        /* DELETE   */  1,
+        /* PERSIST   */ 1,
+        /* OFF       */ 0,
+        /* TRUNCATE  */ 1,
+        /* MEMORY    */ 0,
+        /* WAL       */ 0
+      };
+      Pager *pPager;   /* Pager associated with pBt */
       needXcommit = 1;
-      if( i!=1 ) nTrans++;
       sqlite3BtreeEnter(pBt);
-      rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt));
+      pPager = sqlite3BtreePager(pBt);
+      if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF
+       && aMJNeeded[sqlite3PagerGetJournalMode(pPager)]
+      ){ 
+        assert( i!=1 );
+        nTrans++;
+      }
+      rc = sqlite3PagerExclusiveLock(pPager);
       sqlite3BtreeLeave(pBt);
     }
   }
@@ -2225,7 +2274,6 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
 #ifndef SQLITE_OMIT_DISKIO
   else{
     sqlite3_vfs *pVfs = db->pVfs;
-    int needSync = 0;
     char *zMaster = 0;   /* File-name for the master journal */
     char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt);
     sqlite3_file *pMaster = 0;
@@ -2237,7 +2285,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
     /* Select a master journal file name */
     nMainFile = sqlite3Strlen30(zMainFile);
     zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile);
-    if( zMaster==0 ) return SQLITE_NOMEM;
+    if( zMaster==0 ) return SQLITE_NOMEM_BKPT;
     do {
       u32 iRandom;
       if( retryCount ){
@@ -2285,9 +2333,6 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
           continue;  /* Ignore TEMP and :memory: databases */
         }
         assert( zFile[0]!=0 );
-        if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){
-          needSync = 1;
-        }
         rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset);
         offset += sqlite3Strlen30(zFile)+1;
         if( rc!=SQLITE_OK ){
@@ -2302,8 +2347,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
     /* Sync the master journal file. If the IOCAP_SEQUENTIAL device
     ** flag is set this is not required.
     */
-    if( needSync 
-     && 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL)
+    if( 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL)
      && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL))
     ){
       sqlite3OsCloseFree(pMaster);
@@ -2339,7 +2383,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){
     ** doing this the directory is synced again before any individual
     ** transaction files are deleted.
     */
-    rc = sqlite3OsDelete(pVfs, zMaster, needSync);
+    rc = sqlite3OsDelete(pVfs, zMaster, 1);
     sqlite3DbFree(db, zMaster);
     zMaster = 0;
     if( rc ){
@@ -2527,9 +2571,8 @@ int sqlite3VdbeHalt(Vdbe *p){
   */
 
   if( db->mallocFailed ){
-    p->rc = SQLITE_NOMEM;
+    p->rc = SQLITE_NOMEM_BKPT;
   }
-  if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag);
   closeAllCursors(p);
   if( p->magic!=VDBE_MAGIC_RUN ){
     return SQLITE_OK;
@@ -2688,7 +2731,7 @@ int sqlite3VdbeHalt(Vdbe *p){
   p->magic = VDBE_MAGIC_HALT;
   checkActiveVdbeCnt(db);
   if( db->mallocFailed ){
-    p->rc = SQLITE_NOMEM;
+    p->rc = SQLITE_NOMEM_BKPT;
   }
 
   /* If the auto-commit flag is set to true, then any locks that were held
@@ -2841,7 +2884,7 @@ int sqlite3VdbeReset(Vdbe *p){
   }
 #endif
   p->iCurrentTime = 0;
-  p->magic = VDBE_MAGIC_INIT;
+  p->magic = VDBE_MAGIC_RESET;
   return p->rc & db->errMask;
 }
  
@@ -2875,8 +2918,7 @@ int sqlite3VdbeFinalize(Vdbe *p){
 **    * the corresponding bit in argument mask is clear (where the first
 **      function parameter corresponds to bit 0 etc.).
 */
-void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){
-  AuxData **pp = &pVdbe->pAuxData;
+void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){
   while( *pp ){
     AuxData *pAux = *pp;
     if( (iOp<0)
@@ -2887,7 +2929,7 @@ void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){
         pAux->xDelete(pAux->pAux);
       }
       *pp = pAux->pNext;
-      sqlite3DbFree(pVdbe->db, pAux);
+      sqlite3DbFree(db, pAux);
     }else{
       pp= &pAux->pNext;
     }
@@ -2906,19 +2948,21 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
   SubProgram *pSub, *pNext;
   int i;
   assert( p->db==0 || p->db==db );
-  releaseMemArray(p->aVar, p->nVar);
   releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
   for(pSub=p->pProgram; pSub; pSub=pNext){
     pNext = pSub->pNext;
     vdbeFreeOpArray(db, pSub->aOp, pSub->nOp);
     sqlite3DbFree(db, pSub);
   }
-  for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]);
-  sqlite3DbFree(db, p->azVar);
+  if( p->magic!=VDBE_MAGIC_INIT ){
+    releaseMemArray(p->aVar, p->nVar);
+    for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]);
+    sqlite3DbFree(db, p->azVar);
+    sqlite3DbFree(db, p->pFree);
+  }
   vdbeFreeOpArray(db, p->aOp, p->nOp);
   sqlite3DbFree(db, p->aColName);
   sqlite3DbFree(db, p->zSql);
-  sqlite3DbFree(db, p->pFree);
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   for(i=0; inScan; i++){
     sqlite3DbFree(db, p->aScan[i].zName);
@@ -3486,6 +3530,7 @@ void sqlite3VdbeRecordUnpack(
     pMem->db = pKeyInfo->db;
     /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */
     pMem->szMalloc = 0;
+    pMem->z = 0;
     d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem);
     pMem++;
     if( (++u)>=p->nField ) break;
@@ -3666,22 +3711,56 @@ static int vdbeCompareMemString(
     v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc);
     n2 = v2==0 ? 0 : c2.n;
     rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2);
-    if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM;
+    if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM_BKPT;
     sqlite3VdbeMemRelease(&c1);
     sqlite3VdbeMemRelease(&c2);
     return rc;
   }
 }
 
+/*
+** The input pBlob is guaranteed to be a Blob that is not marked
+** with MEM_Zero.  Return true if it could be a zero-blob.
+*/
+static int isAllZero(const char *z, int n){
+  int i;
+  for(i=0; iz, pB2->z, pB1->n>pB2->n ? pB2->n : pB1->n);
+  int c;
+  int n1 = pB1->n;
+  int n2 = pB2->n;
+
+  /* It is possible to have a Blob value that has some non-zero content
+  ** followed by zero content.  But that only comes up for Blobs formed
+  ** by the OP_MakeRecord opcode, and such Blobs never get passed into
+  ** sqlite3MemCompare(). */
+  assert( (pB1->flags & MEM_Zero)==0 || n1==0 );
+  assert( (pB2->flags & MEM_Zero)==0 || n2==0 );
+
+  if( (pB1->flags|pB2->flags) & MEM_Zero ){
+    if( pB1->flags & pB2->flags & MEM_Zero ){
+      return pB1->u.nZero - pB2->u.nZero;
+    }else if( pB1->flags & MEM_Zero ){
+      if( !isAllZero(pB2->z, pB2->n) ) return -1;
+      return pB1->u.nZero - n2;
+    }else{
+      if( !isAllZero(pB1->z, pB1->n) ) return +1;
+      return n1 - pB2->u.nZero;
+    }
+  }
+  c = memcmp(pB1->z, pB2->z, n1>n2 ? n2 : n1);
   if( c ) return c;
-  return pB1->n - pB2->n;
+  return n1 - n2;
 }
 
 /*
@@ -3987,6 +4066,7 @@ int sqlite3VdbeRecordCompareWithSkip(
 
     /* RHS is a blob */
     else if( pRhs->flags & MEM_Blob ){
+      assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 );
       getVarint32(&aKey1[idx1], serial_type);
       testcase( serial_type==12 );
       if( serial_type<12 || (serial_type & 0x01) ){
@@ -3998,6 +4078,12 @@ int sqlite3VdbeRecordCompareWithSkip(
         if( (d1+nStr) > (unsigned)nKey1 ){
           pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT;
           return 0;                /* Corruption */
+        }else if( pRhs->flags & MEM_Zero ){
+          if( !isAllZero((const char*)&aKey1[d1],nStr) ){
+            rc = 1;
+          }else{
+            rc = nStr - pRhs->u.nZero;
+          }
         }else{
           int nCmp = MIN(nStr, pRhs->n);
           rc = memcmp(&aKey1[d1], pRhs->z, nCmp);
@@ -4068,7 +4154,7 @@ static int vdbeRecordCompareInt(
   int res;
   u32 y;
   u64 x;
-  i64 v = pPKey2->aMem[0].u.i;
+  i64 v;
   i64 lhs;
 
   vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo);
@@ -4127,6 +4213,7 @@ static int vdbeRecordCompareInt(
       return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2);
   }
 
+  v = pPKey2->aMem[0].u.i;
   if( v>lhs ){
     res = pPKey2->r1;
   }else if( veCurType==CURTYPE_BTREE );
   pCur = pC->uc.pCursor;
   assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );    /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
   /* nCellKey will always be between 0 and 0xffffffff because of the way
   ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */
   if( nCellKey<=0 || nCellKey>0x7fffffff ){
@@ -4465,3 +4550,87 @@ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){
   }
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+
+/*
+** If the second argument is not NULL, release any allocations associated 
+** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord
+** structure itself, using sqlite3DbFree().
+**
+** This function is used to free UnpackedRecord structures allocated by
+** the vdbeUnpackRecord() function found in vdbeapi.c.
+*/
+static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){
+  if( p ){
+    int i;
+    for(i=0; inField; i++){
+      Mem *pMem = &p->aMem[i];
+      if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem);
+    }
+    sqlite3DbFree(db, p);
+  }
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** Invoke the pre-update hook. If this is an UPDATE or DELETE pre-update call,
+** then cursor passed as the second argument should point to the row about
+** to be update or deleted. If the application calls sqlite3_preupdate_old(),
+** the required value will be read from the row the cursor points to.
+*/
+void sqlite3VdbePreUpdateHook(
+  Vdbe *v,                        /* Vdbe pre-update hook is invoked by */
+  VdbeCursor *pCsr,               /* Cursor to grab old.* values from */
+  int op,                         /* SQLITE_INSERT, UPDATE or DELETE */
+  const char *zDb,                /* Database name */
+  Table *pTab,                    /* Modified table */
+  i64 iKey1,                      /* Initial key value */
+  int iReg                        /* Register for new.* record */
+){
+  sqlite3 *db = v->db;
+  i64 iKey2;
+  PreUpdate preupdate;
+  const char *zTbl = pTab->zName;
+  static const u8 fakeSortOrder = 0;
+
+  assert( db->pPreUpdate==0 );
+  memset(&preupdate, 0, sizeof(PreUpdate));
+  if( op==SQLITE_UPDATE ){
+    iKey2 = v->aMem[iReg].u.i;
+  }else{
+    iKey2 = iKey1;
+  }
+
+  assert( pCsr->nField==pTab->nCol 
+       || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1)
+  );
+
+  preupdate.v = v;
+  preupdate.pCsr = pCsr;
+  preupdate.op = op;
+  preupdate.iNewReg = iReg;
+  preupdate.keyinfo.db = db;
+  preupdate.keyinfo.enc = ENC(db);
+  preupdate.keyinfo.nField = pTab->nCol;
+  preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder;
+  preupdate.iKey1 = iKey1;
+  preupdate.iKey2 = iKey2;
+  preupdate.pTab = pTab;
+
+  db->pPreUpdate = &preupdate;
+  db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2);
+  db->pPreUpdate = 0;
+  sqlite3DbFree(db, preupdate.aRecord);
+  vdbeFreeUnpacked(db, preupdate.pUnpacked);
+  vdbeFreeUnpacked(db, preupdate.pNewUnpacked);
+  if( preupdate.aNew ){
+    int i;
+    for(i=0; inField; i++){
+      sqlite3VdbeMemRelease(&preupdate.aNew[i]);
+    }
+    sqlite3DbFree(db, preupdate.aNew);
+  }
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
diff --git a/src/vdbeblob.c b/src/vdbeblob.c
index 2ba5301e..570e253f 100644
--- a/src/vdbeblob.c
+++ b/src/vdbeblob.c
@@ -30,6 +30,8 @@ struct Incrblob {
   BtCursor *pCsr;         /* Cursor pointing at blob row */
   sqlite3_stmt *pStmt;    /* Statement holding cursor open */
   sqlite3 *db;            /* The associated database */
+  char *zDb;              /* Database name */
+  Table *pTab;            /* Table object */
 };
 
 
@@ -173,6 +175,8 @@ int sqlite3_blob_open(
       sqlite3BtreeLeaveAll(db);
       goto blob_open_out;
     }
+    pBlob->pTab = pTab;
+    pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName;
 
     /* Now search pTab for the exact column. */
     for(iCol=0; iColnCol; iCol++) {
@@ -394,6 +398,30 @@ static int blobReadWrite(
     */
     assert( db == v->db );
     sqlite3BtreeEnterCursor(p->pCsr);
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+    if( xCall==sqlite3BtreePutData && db->xPreUpdateCallback ){
+      /* If a pre-update hook is registered and this is a write cursor, 
+      ** invoke it here. 
+      ** 
+      ** TODO: The preupdate-hook is passed SQLITE_DELETE, even though this
+      ** operation should really be an SQLITE_UPDATE. This is probably
+      ** incorrect, but is convenient because at this point the new.* values 
+      ** are not easily obtainable. And for the sessions module, an 
+      ** SQLITE_UPDATE where the PK columns do not change is handled in the 
+      ** same way as an SQLITE_DELETE (the SQLITE_DELETE code is actually
+      ** slightly more efficient). Since you cannot write to a PK column
+      ** using the incremental-blob API, this works. For the sessions module
+      ** anyhow.
+      */
+      sqlite3_int64 iKey;
+      iKey = sqlite3BtreeIntegerKey(p->pCsr);
+      sqlite3VdbePreUpdateHook(
+          v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1
+      );
+    }
+#endif
+
     rc = xCall(p->pCsr, iOffset+p->iOffset, n, z);
     sqlite3BtreeLeaveCursor(p->pCsr);
     if( rc==SQLITE_ABORT ){
diff --git a/src/vdbemem.c b/src/vdbemem.c
index 6fb7cebb..9b78d543 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -138,7 +138,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){
       sqlite3VdbeMemSetNull(pMem);
       pMem->z = 0;
       pMem->szMalloc = 0;
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }else{
       pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc);
     }
@@ -189,18 +189,18 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){
 ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails.
 */
 int sqlite3VdbeMemMakeWriteable(Mem *pMem){
-  int f;
   assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
   assert( (pMem->flags&MEM_RowSet)==0 );
-  ExpandBlob(pMem);
-  f = pMem->flags;
-  if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){
-    if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){
-      return SQLITE_NOMEM;
+  if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){
+    if( ExpandBlob(pMem) ) return SQLITE_NOMEM;
+    if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){
+      if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){
+        return SQLITE_NOMEM_BKPT;
+      }
+      pMem->z[pMem->n] = 0;
+      pMem->z[pMem->n+1] = 0;
+      pMem->flags |= MEM_Term;
     }
-    pMem->z[pMem->n] = 0;
-    pMem->z[pMem->n+1] = 0;
-    pMem->flags |= MEM_Term;
   }
   pMem->flags &= ~MEM_Ephem;
 #ifdef SQLITE_DEBUG
@@ -216,25 +216,24 @@ int sqlite3VdbeMemMakeWriteable(Mem *pMem){
 */
 #ifndef SQLITE_OMIT_INCRBLOB
 int sqlite3VdbeMemExpandBlob(Mem *pMem){
-  if( pMem->flags & MEM_Zero ){
-    int nByte;
-    assert( pMem->flags&MEM_Blob );
-    assert( (pMem->flags&MEM_RowSet)==0 );
-    assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
+  int nByte;
+  assert( pMem->flags & MEM_Zero );
+  assert( pMem->flags&MEM_Blob );
+  assert( (pMem->flags&MEM_RowSet)==0 );
+  assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
 
-    /* Set nByte to the number of bytes required to store the expanded blob. */
-    nByte = pMem->n + pMem->u.nZero;
-    if( nByte<=0 ){
-      nByte = 1;
-    }
-    if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){
-      return SQLITE_NOMEM;
-    }
-
-    memset(&pMem->z[pMem->n], 0, pMem->u.nZero);
-    pMem->n += pMem->u.nZero;
-    pMem->flags &= ~(MEM_Zero|MEM_Term);
+  /* Set nByte to the number of bytes required to store the expanded blob. */
+  nByte = pMem->n + pMem->u.nZero;
+  if( nByte<=0 ){
+    nByte = 1;
   }
+  if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){
+    return SQLITE_NOMEM_BKPT;
+  }
+
+  memset(&pMem->z[pMem->n], 0, pMem->u.nZero);
+  pMem->n += pMem->u.nZero;
+  pMem->flags &= ~(MEM_Zero|MEM_Term);
   return SQLITE_OK;
 }
 #endif
@@ -245,7 +244,7 @@ int sqlite3VdbeMemExpandBlob(Mem *pMem){
 */
 static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){
   if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   pMem->z[pMem->n] = 0;
   pMem->z[pMem->n+1] = 0;
@@ -294,7 +293,8 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){
 
 
   if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){
-    return SQLITE_NOMEM;
+    pMem->enc = 0;
+    return SQLITE_NOMEM_BKPT;
   }
 
   /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8
@@ -575,7 +575,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){
     }
   }
   assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 );
-  pMem->flags &= ~(MEM_Str|MEM_Blob);
+  pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero);
   return SQLITE_OK;
 }
 
@@ -593,7 +593,7 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){
       if( (pMem->flags & MEM_Blob)==0 ){
         sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding);
         assert( pMem->flags & MEM_Str || pMem->db->mallocFailed );
-        MemSetTypeFlag(pMem, MEM_Blob);
+        if( pMem->flags & MEM_Str ) MemSetTypeFlag(pMem, MEM_Blob);
       }else{
         pMem->flags &= ~(MEM_TypeMask&~MEM_Blob);
       }
@@ -761,7 +761,7 @@ int sqlite3VdbeMemTooBig(Mem *p){
 void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){
   int i;
   Mem *pX;
-  for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){
+  for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){
     if( pX->pScopyFrom==pMem ){
       pX->flags |= MEM_Undefined;
       pX->pScopyFrom = 0;
@@ -802,10 +802,6 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){
 int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){
   int rc = SQLITE_OK;
 
-  /* The pFrom==0 case in the following assert() is when an sqlite3_value
-  ** from sqlite3_value_dup() is used as the argument
-  ** to sqlite3_result_value(). */
-  assert( pTo->db==pFrom->db || pFrom->db==0 );
   assert( (pFrom->flags & MEM_RowSet)==0 );
   if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo);
   memcpy(pTo, pFrom, MEMCELLSIZE);
@@ -905,7 +901,7 @@ int sqlite3VdbeMemSetStr(
     testcase( nAlloc==31 );
     testcase( nAlloc==32 );
     if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }
     memcpy(pMem->z, z, nAlloc);
   }else if( xDel==SQLITE_DYNAMIC ){
@@ -925,7 +921,7 @@ int sqlite3VdbeMemSetStr(
 
 #ifndef SQLITE_OMIT_UTF16
   if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
 #endif
 
@@ -995,11 +991,7 @@ int sqlite3VdbeMemFromBtree(
   /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() 
   ** that both the BtShared and database handle mutexes are held. */
   assert( (pMem->flags & MEM_RowSet)==0 );
-  if( key ){
-    zData = (char *)sqlite3BtreeKeyFetch(pCur, &available);
-  }else{
-    zData = (char *)sqlite3BtreeDataFetch(pCur, &available);
-  }
+  zData = (char *)sqlite3BtreePayloadFetch(pCur, &available);
   assert( zData!=0 );
 
   if( offset+amt<=available ){
@@ -1026,9 +1018,6 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){
   assert( (pVal->flags & (MEM_Null))==0 );
   if( pVal->flags & (MEM_Blob|MEM_Str) ){
     pVal->flags |= MEM_Str;
-    if( pVal->flags & MEM_Zero ){
-      sqlite3VdbeMemExpandBlob(pVal);
-    }
     if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){
       sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED);
     }
@@ -1186,7 +1175,6 @@ static int valueFromFunction(
   FuncDef *pFunc = 0;             /* Function definition */
   sqlite3_value *pVal = 0;        /* New value */
   int rc = SQLITE_OK;             /* Return code */
-  int nName;                      /* Size of function name in bytes */
   ExprList *pList = 0;            /* Function arguments */
   int i;                          /* Iterator variable */
 
@@ -1194,8 +1182,7 @@ static int valueFromFunction(
   assert( (p->flags & EP_TokenOnly)==0 );
   pList = p->x.pList;
   if( pList ) nVal = pList->nExpr;
-  nName = sqlite3Strlen30(p->u.zToken);
-  pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0);
+  pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0);
   assert( pFunc );
   if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 
    || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
@@ -1206,7 +1193,7 @@ static int valueFromFunction(
   if( pList ){
     apVal = (sqlite3_value**)sqlite3DbMallocZero(db, sizeof(apVal[0]) * nVal);
     if( apVal==0 ){
-      rc = SQLITE_NOMEM;
+      rc = SQLITE_NOMEM_BKPT;
       goto value_from_function_out;
     }
     for(i=0; iop)==TK_UPLUS ) pExpr = pExpr->pLeft;
+  assert( pExpr!=0 );
+  while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft;
   if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;
 
   /* Compressed expressions only appear when parsing the DEFAULT clause
@@ -1390,7 +1374,7 @@ no_mem:
 #else
   assert( pCtx==0 ); sqlite3ValueFree(pVal);
 #endif
-  return SQLITE_NOMEM;
+  return SQLITE_NOMEM_BKPT;
 }
 
 /*
@@ -1410,7 +1394,7 @@ int sqlite3ValueFromExpr(
   u8 affinity,              /* Affinity to use */
   sqlite3_value **ppVal     /* Write the new value here */
 ){
-  return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0);
+  return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0;
 }
 
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
@@ -1457,15 +1441,10 @@ static void recordFunc(
 ** Register built-in functions used to help read ANALYZE data.
 */
 void sqlite3AnalyzeFunctions(void){
-  static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = {
+  static FuncDef aAnalyzeTableFuncs[] = {
     FUNCTION(sqlite_record,   1, 0, 0, recordFunc),
   };
-  int i;
-  FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions);
-  FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs);
-  for(i=0; iop!=TK_SELECT ){
+    int i;
+    struct ValueNewStat4Ctx alloc;
 
-  rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal);
-  assert( pVal==0 || pVal->db==pParse->db );
-  *pbOk = (pVal!=0);
+    alloc.pParse = pParse;
+    alloc.pIdx = pIdx;
+    alloc.ppRec = ppRec;
+
+    for(i=0; idb, pIdx, iVal+i);
+      alloc.iVal = iVal+i;
+      rc = stat4ValueFromExpr(pParse, pElem, aff, &alloc, &pVal);
+      if( !pVal ) break;
+      nExtract++;
+    }
+  }
+
+  *pnExtract = nExtract;
   return rc;
 }
 
@@ -1644,7 +1638,7 @@ int sqlite3Stat4Column(
   if( iField>nRec ) return SQLITE_CORRUPT_BKPT;
   if( pMem==0 ){
     pMem = *ppVal = sqlite3ValueNew(db);
-    if( pMem==0 ) return SQLITE_NOMEM;
+    if( pMem==0 ) return SQLITE_NOMEM_BKPT;
   }
   sqlite3VdbeSerialGet(&a[iField-szField], t, pMem);
   pMem->enc = ENC(db);
diff --git a/src/vdbesort.c b/src/vdbesort.c
index 380772ee..69619802 100644
--- a/src/vdbesort.c
+++ b/src/vdbesort.c
@@ -540,7 +540,7 @@ static int vdbePmaReadBlob(
       int nNew = MAX(128, p->nAlloc*2);
       while( nByte>nNew ) nNew = nNew*2;
       aNew = sqlite3Realloc(p->aAlloc, nNew);
-      if( !aNew ) return SQLITE_NOMEM;
+      if( !aNew ) return SQLITE_NOMEM_BKPT;
       p->nAlloc = nNew;
       p->aAlloc = aNew;
     }
@@ -652,7 +652,7 @@ static int vdbePmaReaderSeek(
     int iBuf = pReadr->iReadOff % pgsz;
     if( pReadr->aBuffer==0 ){
       pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz);
-      if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM;
+      if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM_BKPT;
       pReadr->nBuffer = pgsz;
     }
     if( rc==SQLITE_OK && iBuf ){
@@ -931,7 +931,6 @@ int sqlite3VdbeSorterInit(
 ){
   int pgsz;                       /* Page size of main database */
   int i;                          /* Used to iterate through aTask[] */
-  int mxCache;                    /* Cache size */
   VdbeSorter *pSorter;            /* The new sorter */
   KeyInfo *pKeyInfo;              /* Copy of pCsr->pKeyInfo with db==0 */
   int szKeyInfo;                  /* Size of pCsr->pKeyInfo in bytes */
@@ -968,7 +967,7 @@ int sqlite3VdbeSorterInit(
   pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo);
   pCsr->uc.pSorter = pSorter;
   if( pSorter==0 ){
-    rc = SQLITE_NOMEM;
+    rc = SQLITE_NOMEM_BKPT;
   }else{
     pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
     memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
@@ -988,11 +987,20 @@ int sqlite3VdbeSorterInit(
     }
 
     if( !sqlite3TempInMemory(db) ){
+      i64 mxCache;                /* Cache size in bytes*/
       u32 szPma = sqlite3GlobalConfig.szPma;
       pSorter->mnPmaSize = szPma * pgsz;
+
       mxCache = db->aDb[0].pSchema->cache_size;
-      if( mxCache<(int)szPma ) mxCache = (int)szPma;
-      pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_PMASZ);
+      if( mxCache<0 ){
+        /* A negative cache-size value C indicates that the cache is abs(C)
+        ** KiB in size.  */
+        mxCache = mxCache * -1024;
+      }else{
+        mxCache = mxCache * pgsz;
+      }
+      mxCache = MIN(mxCache, SQLITE_MAX_PMASZ);
+      pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache);
 
       /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of
       ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary
@@ -1002,7 +1010,7 @@ int sqlite3VdbeSorterInit(
         assert( pSorter->iMemory==0 );
         pSorter->nMemory = pgsz;
         pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz);
-        if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM;
+        if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM_BKPT;
       }
     }
 
@@ -1324,7 +1332,7 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){
         pTask->pSorter->pKeyInfo, 0, 0, &pFree
     );
     assert( pTask->pUnpacked==(UnpackedRecord*)pFree );
-    if( pFree==0 ) return SQLITE_NOMEM;
+    if( pFree==0 ) return SQLITE_NOMEM_BKPT;
     pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField;
     pTask->pUnpacked->errCode = 0;
   }
@@ -1334,19 +1342,18 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){
 
 /*
 ** Merge the two sorted lists p1 and p2 into a single list.
-** Set *ppOut to the head of the new list.
 */
-static void vdbeSorterMerge(
+static SorterRecord *vdbeSorterMerge(
   SortSubtask *pTask,             /* Calling thread context */
   SorterRecord *p1,               /* First list to merge */
-  SorterRecord *p2,               /* Second list to merge */
-  SorterRecord **ppOut            /* OUT: Head of merged list */
+  SorterRecord *p2                /* Second list to merge */
 ){
   SorterRecord *pFinal = 0;
   SorterRecord **pp = &pFinal;
   int bCached = 0;
 
-  while( p1 && p2 ){
+  assert( p1!=0 && p2!=0 );
+  for(;;){
     int res;
     res = pTask->xCompare(
         pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal
@@ -1356,15 +1363,22 @@ static void vdbeSorterMerge(
       *pp = p1;
       pp = &p1->u.pNext;
       p1 = p1->u.pNext;
+      if( p1==0 ){
+        *pp = p2;
+        break;
+      }
     }else{
       *pp = p2;
       pp = &p2->u.pNext;
       p2 = p2->u.pNext;
       bCached = 0;
+      if( p2==0 ){
+        *pp = p1;
+        break;
+      }
     }
   }
-  *pp = p1 ? p1 : p2;
-  *ppOut = pFinal;
+  return pFinal;
 }
 
 /*
@@ -1399,7 +1413,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
   aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
   if( !aSlot ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
 
   while( p ){
@@ -1417,7 +1431,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
     p->u.pNext = 0;
     for(i=0; aSlot[i]; i++){
-      vdbeSorterMerge(pTask, p, aSlot[i], &p);
+      p = vdbeSorterMerge(pTask, p, aSlot[i]);
       aSlot[i] = 0;
     }
     aSlot[i] = p;
@@ -1426,7 +1440,8 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
   p = 0;
   for(i=0; i<64; i++){
-    vdbeSorterMerge(pTask, p, aSlot[i], &p);
+    if( aSlot[i]==0 ) continue;
+    p = p ? vdbeSorterMerge(pTask, p, aSlot[i]) : aSlot[i];
   }
   pList->pList = p;
 
@@ -1449,7 +1464,7 @@ static void vdbePmaWriterInit(
   memset(p, 0, sizeof(PmaWriter));
   p->aBuffer = (u8*)sqlite3Malloc(nBuf);
   if( !p->aBuffer ){
-    p->eFWErr = SQLITE_NOMEM;
+    p->eFWErr = SQLITE_NOMEM_BKPT;
   }else{
     p->iBufEnd = p->iBufStart = (iStart % nBuf);
     p->iWriteOff = iStart - p->iBufStart;
@@ -1737,7 +1752,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){
         pSorter->nMemory = sqlite3MallocSize(aMem);
       }else if( pSorter->list.aMemory ){
         pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory);
-        if( !pSorter->list.aMemory ) return SQLITE_NOMEM;
+        if( !pSorter->list.aMemory ) return SQLITE_NOMEM_BKPT;
       }
 
       rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx);
@@ -1828,7 +1843,7 @@ int sqlite3VdbeSorterWrite(
       if( nNew < nMin ) nNew = nMin;
 
       aNew = sqlite3Realloc(pSorter->list.aMemory, nNew);
-      if( !aNew ) return SQLITE_NOMEM;
+      if( !aNew ) return SQLITE_NOMEM_BKPT;
       pSorter->list.pList = (SorterRecord*)&aNew[iListOff];
       pSorter->list.aMemory = aNew;
       pSorter->nMemory = nNew;
@@ -1842,7 +1857,7 @@ int sqlite3VdbeSorterWrite(
   }else{
     pNew = (SorterRecord *)sqlite3Malloc(nReq);
     if( pNew==0 ){
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }
     pNew->u.pNext = pSorter->list.pList;
   }
@@ -1989,7 +2004,7 @@ static int vdbeIncrMergerNew(
     pTask->file2.iEof += pIncr->mxSz;
   }else{
     vdbeMergeEngineFree(pMerger);
-    rc = SQLITE_NOMEM;
+    rc = SQLITE_NOMEM_BKPT;
   }
   return rc;
 }
@@ -2294,10 +2309,10 @@ static int vdbeMergeEngineLevel0(
   int rc = SQLITE_OK;
 
   *ppOut = pNew = vdbeMergeEngineNew(nPMA);
-  if( pNew==0 ) rc = SQLITE_NOMEM;
+  if( pNew==0 ) rc = SQLITE_NOMEM_BKPT;
 
   for(i=0; iaReadr[i];
     rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy);
     iOff = pReadr->iEof;
@@ -2365,7 +2380,7 @@ static int vdbeSorterAddToTree(
     if( pReadr->pIncr==0 ){
       MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT);
       if( pNew==0 ){
-        rc = SQLITE_NOMEM;
+        rc = SQLITE_NOMEM_BKPT;
       }else{
         rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr);
       }
@@ -2410,7 +2425,7 @@ static int vdbeSorterMergeTreeBuild(
   assert( pSorter->bUseThreads || pSorter->nTask==1 );
   if( pSorter->nTask>1 ){
     pMain = vdbeMergeEngineNew(pSorter->nTask);
-    if( pMain==0 ) rc = SQLITE_NOMEM;
+    if( pMain==0 ) rc = SQLITE_NOMEM_BKPT;
   }
 #endif
 
@@ -2428,7 +2443,7 @@ static int vdbeSorterMergeTreeBuild(
         int i;
         int iSeq = 0;
         pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT);
-        if( pRoot==0 ) rc = SQLITE_NOMEM;
+        if( pRoot==0 ) rc = SQLITE_NOMEM_BKPT;
         for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){
           MergeEngine *pMerger = 0; /* New level-0 PMA merger */
           int nReader;              /* Number of level-0 PMAs to merge */
@@ -2499,7 +2514,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
       if( rc==SQLITE_OK ){
         pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader));
         pSorter->pReader = pReadr;
-        if( pReadr==0 ) rc = SQLITE_NOMEM;
+        if( pReadr==0 ) rc = SQLITE_NOMEM_BKPT;
       }
       if( rc==SQLITE_OK ){
         rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr);
@@ -2676,7 +2691,7 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){
   pSorter = pCsr->uc.pSorter;
   pKey = vdbeSorterRowkey(pSorter, &nKey);
   if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   pOut->n = nKey;
   MemSetTypeFlag(pOut, MEM_Blob);
@@ -2721,7 +2736,7 @@ int sqlite3VdbeSorterCompare(
     char *p;
     r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p);
     assert( pSorter->pUnpacked==(UnpackedRecord*)p );
-    if( r2==0 ) return SQLITE_NOMEM;
+    if( r2==0 ) return SQLITE_NOMEM_BKPT;
     r2->nField = nKeyCol;
   }
   assert( r2->nField==nKeyCol );
diff --git a/src/vdbetrace.c b/src/vdbetrace.c
index 07235c93..7b473639 100644
--- a/src/vdbetrace.c
+++ b/src/vdbetrace.c
@@ -81,10 +81,13 @@ char *sqlite3VdbeExpandSql(
   int i;                   /* Loop counter */
   Mem *pVar;               /* Value of a host parameter */
   StrAccum out;            /* Accumulate the output here */
+#ifndef SQLITE_OMIT_UTF16
+  Mem utf8;                /* Used to convert UTF16 parameters into UTF8 for display */
+#endif
   char zBase[100];         /* Initial working space */
 
   db = p->db;
-  sqlite3StrAccumInit(&out, db, zBase, sizeof(zBase), 
+  sqlite3StrAccumInit(&out, 0, zBase, sizeof(zBase), 
                       db->aLimit[SQLITE_LIMIT_LENGTH]);
   if( db->nVdbeExec>1 ){
     while( *zRawSql ){
@@ -135,12 +138,14 @@ char *sqlite3VdbeExpandSql(
         int nOut;  /* Number of bytes of the string text to include in output */
 #ifndef SQLITE_OMIT_UTF16
         u8 enc = ENC(db);
-        Mem utf8;
         if( enc!=SQLITE_UTF8 ){
           memset(&utf8, 0, sizeof(utf8));
           utf8.db = db;
           sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
-          sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
+          if( SQLITE_NOMEM==sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8) ){
+            out.accError = STRACCUM_NOMEM;
+            out.nAlloc = 0;
+          }
           pVar = &utf8;
         }
 #endif
@@ -182,6 +187,7 @@ char *sqlite3VdbeExpandSql(
       }
     }
   }
+  if( out.accError ) sqlite3StrAccumReset(&out);
   return sqlite3StrAccumFinish(&out);
 }
 
diff --git a/src/vtab.c b/src/vtab.c
index e8794e68..c54dc657 100644
--- a/src/vtab.c
+++ b/src/vtab.c
@@ -344,7 +344,7 @@ void sqlite3VtabBeginParse(
   */
   if( pTable->azModuleArg ){
     sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, 
-            pTable->azModuleArg[0], pParse->db->aDb[iDb].zName);
+            pTable->azModuleArg[0], pParse->db->aDb[iDb].zDbSName);
   }
 #endif
 }
@@ -408,7 +408,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
       "UPDATE %Q.%s "
          "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q "
        "WHERE rowid=#%d",
-      db->aDb[iDb].zName, SCHEMA_TABLE(iDb),
+      db->aDb[iDb].zDbSName, SCHEMA_TABLE(iDb),
       pTab->zName,
       pTab->zName,
       zStmt,
@@ -418,7 +418,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
     v = sqlite3GetVdbe(pParse);
     sqlite3ChangeCookie(pParse, iDb);
 
-    sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
+    sqlite3VdbeAddOp0(v, OP_Expire);
     zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
     sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
 
@@ -506,19 +506,19 @@ static int vtabCallConstructor(
 
   zModuleName = sqlite3MPrintf(db, "%s", pTab->zName);
   if( !zModuleName ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
 
   pVTable = sqlite3DbMallocZero(db, sizeof(VTable));
   if( !pVTable ){
     sqlite3DbFree(db, zModuleName);
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   pVTable->db = db;
   pVTable->pMod = pMod;
 
   iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-  pTab->azModuleArg[1] = db->aDb[iDb].zName;
+  pTab->azModuleArg[1] = db->aDb[iDb].zDbSName;
 
   /* Invoke the virtual table constructor */
   assert( &db->pVtabCtx );
@@ -564,22 +564,16 @@ static int vtabCallConstructor(
       pTab->pVTable = pVTable;
 
       for(iCol=0; iColnCol; iCol++){
-        char *zType = pTab->aCol[iCol].zType;
+        char *zType = sqlite3ColumnType(&pTab->aCol[iCol], "");
         int nType;
         int i = 0;
-        if( !zType ){
-          pTab->tabFlags |= oooHidden;
-          continue;
-        }
         nType = sqlite3Strlen30(zType);
-        if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){
-          for(i=0; inVTrans + ARRAY_INCR);
     aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes);
     if( !aVTrans ){
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }
     memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR);
     db->aVTrans = aVTrans;
@@ -678,7 +672,7 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){
 ** This function is invoked by the vdbe to call the xCreate method
 ** of the virtual table named zTab in database iDb. 
 **
-** If an error occurs, *pzErr is set to point an an English language
+** If an error occurs, *pzErr is set to point to an English language
 ** description of the error and an SQLITE_XXX error code is returned.
 ** In this case the caller must call sqlite3DbFree(db, ) on *pzErr.
 */
@@ -688,7 +682,7 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){
   Module *pMod;
   const char *zMod;
 
-  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName);
+  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName);
   assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable );
 
   /* Locate the required virtual table module */
@@ -747,7 +741,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
 
   pParse = sqlite3StackAllocZero(db, sizeof(*pParse));
   if( pParse==0 ){
-    rc = SQLITE_NOMEM;
+    rc = SQLITE_NOMEM_BKPT;
   }else{
     pParse->declareVtab = 1;
     pParse->db = db;
@@ -760,10 +754,24 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
      && (pParse->pNewTable->tabFlags & TF_Virtual)==0
     ){
       if( !pTab->aCol ){
-        pTab->aCol = pParse->pNewTable->aCol;
-        pTab->nCol = pParse->pNewTable->nCol;
-        pParse->pNewTable->nCol = 0;
-        pParse->pNewTable->aCol = 0;
+        Table *pNew = pParse->pNewTable;
+        Index *pIdx;
+        pTab->aCol = pNew->aCol;
+        pTab->nCol = pNew->nCol;
+        pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
+        pNew->nCol = 0;
+        pNew->aCol = 0;
+        assert( pTab->pIndex==0 );
+        if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){
+          rc = SQLITE_ERROR;
+        }
+        pIdx = pNew->pIndex;
+        if( pIdx ){
+          assert( pIdx->pNext==0 );
+          pTab->pIndex = pIdx;
+          pNew->pIndex = 0;
+          pIdx->pTable = pTab;
+        }
       }
       pCtx->bDeclared = 1;
     }else{
@@ -798,8 +806,8 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){
   int rc = SQLITE_OK;
   Table *pTab;
 
-  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName);
-  if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){
+  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName);
+  if( pTab!=0 && ALWAYS(pTab->pVTable!=0) ){
     VTable *p;
     int (*xDestroy)(sqlite3_vtab *);
     for(p=pTab->pVTable; p; p=p->pNext){
@@ -939,7 +947,10 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){
       if( rc==SQLITE_OK ){
         int iSvpt = db->nStatement + db->nSavepoint;
         addToVTrans(db, pVTab);
-        if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1);
+        if( iSvpt && pModule->xSavepoint ){
+          pVTab->iSavepoint = iSvpt;
+          rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1);
+        }
       }
     }
   }
@@ -1059,8 +1070,8 @@ FuncDef *sqlite3VtabOverloadFunction(
     return pDef;
   }
   *pNew = *pDef;
-  pNew->zName = (char *)&pNew[1];
-  memcpy(pNew->zName, pDef->zName, sqlite3Strlen30(pDef->zName)+1);
+  pNew->zName = (const char*)&pNew[1];
+  memcpy((char*)&pNew[1], pDef->zName, sqlite3Strlen30(pDef->zName)+1);
   pNew->xSFunc = xSFunc;
   pNew->pUserData = pArg;
   pNew->funcFlags |= SQLITE_FUNC_EPHEM;
@@ -1093,7 +1104,7 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){
 }
 
 /*
-** Check to see if virtual tale module pMod can be have an eponymous
+** Check to see if virtual table module pMod can be have an eponymous
 ** virtual table instance.  If it can, create one if one does not already
 ** exist. Return non-zero if the eponymous virtual table instance exists
 ** when this routine returns, and return zero if it does not exist.
@@ -1110,17 +1121,18 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
   const sqlite3_module *pModule = pMod->pModule;
   Table *pTab;
   char *zErr = 0;
-  int nName;
   int rc;
   sqlite3 *db = pParse->db;
   if( pMod->pEpoTab ) return 1;
   if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0;
-  nName = sqlite3Strlen30(pMod->zName) + 1;
-  pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName);
+  pTab = sqlite3DbMallocZero(db, sizeof(Table));
   if( pTab==0 ) return 0;
+  pTab->zName = sqlite3DbStrDup(db, pMod->zName);
+  if( pTab->zName==0 ){
+    sqlite3DbFree(db, pTab);
+    return 0;
+  }
   pMod->pEpoTab = pTab;
-  pTab->zName = (char*)&pTab[1];
-  memcpy(pTab->zName, pMod->zName, nName);
   pTab->nRef = 1;
   pTab->pSchema = db->aDb[0].pSchema;
   pTab->tabFlags |= TF_Virtual;
@@ -1146,9 +1158,11 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
 void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
   Table *pTab = pMod->pEpoTab;
   if( pTab!=0 ){
-    sqlite3DeleteColumnNames(db, pTab);
-    sqlite3VtabClear(db, pTab);
-    sqlite3DbFree(db, pTab);
+    /* Mark the table as Ephemeral prior to deleting it, so that the
+    ** sqlite3DeleteTable() routine will know that it is not stored in 
+    ** the schema. */
+    pTab->tabFlags |= TF_Ephemeral;
+    sqlite3DeleteTable(db, pTab);
     pMod->pEpoTab = 0;
   }
 }
diff --git a/src/wal.c b/src/wal.c
index d7a6b701..235d383e 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -546,7 +546,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
     apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte);
     if( !apNew ){
       *ppPage = 0;
-      return SQLITE_NOMEM;
+      return SQLITE_NOMEM_BKPT;
     }
     memset((void*)&apNew[pWal->nWiData], 0,
            sizeof(u32*)*(iPage+1-pWal->nWiData));
@@ -558,7 +558,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){
   if( pWal->apWiData[iPage]==0 ){
     if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){
       pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ);
-      if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM;
+      if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT;
     }else{
       rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, 
           pWal->writeLock, (void volatile **)&pWal->apWiData[iPage]
@@ -1173,7 +1173,7 @@ static int walIndexRecover(Wal *pWal){
     szFrame = szPage + WAL_FRAME_HDRSIZE;
     aFrame = (u8 *)sqlite3_malloc64(szFrame);
     if( !aFrame ){
-      rc = SQLITE_NOMEM;
+      rc = SQLITE_NOMEM_BKPT;
       goto recovery_error;
     }
     aData = &aFrame[WAL_FRAME_HDRSIZE];
@@ -1311,7 +1311,7 @@ int sqlite3WalOpen(
   *ppWal = 0;
   pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile);
   if( !pRet ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
 
   pRet->pVfs = pVfs;
@@ -1575,7 +1575,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){
         + iLast*sizeof(ht_slot);
   p = (WalIterator *)sqlite3_malloc64(nByte);
   if( !p ){
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   memset(p, 0, nByte);
   p->nSegment = nSegment;
@@ -1587,7 +1587,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){
       sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast)
   );
   if( !aTmp ){
-    rc = SQLITE_NOMEM;
+    rc = SQLITE_NOMEM_BKPT;
   }
 
   for(i=0; rc==SQLITE_OK && ipData;
 #endif
@@ -2909,7 +2909,7 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){
   i64 iCksumOff;
 
   aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE);
-  if( aBuf==0 ) return SQLITE_NOMEM;
+  if( aBuf==0 ) return SQLITE_NOMEM_BKPT;
 
   /* Find the checksum values to use as input for the recalculating the
   ** first checksum. If the first frame is frame 1 (implying that the current
@@ -3109,16 +3109,21 @@ int sqlite3WalFrames(
   ** past the sector boundary is written after the sync.
   */
   if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){
+    int bSync = 1;
     if( pWal->padToSectorBoundary ){
       int sectorSize = sqlite3SectorSize(pWal->pWalFd);
       w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize;
+      bSync = (w.iSyncPoint==iOffset);
+      testcase( bSync );
       while( iOffsethdr, sizeof(WalIndexHdr));
     *ppSnapshot = (sqlite3_snapshot*)pRet;
@@ -3399,6 +3404,23 @@ int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){
 void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){
   pWal->pSnapshot = (WalIndexHdr*)pSnapshot;
 }
+
+/* 
+** Return a +ve value if snapshot p1 is newer than p2. A -ve value if
+** p1 is older than p2 and zero if p1 and p2 are the same snapshot.
+*/
+int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){
+  WalIndexHdr *pHdr1 = (WalIndexHdr*)p1;
+  WalIndexHdr *pHdr2 = (WalIndexHdr*)p2;
+
+  /* aSalt[0] is a copy of the value stored in the wal file header. It
+  ** is incremented each time the wal file is restarted.  */
+  if( pHdr1->aSalt[0]aSalt[0] ) return -1;
+  if( pHdr1->aSalt[0]>pHdr2->aSalt[0] ) return +1;
+  if( pHdr1->mxFramemxFrame ) return -1;
+  if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1;
+  return 0;
+}
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
 #ifdef SQLITE_ENABLE_ZIPVFS
diff --git a/src/wal.h b/src/wal.h
index 97e6ab4f..6048e798 100644
--- a/src/wal.h
+++ b/src/wal.h
@@ -14,8 +14,8 @@
 ** the implementation of each function in log.c for further details.
 */
 
-#ifndef _WAL_H_
-#define _WAL_H_
+#ifndef SQLITE_WAL_H
+#define SQLITE_WAL_H
 
 #include "sqliteInt.h"
 
@@ -143,4 +143,4 @@ int sqlite3WalFramesize(Wal *pWal);
 sqlite3_file *sqlite3WalFile(Wal *pWal);
 
 #endif /* ifndef SQLITE_OMIT_WAL */
-#endif /* _WAL_H_ */
+#endif /* SQLITE_WAL_H */
diff --git a/src/walker.c b/src/walker.c
index 1e0ad328..d1b1e96a 100644
--- a/src/walker.c
+++ b/src/walker.c
@@ -41,17 +41,17 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
   testcase( ExprHasProperty(pExpr, EP_TokenOnly) );
   testcase( ExprHasProperty(pExpr, EP_Reduced) );
   rc = pWalker->xExprCallback(pWalker, pExpr);
-  if( rc==WRC_Continue
-              && !ExprHasProperty(pExpr,EP_TokenOnly) ){
-    if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
-    if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort;
-    if( ExprHasProperty(pExpr, EP_xIsSelect) ){
-      if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
-    }else{
-      if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
-    }
+  if( rc || ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){
+    return rc & WRC_Abort;
   }
-  return rc & WRC_Abort;
+  if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
+  if( pExpr->pRight && walkExpr(pWalker, pExpr->pRight) ) return WRC_Abort;
+  if( ExprHasProperty(pExpr, EP_xIsSelect) ){
+    if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
+  }else if( pExpr->x.pList ){
+    if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
+  }
+  return WRC_Continue;
 }
 int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){
   return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue;
diff --git a/src/where.c b/src/where.c
index 8ecdd840..d77c333c 100644
--- a/src/where.c
+++ b/src/where.c
@@ -31,8 +31,8 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int);
 /*
 ** Return the estimated number of output rows from a WHERE clause
 */
-u64 sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
-  return sqlite3LogEstToInt(pWInfo->nRowOut);
+LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){
+  return pWInfo->nRowOut;
 }
 
 /*
@@ -51,6 +51,18 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
   return pWInfo->nOBSat;
 }
 
+/*
+** Return TRUE if the innermost loop of the WHERE clause implementation
+** returns rows in ORDER BY order for complete run of the inner loop.
+**
+** Across multiple iterations of outer loops, the output rows need not be
+** sorted.  As long as rows are sorted for just the innermost loop, this
+** routine can return TRUE.
+*/
+int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){
+  return pWInfo->bOrderedInnerLoop;
+}
+
 /*
 ** Return the VDBE address or label to jump to in order to continue
 ** immediately with the next row of a WHERE clause.
@@ -261,7 +273,10 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
 **
 ** The scanner will be searching the WHERE clause pWC.  It will look
 ** for terms of the form "X  " where X is column iColumn of table
-** iCur.  The  must be one of the operators described by opMask.
+** iCur.   Or if pIdx!=0 then X is column iColumn of index pIdx.  pIdx
+** must be one of the indexes of table iCur.
+**
+** The  must be one of the operators described by opMask.
 **
 ** If the search is for X and the WHERE clause contains terms of the
 ** form X=Y then this routine might also return terms of the form
@@ -289,6 +304,7 @@ static WhereTerm *whereScanInit(
     j = iColumn;
     iColumn = pIdx->aiColumn[j];
     if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
+    if( iColumn==pIdx->pTable->iPKey ) iColumn = XN_ROWID;
   }
   if( pIdx && iColumn>=0 ){
     pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
@@ -308,11 +324,12 @@ static WhereTerm *whereScanInit(
 
 /*
 ** Search for a term in the WHERE clause that is of the form "X  "
-** where X is a reference to the iColumn of table iCur and  is one of
-** the WO_xx operator codes specified by the op parameter.
-** Return a pointer to the term.  Return 0 if not found.
+** where X is a reference to the iColumn of table iCur or of index pIdx
+** if pIdx!=0 and  is one of the WO_xx operator codes specified by
+** the op parameter.  Return a pointer to the term.  Return 0 if not found.
 **
-** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx
+** If pIdx!=0 then it must be one of the indexes of table iCur.  
+** Search for terms matching the iColumn-th column of pIdx
 ** rather than the iColumn-th column of table iCur.
 **
 ** The term returned might by Y= if there is another constraint in
@@ -634,7 +651,7 @@ static void constructAutomaticIndex(
   ** transient index on 2nd and subsequent iterations of the loop. */
   v = pParse->pVdbe;
   assert( v!=0 );
-  addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v);
+  addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
 
   /* Count the number of columns that will be added to the index
   ** and used to match WHERE clause constraints */
@@ -809,7 +826,8 @@ static sqlite3_index_info *allocateIndexInfo(
   WhereClause *pWC,
   Bitmask mUnusable,              /* Ignore terms with these prereqs */
   struct SrcList_item *pSrc,
-  ExprList *pOrderBy
+  ExprList *pOrderBy,
+  u16 *pmNoOmit                   /* Mask of terms not to omit */
 ){
   int i, j;
   int nTerm;
@@ -819,6 +837,7 @@ static sqlite3_index_info *allocateIndexInfo(
   WhereTerm *pTerm;
   int nOrderBy;
   sqlite3_index_info *pIdxInfo;
+  u16 mNoOmit = 0;
 
   /* Count the number of possible WHERE clause constraints referring
   ** to this virtual table */
@@ -907,6 +926,15 @@ static sqlite3_index_info *allocateIndexInfo(
     assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
     assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH );
     assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
+
+    if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
+     && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
+    ){
+      if( i<16 ) mNoOmit |= (1 << i);
+      if( op==WO_LT ) pIdxCons[j].op = WO_LE;
+      if( op==WO_GT ) pIdxCons[j].op = WO_GE;
+    }
+
     j++;
   }
   for(i=0; ia[i].sortOrder;
   }
 
+  *pmNoOmit = mNoOmit;
   return pIdxInfo;
 }
 
@@ -934,7 +963,6 @@ static sqlite3_index_info *allocateIndexInfo(
 */
 static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
   sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab;
-  int i;
   int rc;
 
   TRACE_IDX_INPUTS(p);
@@ -953,12 +981,16 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
   sqlite3_free(pVtab->zErrMsg);
   pVtab->zErrMsg = 0;
 
+#if 0
+  /* This error is now caught by the caller.
+  ** Search for "xBestIndex malfunction" below */
   for(i=0; inConstraint; i++){
     if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){
       sqlite3ErrorMsg(pParse, 
           "table %s: xBestIndex returned an invalid plan", pTab->zName);
     }
   }
+#endif
 
   return pParse->nErr;
 }
@@ -1187,7 +1219,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
 /*
 ** Return the affinity for a single column of an index.
 */
-static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
+char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
   assert( iCol>=0 && iColnColumn );
   if( !pIdx->zColAff ){
     if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
@@ -1364,7 +1396,8 @@ static int whereRangeScanEst(
     if( nEq==pBuilder->nRecValid ){
       UnpackedRecord *pRec = pBuilder->pRec;
       tRowcnt a[2];
-      u8 aff;
+      int nBtm = pLoop->u.btree.nBtm;
+      int nTop = pLoop->u.btree.nTop;
 
       /* Variable iLower will be set to the estimate of the number of rows in 
       ** the index that are less than the lower bound of the range query. The
@@ -1394,8 +1427,6 @@ static int whereRangeScanEst(
         testcase( pRec->nField!=pBuilder->nRecValid );
         pRec->nField = pBuilder->nRecValid;
       }
-      aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq);
-      assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER );
       /* Determine iLower and iUpper using ($P) only. */
       if( nEq==0 ){
         iLower = 0;
@@ -1414,17 +1445,20 @@ static int whereRangeScanEst(
       if( p->aSortOrder[nEq] ){
         /* The roles of pLower and pUpper are swapped for a DESC index */
         SWAP(WhereTerm*, pLower, pUpper);
+        SWAP(int, nBtm, nTop);
       }
 
       /* If possible, improve on the iLower estimate using ($P:$L). */
       if( pLower ){
-        int bOk;                    /* True if value is extracted from pExpr */
+        int n;                    /* Values extracted from pExpr */
         Expr *pExpr = pLower->pExpr->pRight;
-        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
-        if( rc==SQLITE_OK && bOk ){
+        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
+        if( rc==SQLITE_OK && n ){
           tRowcnt iNew;
+          u16 mask = WO_GT|WO_LE;
+          if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
           iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
-          iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0);
+          iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
           if( iNew>iLower ) iLower = iNew;
           nOut--;
           pLower = 0;
@@ -1433,13 +1467,15 @@ static int whereRangeScanEst(
 
       /* If possible, improve on the iUpper estimate using ($P:$U). */
       if( pUpper ){
-        int bOk;                    /* True if value is extracted from pExpr */
+        int n;                    /* Values extracted from pExpr */
         Expr *pExpr = pUpper->pExpr->pRight;
-        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
-        if( rc==SQLITE_OK && bOk ){
+        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
+        if( rc==SQLITE_OK && n ){
           tRowcnt iNew;
+          u16 mask = WO_GT|WO_LE;
+          if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
           iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
-          iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0);
+          iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
           if( iNewpNew->u.btree.pIndex;
   int nEq = pBuilder->pNew->u.btree.nEq;
   UnpackedRecord *pRec = pBuilder->pRec;
-  u8 aff;                   /* Column affinity */
   int rc;                   /* Subfunction return code */
   tRowcnt a[2];             /* Statistics */
   int bOk;
@@ -1553,15 +1588,15 @@ static int whereEqualScanEst(
     return SQLITE_OK;
   }
 
-  aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1);
-  rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk);
+  rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
   pBuilder->pRec = pRec;
   if( rc!=SQLITE_OK ) return rc;
   if( bOk==0 ) return SQLITE_NOTFOUND;
   pBuilder->nRecValid = nEq;
 
   whereKeyStats(pParse, p, pRec, 0, a);
-  WHERETRACE(0x10,("equality scan regions: %d\n", (int)a[1]));
+  WHERETRACE(0x10,("equality scan regions %s(%d): %d\n",
+                   p->zName, nEq-1, (int)a[1]));
   *pnRow = a[1];
   
   return rc;
@@ -1627,30 +1662,58 @@ static void whereTermPrint(WhereTerm *pTerm, int iTerm){
     sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
   }else{
     char zType[4];
+    char zLeft[50];
     memcpy(zType, "...", 4);
     if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
     if( pTerm->eOperator & WO_EQUIV  ) zType[1] = 'E';
     if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L';
+    if( pTerm->eOperator & WO_SINGLE ){
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
+                       pTerm->leftCursor, pTerm->u.leftColumn);
+    }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld", 
+                       pTerm->u.pOrInfo->indexable);
+    }else{
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
+    }
     sqlite3DebugPrintf(
-       "TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x wtFlags=0x%04x\n",
-       iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb,
+       "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x",
+       iTerm, pTerm, zType, zLeft, pTerm->truthProb,
        pTerm->eOperator, pTerm->wtFlags);
+    if( pTerm->iField ){
+      sqlite3DebugPrintf(" iField=%d\n", pTerm->iField);
+    }else{
+      sqlite3DebugPrintf("\n");
+    }
     sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
   }
 }
 #endif
 
+#ifdef WHERETRACE_ENABLED
+/*
+** Show the complete content of a WhereClause
+*/
+void sqlite3WhereClausePrint(WhereClause *pWC){
+  int i;
+  for(i=0; inTerm; i++){
+    whereTermPrint(&pWC->a[i], i);
+  }
+}
+#endif
+
 #ifdef WHERETRACE_ENABLED
 /*
 ** Print a WhereLoop object for debugging purposes
 */
 static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){
   WhereInfo *pWInfo = pWC->pWInfo;
-  int nb = 1+(pWInfo->pTabList->nSrc+7)/8;
+  int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
   struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab;
   Table *pTab = pItem->pTab;
+  Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
   sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
-                     p->iTab, nb, p->maskSelf, nb, p->prereq);
+                     p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
   sqlite3DebugPrintf(" %12s",
                      pItem->zAlias ? pItem->zAlias : pTab->zName);
   if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
@@ -1736,7 +1799,7 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){
   if( p->nLSlot>=n ) return SQLITE_OK;
   n = (n+7)&~7;
   paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n);
-  if( paNew==0 ) return SQLITE_NOMEM;
+  if( paNew==0 ) return SQLITE_NOMEM_BKPT;
   memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot);
   if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm);
   p->aLTerm = paNew;
@@ -1751,7 +1814,7 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){
   whereLoopClearUnion(db, pTo);
   if( whereLoopResize(db, pTo, pFrom->nLTerm) ){
     memset(&pTo->u, 0, sizeof(pTo->u));
-    return SQLITE_NOMEM;
+    return SQLITE_NOMEM_BKPT;
   }
   memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ);
   memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0]));
@@ -1975,6 +2038,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
   WhereLoop **ppPrev, *p;
   WhereInfo *pWInfo = pBuilder->pWInfo;
   sqlite3 *db = pWInfo->pParse->db;
+  int rc;
 
   /* If pBuilder->pOrSet is defined, then only keep track of the costs
   ** and prereqs.
@@ -2033,7 +2097,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
   if( p==0 ){
     /* Allocate a new WhereLoop to add to the end of the list */
     *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop));
-    if( p==0 ) return SQLITE_NOMEM;
+    if( p==0 ) return SQLITE_NOMEM_BKPT;
     whereLoopInit(p);
     p->pNextLoop = 0;
   }else{
@@ -2057,14 +2121,14 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
       whereLoopDelete(db, pToDel);
     }
   }
-  whereLoopXfer(db, p, pTemplate);
+  rc = whereLoopXfer(db, p, pTemplate);
   if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
     Index *pIndex = p->u.btree.pIndex;
     if( pIndex && pIndex->tnum==0 ){
       p->u.btree.pIndex = 0;
     }
   }
-  return SQLITE_OK;
+  return rc;
 }
 
 /*
@@ -2142,6 +2206,72 @@ static void whereLoopOutputAdjust(
   if( pLoop->nOut > nRow-iReduce )  pLoop->nOut = nRow - iReduce;
 }
 
+/* 
+** Term pTerm is a vector range comparison operation. The first comparison
+** in the vector can be optimized using column nEq of the index. This
+** function returns the total number of vector elements that can be used
+** as part of the range comparison.
+**
+** For example, if the query is:
+**
+**   WHERE a = ? AND (b, c, d) > (?, ?, ?)
+**
+** and the index:
+**
+**   CREATE INDEX ... ON (a, b, c, d, e)
+**
+** then this function would be invoked with nEq=1. The value returned in
+** this case is 3.
+*/
+static int whereRangeVectorLen(
+  Parse *pParse,       /* Parsing context */
+  int iCur,            /* Cursor open on pIdx */
+  Index *pIdx,         /* The index to be used for a inequality constraint */
+  int nEq,             /* Number of prior equality constraints on same index */
+  WhereTerm *pTerm     /* The vector inequality constraint */
+){
+  int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
+  int i;
+
+  nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
+  for(i=1; ipExpr->pLeft->x.pList->a[i].pExpr;
+    Expr *pRhs = pTerm->pExpr->pRight;
+    if( pRhs->flags & EP_xIsSelect ){
+      pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
+    }else{
+      pRhs = pRhs->x.pList->a[i].pExpr;
+    }
+
+    /* Check that the LHS of the comparison is a column reference to
+    ** the right column of the right source table. And that the sort
+    ** order of the index column is the same as the sort order of the
+    ** leftmost index column.  */
+    if( pLhs->op!=TK_COLUMN 
+     || pLhs->iTable!=iCur 
+     || pLhs->iColumn!=pIdx->aiColumn[i+nEq] 
+     || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
+    ){
+      break;
+    }
+
+    testcase( pLhs->iColumn==XN_ROWID );
+    aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
+    idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
+    if( aff!=idxaff ) break;
+
+    pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
+    if( pColl==0 ) break;
+    if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
+  }
+  return i;
+}
+
 /*
 ** Adjust the cost C by the costMult facter T.  This only occurs if
 ** compiled with -DSQLITE_ENABLE_COSTMULT
@@ -2180,6 +2310,8 @@ static int whereLoopAddBtreeIndex(
   Bitmask saved_prereq;           /* Original value of pNew->prereq */
   u16 saved_nLTerm;               /* Original value of pNew->nLTerm */
   u16 saved_nEq;                  /* Original value of pNew->u.btree.nEq */
+  u16 saved_nBtm;                 /* Original value of pNew->u.btree.nBtm */
+  u16 saved_nTop;                 /* Original value of pNew->u.btree.nTop */
   u16 saved_nSkip;                /* Original value of pNew->nSkip */
   u32 saved_wsFlags;              /* Original value of pNew->wsFlags */
   LogEst saved_nOut;              /* Original value of pNew->nOut */
@@ -2189,15 +2321,16 @@ static int whereLoopAddBtreeIndex(
   WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */
 
   pNew = pBuilder->pNew;
-  if( db->mallocFailed ) return SQLITE_NOMEM;
+  if( db->mallocFailed ) return SQLITE_NOMEM_BKPT;
+  WHERETRACE(0x800, ("BEGIN addBtreeIdx(%s), nEq=%d\n",
+                     pProbe->zName, pNew->u.btree.nEq));
 
   assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
   assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
   if( pNew->wsFlags & WHERE_BTM_LIMIT ){
     opMask = WO_LT|WO_LE;
-  }else if( /*pProbe->tnum<=0 ||*/ (pSrc->fg.jointype & JT_LEFT)!=0 ){
-    opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE;
   }else{
+    assert( pNew->u.btree.nBtm==0 );
     opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
   }
   if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
@@ -2205,6 +2338,8 @@ static int whereLoopAddBtreeIndex(
   assert( pNew->u.btree.nEqnColumn );
 
   saved_nEq = pNew->u.btree.nEq;
+  saved_nBtm = pNew->u.btree.nBtm;
+  saved_nTop = pNew->u.btree.nTop;
   saved_nSkip = pNew->nSkip;
   saved_nLTerm = pNew->nLTerm;
   saved_wsFlags = pNew->wsFlags;
@@ -2234,8 +2369,22 @@ static int whereLoopAddBtreeIndex(
     ** to mix with a lower range bound from some other source */
     if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
 
+    /* Do not allow IS constraints from the WHERE clause to be used by the
+    ** right table of a LEFT JOIN.  Only constraints in the ON clause are
+    ** allowed */
+    if( (pSrc->fg.jointype & JT_LEFT)!=0
+     && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
+     && (eOp & (WO_IS|WO_ISNULL))!=0
+    ){
+      testcase( eOp & WO_IS );
+      testcase( eOp & WO_ISNULL );
+      continue;
+    }
+
     pNew->wsFlags = saved_wsFlags;
     pNew->u.btree.nEq = saved_nEq;
+    pNew->u.btree.nBtm = saved_nBtm;
+    pNew->u.btree.nTop = saved_nTop;
     pNew->nLTerm = saved_nLTerm;
     if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
     pNew->aLTerm[pNew->nLTerm++] = pTerm;
@@ -2252,14 +2401,23 @@ static int whereLoopAddBtreeIndex(
       pNew->wsFlags |= WHERE_COLUMN_IN;
       if( ExprHasProperty(pExpr, EP_xIsSelect) ){
         /* "x IN (SELECT ...)":  TUNING: the SELECT returns 25 rows */
+        int i;
         nIn = 46;  assert( 46==sqlite3LogEst(25) );
+
+        /* The expression may actually be of the form (x, y) IN (SELECT...).
+        ** In this case there is a separate term for each of (x) and (y).
+        ** However, the nIn multiplier should only be applied once, not once
+        ** for each such term. The following loop checks that pTerm is the
+        ** first such term in use, and sets nIn back to 0 if it is not. */
+        for(i=0; inLTerm-1; i++){
+          if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
+        }
       }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
         /* "x IN (value, value, ...)" */
         nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
+        assert( nIn>0 );  /* RHS always has 2 or more terms...  The parser
+                          ** changes "x IN (?)" into "x=?". */
       }
-      assert( nIn>0 );  /* RHS always has 2 or more terms...  The parser
-                        ** changes "x IN (?)" into "x=?". */
-
     }else if( eOp & (WO_EQ|WO_IS) ){
       int iCol = pProbe->aiColumn[saved_nEq];
       pNew->wsFlags |= WHERE_COLUMN_EQ;
@@ -2279,6 +2437,9 @@ static int whereLoopAddBtreeIndex(
       testcase( eOp & WO_GT );
       testcase( eOp & WO_GE );
       pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
+      pNew->u.btree.nBtm = whereRangeVectorLen(
+          pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
+      );
       pBtm = pTerm;
       pTop = 0;
       if( pTerm->wtFlags & TERM_LIKEOPT ){
@@ -2291,12 +2452,16 @@ static int whereLoopAddBtreeIndex(
         if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
         pNew->aLTerm[pNew->nLTerm++] = pTop;
         pNew->wsFlags |= WHERE_TOP_LIMIT;
+        pNew->u.btree.nTop = 1;
       }
     }else{
       assert( eOp & (WO_LT|WO_LE) );
       testcase( eOp & WO_LT );
       testcase( eOp & WO_LE );
       pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
+      pNew->u.btree.nTop = whereRangeVectorLen(
+          pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
+      );
       pTop = pTerm;
       pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
                      pNew->aLTerm[pNew->nLTerm-2] : 0;
@@ -2396,6 +2561,8 @@ static int whereLoopAddBtreeIndex(
   }
   pNew->prereq = saved_prereq;
   pNew->u.btree.nEq = saved_nEq;
+  pNew->u.btree.nBtm = saved_nBtm;
+  pNew->u.btree.nTop = saved_nTop;
   pNew->nSkip = saved_nSkip;
   pNew->wsFlags = saved_wsFlags;
   pNew->nOut = saved_nOut;
@@ -2435,6 +2602,8 @@ static int whereLoopAddBtreeIndex(
     pNew->wsFlags = saved_wsFlags;
   }
 
+  WHERETRACE(0x800, ("END addBtreeIdx(%s), nEq=%d, rc=%d\n",
+                      pProbe->zName, saved_nEq, rc));
   return rc;
 }
 
@@ -2517,7 +2686,7 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){
 
 /*
 ** Add all WhereLoop objects for a single table of the join where the table
-** is idenfied by pBuilder->pNew->iTab.  That table is guaranteed to be
+** is identified by pBuilder->pNew->iTab.  That table is guaranteed to be
 ** a b-tree table, not a virtual table.
 **
 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
@@ -2553,7 +2722,7 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){
 */
 static int whereLoopAddBtree(
   WhereLoopBuilder *pBuilder, /* WHERE clause information */
-  Bitmask mExtra              /* Extra prerequesites for using this table */
+  Bitmask mPrereq             /* Extra prerequesites for using this table */
 ){
   WhereInfo *pWInfo;          /* WHERE analysis context */
   Index *pProbe;              /* An index we are evaluating */
@@ -2614,7 +2783,7 @@ static int whereLoopAddBtree(
 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
   /* Automatic indexes */
   if( !pBuilder->pOrSet      /* Not part of an OR optimization */
-   && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0
+   && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
    && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
    && pSrc->pIBIndex==0      /* Has no INDEXED BY clause */
    && !pSrc->fg.notIndexed   /* Has no NOT INDEXED clause */
@@ -2646,6 +2815,7 @@ static int whereLoopAddBtree(
           pNew->rSetup += 24;
         }
         ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
+        if( pNew->rSetup<0 ) pNew->rSetup = 0;
         /* TUNING: Each index lookup yields 20 rows in the table.  This
         ** is more than the usual guess of 10 rows, since we have no way
         ** of knowing how selective the index will ultimately be.  It would
@@ -2653,7 +2823,7 @@ static int whereLoopAddBtree(
         pNew->nOut = 43;  assert( 43==sqlite3LogEst(20) );
         pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut);
         pNew->wsFlags = WHERE_AUTO_INDEX;
-        pNew->prereq = mExtra | pTerm->prereqRight;
+        pNew->prereq = mPrereq | pTerm->prereqRight;
         rc = whereLoopInsert(pBuilder, pNew);
       }
     }
@@ -2670,11 +2840,13 @@ static int whereLoopAddBtree(
     }
     rSize = pProbe->aiRowLogEst[0];
     pNew->u.btree.nEq = 0;
+    pNew->u.btree.nBtm = 0;
+    pNew->u.btree.nTop = 0;
     pNew->nSkip = 0;
     pNew->nLTerm = 0;
     pNew->iSortIdx = 0;
     pNew->rSetup = 0;
-    pNew->prereq = mExtra;
+    pNew->prereq = mPrereq;
     pNew->nOut = rSize;
     pNew->u.btree.pIndex = pProbe;
     b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);
@@ -2706,6 +2878,7 @@ static int whereLoopAddBtree(
       /* Full scan via index */
       if( b
        || !HasRowid(pTab)
+       || pProbe->pPartIdxWhere!=0
        || ( m==0
          && pProbe->bUnordered==0
          && (pProbe->szIdxRowszTabRow)
@@ -2718,11 +2891,34 @@ static int whereLoopAddBtree(
 
         /* The cost of visiting the index rows is N*K, where K is
         ** between 1.1 and 3.0, depending on the relative sizes of the
-        ** index and table rows. If this is a non-covering index scan,
-        ** also add the cost of visiting table rows (N*3.0).  */
+        ** index and table rows. */
         pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
         if( m!=0 ){
-          pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16);
+          /* If this is a non-covering index scan, add in the cost of
+          ** doing table lookups.  The cost will be 3x the number of
+          ** lookups.  Take into account WHERE clause terms that can be
+          ** satisfied using just the index, and that do not require a
+          ** table lookup. */
+          LogEst nLookup = rSize + 16;  /* Base cost:  N*3 */
+          int ii;
+          int iCur = pSrc->iCursor;
+          WhereClause *pWC2 = &pWInfo->sWC;
+          for(ii=0; iinTerm; ii++){
+            WhereTerm *pTerm = &pWC2->a[ii];
+            if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
+              break;
+            }
+            /* pTerm can be evaluated using just the index.  So reduce
+            ** the expected number of table lookups accordingly */
+            if( pTerm->truthProb<=0 ){
+              nLookup += pTerm->truthProb;
+            }else{
+              nLookup--;
+              if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
+            }
+          }
+          
+          pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
         }
         ApplyCostMultiplier(pNew->rRun, pTab->costMult);
         whereLoopOutputAdjust(pWC, pNew, rSize);
@@ -2747,12 +2943,162 @@ static int whereLoopAddBtree(
 }
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Argument pIdxInfo is already populated with all constraints that may
+** be used by the virtual table identified by pBuilder->pNew->iTab. This
+** function marks a subset of those constraints usable, invokes the
+** xBestIndex method and adds the returned plan to pBuilder.
+**
+** A constraint is marked usable if:
+**
+**   * Argument mUsable indicates that its prerequisites are available, and
+**
+**   * It is not one of the operators specified in the mExclude mask passed
+**     as the fourth argument (which in practice is either WO_IN or 0).
+**
+** Argument mPrereq is a mask of tables that must be scanned before the
+** virtual table in question. These are added to the plans prerequisites
+** before it is added to pBuilder.
+**
+** Output parameter *pbIn is set to true if the plan added to pBuilder
+** uses one or more WO_IN terms, or false otherwise.
+*/
+static int whereLoopAddVirtualOne(
+  WhereLoopBuilder *pBuilder,
+  Bitmask mPrereq,                /* Mask of tables that must be used. */
+  Bitmask mUsable,                /* Mask of usable tables */
+  u16 mExclude,                   /* Exclude terms using these operators */
+  sqlite3_index_info *pIdxInfo,   /* Populated object for xBestIndex */
+  u16 mNoOmit,                    /* Do not omit these constraints */
+  int *pbIn                       /* OUT: True if plan uses an IN(...) op */
+){
+  WhereClause *pWC = pBuilder->pWC;
+  struct sqlite3_index_constraint *pIdxCons;
+  struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage;
+  int i;
+  int mxTerm;
+  int rc = SQLITE_OK;
+  WhereLoop *pNew = pBuilder->pNew;
+  Parse *pParse = pBuilder->pWInfo->pParse;
+  struct SrcList_item *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab];
+  int nConstraint = pIdxInfo->nConstraint;
+
+  assert( (mUsable & mPrereq)==mPrereq );
+  *pbIn = 0;
+  pNew->prereq = mPrereq;
+
+  /* Set the usable flag on the subset of constraints identified by 
+  ** arguments mUsable and mExclude. */
+  pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
+  for(i=0; ia[pIdxCons->iTermOffset];
+    pIdxCons->usable = 0;
+    if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight 
+     && (pTerm->eOperator & mExclude)==0
+    ){
+      pIdxCons->usable = 1;
+    }
+  }
+
+  /* Initialize the output fields of the sqlite3_index_info structure */
+  memset(pUsage, 0, sizeof(pUsage[0])*nConstraint);
+  assert( pIdxInfo->needToFreeIdxStr==0 );
+  pIdxInfo->idxStr = 0;
+  pIdxInfo->idxNum = 0;
+  pIdxInfo->orderByConsumed = 0;
+  pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
+  pIdxInfo->estimatedRows = 25;
+  pIdxInfo->idxFlags = 0;
+  pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
+
+  /* Invoke the virtual table xBestIndex() method */
+  rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo);
+  if( rc ) return rc;
+
+  mxTerm = -1;
+  assert( pNew->nLSlot>=nConstraint );
+  for(i=0; iaLTerm[i] = 0;
+  pNew->u.vtab.omitMask = 0;
+  pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
+  for(i=0; i=0 ){
+      WhereTerm *pTerm;
+      int j = pIdxCons->iTermOffset;
+      if( iTerm>=nConstraint
+       || j<0
+       || j>=pWC->nTerm
+       || pNew->aLTerm[iTerm]!=0
+       || pIdxCons->usable==0
+      ){
+        rc = SQLITE_ERROR;
+        sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName);
+        return rc;
+      }
+      testcase( iTerm==nConstraint-1 );
+      testcase( j==0 );
+      testcase( j==pWC->nTerm-1 );
+      pTerm = &pWC->a[j];
+      pNew->prereq |= pTerm->prereqRight;
+      assert( iTermnLSlot );
+      pNew->aLTerm[iTerm] = pTerm;
+      if( iTerm>mxTerm ) mxTerm = iTerm;
+      testcase( iTerm==15 );
+      testcase( iTerm==16 );
+      if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){
+        /* A virtual table that is constrained by an IN clause may not
+        ** consume the ORDER BY clause because (1) the order of IN terms
+        ** is not necessarily related to the order of output terms and
+        ** (2) Multiple outputs from a single IN value will not merge
+        ** together.  */
+        pIdxInfo->orderByConsumed = 0;
+        pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
+        *pbIn = 1; assert( (mExclude & WO_IN)==0 );
+      }
+    }
+  }
+  pNew->u.vtab.omitMask &= ~mNoOmit;
+
+  pNew->nLTerm = mxTerm+1;
+  assert( pNew->nLTerm<=pNew->nLSlot );
+  pNew->u.vtab.idxNum = pIdxInfo->idxNum;
+  pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
+  pIdxInfo->needToFreeIdxStr = 0;
+  pNew->u.vtab.idxStr = pIdxInfo->idxStr;
+  pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
+      pIdxInfo->nOrderBy : 0);
+  pNew->rSetup = 0;
+  pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
+  pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
+
+  /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
+  ** that the scan will visit at most one row. Clear it otherwise. */
+  if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
+    pNew->wsFlags |= WHERE_ONEROW;
+  }else{
+    pNew->wsFlags &= ~WHERE_ONEROW;
+  }
+  rc = whereLoopInsert(pBuilder, pNew);
+  if( pNew->u.vtab.needFree ){
+    sqlite3_free(pNew->u.vtab.idxStr);
+    pNew->u.vtab.needFree = 0;
+  }
+  WHERETRACE(0xffff, ("  bIn=%d prereqIn=%04llx prereqOut=%04llx\n",
+                      *pbIn, (sqlite3_uint64)mPrereq,
+                      (sqlite3_uint64)(pNew->prereq & ~mPrereq)));
+
+  return rc;
+}
+
+
 /*
 ** Add all WhereLoop objects for a table of the join identified by
 ** pBuilder->pNew->iTab.  That table is guaranteed to be a virtual table.
 **
-** If there are no LEFT or CROSS JOIN joins in the query, both mExtra and
-** mUnusable are set to 0. Otherwise, mExtra is a mask of all FROM clause
+** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and
+** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause
 ** entries that occur before the virtual table in the FROM clause and are
 ** separated from it by at least one LEFT or CROSS JOIN. Similarly, the
 ** mUnusable mask contains all FROM clause entries that occur after the
@@ -2763,188 +3109,128 @@ static int whereLoopAddBtree(
 **
 **   ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6;
 **
-** then mExtra corresponds to (t1, t2) and mUnusable to (t5, t6).
+** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6).
 **
-** All the tables in mExtra must be scanned before the current virtual 
+** All the tables in mPrereq must be scanned before the current virtual 
 ** table. So any terms for which all prerequisites are satisfied by 
-** mExtra may be specified as "usable" in all calls to xBestIndex. 
+** mPrereq may be specified as "usable" in all calls to xBestIndex. 
 ** Conversely, all tables in mUnusable must be scanned after the current
 ** virtual table, so any terms for which the prerequisites overlap with
 ** mUnusable should always be configured as "not-usable" for xBestIndex.
 */
 static int whereLoopAddVirtual(
   WhereLoopBuilder *pBuilder,  /* WHERE clause information */
-  Bitmask mExtra,              /* Tables that must be scanned before this one */
+  Bitmask mPrereq,             /* Tables that must be scanned before this one */
   Bitmask mUnusable            /* Tables that must be scanned after this one */
 ){
+  int rc = SQLITE_OK;          /* Return code */
   WhereInfo *pWInfo;           /* WHERE analysis context */
   Parse *pParse;               /* The parsing context */
   WhereClause *pWC;            /* The WHERE clause */
   struct SrcList_item *pSrc;   /* The FROM clause term to search */
-  Table *pTab;
-  sqlite3 *db;
-  sqlite3_index_info *pIdxInfo;
-  struct sqlite3_index_constraint *pIdxCons;
-  struct sqlite3_index_constraint_usage *pUsage;
-  WhereTerm *pTerm;
-  int i, j;
-  int iTerm, mxTerm;
-  int nConstraint;
-  int seenIn = 0;              /* True if an IN operator is seen */
-  int seenVar = 0;             /* True if a non-constant constraint is seen */
-  int iPhase;                  /* 0: const w/o IN, 1: const, 2: no IN,  2: IN */
+  sqlite3_index_info *p;       /* Object to pass to xBestIndex() */
+  int nConstraint;             /* Number of constraints in p */
+  int bIn;                     /* True if plan uses IN(...) operator */
   WhereLoop *pNew;
-  int rc = SQLITE_OK;
+  Bitmask mBest;               /* Tables used by best possible plan */
+  u16 mNoOmit;
 
-  assert( (mExtra & mUnusable)==0 );
+  assert( (mPrereq & mUnusable)==0 );
   pWInfo = pBuilder->pWInfo;
   pParse = pWInfo->pParse;
-  db = pParse->db;
   pWC = pBuilder->pWC;
   pNew = pBuilder->pNew;
   pSrc = &pWInfo->pTabList->a[pNew->iTab];
-  pTab = pSrc->pTab;
-  assert( IsVirtual(pTab) );
-  pIdxInfo = allocateIndexInfo(pParse, pWC, mUnusable, pSrc,pBuilder->pOrderBy);
-  if( pIdxInfo==0 ) return SQLITE_NOMEM;
-  pNew->prereq = 0;
+  assert( IsVirtual(pSrc->pTab) );
+  p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, 
+      &mNoOmit);
+  if( p==0 ) return SQLITE_NOMEM_BKPT;
   pNew->rSetup = 0;
   pNew->wsFlags = WHERE_VIRTUALTABLE;
   pNew->nLTerm = 0;
   pNew->u.vtab.needFree = 0;
-  pUsage = pIdxInfo->aConstraintUsage;
-  nConstraint = pIdxInfo->nConstraint;
-  if( whereLoopResize(db, pNew, nConstraint) ){
-    sqlite3DbFree(db, pIdxInfo);
-    return SQLITE_NOMEM;
+  nConstraint = p->nConstraint;
+  if( whereLoopResize(pParse->db, pNew, nConstraint) ){
+    sqlite3DbFree(pParse->db, p);
+    return SQLITE_NOMEM_BKPT;
   }
 
-  for(iPhase=0; iPhase<=3; iPhase++){
-    if( !seenIn && (iPhase&1)!=0 ){
-      iPhase++;
-      if( iPhase>3 ) break;
-    }
-    if( !seenVar && iPhase>1 ) break;
-    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
-    for(i=0; inConstraint; i++, pIdxCons++){
-      j = pIdxCons->iTermOffset;
-      pTerm = &pWC->a[j];
-      switch( iPhase ){
-        case 0:    /* Constants without IN operator */
-          pIdxCons->usable = 0;
-          if( (pTerm->eOperator & WO_IN)!=0 ){
-            seenIn = 1;
-          }
-          if( (pTerm->prereqRight & ~mExtra)!=0 ){
-            seenVar = 1;
-          }else if( (pTerm->eOperator & WO_IN)==0 ){
-            pIdxCons->usable = 1;
-          }
-          break;
-        case 1:    /* Constants with IN operators */
-          assert( seenIn );
-          pIdxCons->usable = (pTerm->prereqRight & ~mExtra)==0;
-          break;
-        case 2:    /* Variables without IN */
-          assert( seenVar );
-          pIdxCons->usable = (pTerm->eOperator & WO_IN)==0;
-          break;
-        default:   /* Variables with IN */
-          assert( seenVar && seenIn );
-          pIdxCons->usable = 1;
-          break;
-      }
-    }
-    memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint);
-    if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr);
-    pIdxInfo->idxStr = 0;
-    pIdxInfo->idxNum = 0;
-    pIdxInfo->needToFreeIdxStr = 0;
-    pIdxInfo->orderByConsumed = 0;
-    pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2;
-    pIdxInfo->estimatedRows = 25;
-    pIdxInfo->idxFlags = 0;
-    pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed;
-    rc = vtabBestIndex(pParse, pTab, pIdxInfo);
-    if( rc ) goto whereLoopAddVtab_exit;
-    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
-    pNew->prereq = mExtra;
-    mxTerm = -1;
-    assert( pNew->nLSlot>=nConstraint );
-    for(i=0; iaLTerm[i] = 0;
-    pNew->u.vtab.omitMask = 0;
-    for(i=0; i=0 ){
-        j = pIdxCons->iTermOffset;
-        if( iTerm>=nConstraint
-         || j<0
-         || j>=pWC->nTerm
-         || pNew->aLTerm[iTerm]!=0
-        ){
-          rc = SQLITE_ERROR;
-          sqlite3ErrorMsg(pParse, "%s.xBestIndex() malfunction", pTab->zName);
-          goto whereLoopAddVtab_exit;
-        }
-        testcase( iTerm==nConstraint-1 );
-        testcase( j==0 );
-        testcase( j==pWC->nTerm-1 );
-        pTerm = &pWC->a[j];
-        pNew->prereq |= pTerm->prereqRight;
-        assert( iTermnLSlot );
-        pNew->aLTerm[iTerm] = pTerm;
-        if( iTerm>mxTerm ) mxTerm = iTerm;
-        testcase( iTerm==15 );
-        testcase( iTerm==16 );
-        if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){
-          if( pUsage[i].omit==0 ){
-            /* Do not attempt to use an IN constraint if the virtual table
-            ** says that the equivalent EQ constraint cannot be safely omitted.
-            ** If we do attempt to use such a constraint, some rows might be
-            ** repeated in the output. */
-            break;
-          }
-          /* A virtual table that is constrained by an IN clause may not
-          ** consume the ORDER BY clause because (1) the order of IN terms
-          ** is not necessarily related to the order of output terms and
-          ** (2) Multiple outputs from a single IN value will not merge
-          ** together.  */
-          pIdxInfo->orderByConsumed = 0;
-          pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE;
-        }
-      }
-    }
-    if( i>=nConstraint ){
-      pNew->nLTerm = mxTerm+1;
-      assert( pNew->nLTerm<=pNew->nLSlot );
-      pNew->u.vtab.idxNum = pIdxInfo->idxNum;
-      pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
-      pIdxInfo->needToFreeIdxStr = 0;
-      pNew->u.vtab.idxStr = pIdxInfo->idxStr;
-      pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
-                                      pIdxInfo->nOrderBy : 0);
-      pNew->rSetup = 0;
-      pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
-      pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
+  /* First call xBestIndex() with all constraints usable. */
+  WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
+  rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
 
-      /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated
-      ** that the scan will visit at most one row. Clear it otherwise. */
-      if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){
-        pNew->wsFlags |= WHERE_ONEROW;
-      }else{
-        pNew->wsFlags &= ~WHERE_ONEROW;
-      }
-      whereLoopInsert(pBuilder, pNew);
-      if( pNew->u.vtab.needFree ){
-        sqlite3_free(pNew->u.vtab.idxStr);
-        pNew->u.vtab.needFree = 0;
+  /* If the call to xBestIndex() with all terms enabled produced a plan
+  ** that does not require any source tables (IOW: a plan with mBest==0),
+  ** then there is no point in making any further calls to xBestIndex() 
+  ** since they will all return the same result (if the xBestIndex()
+  ** implementation is sane). */
+  if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){
+    int seenZero = 0;             /* True if a plan with no prereqs seen */
+    int seenZeroNoIN = 0;         /* Plan with no prereqs and no IN(...) seen */
+    Bitmask mPrev = 0;
+    Bitmask mBestNoIn = 0;
+
+    /* If the plan produced by the earlier call uses an IN(...) term, call
+    ** xBestIndex again, this time with IN(...) terms disabled. */
+    if( bIn ){
+      WHERETRACE(0x40, ("  VirtualOne: all usable w/o IN\n"));
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn);
+      assert( bIn==0 );
+      mBestNoIn = pNew->prereq & ~mPrereq;
+      if( mBestNoIn==0 ){
+        seenZero = 1;
+        seenZeroNoIN = 1;
       }
     }
-  }  
 
-whereLoopAddVtab_exit:
-  if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr);
-  sqlite3DbFree(db, pIdxInfo);
+    /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) 
+    ** in the set of terms that apply to the current virtual table.  */
+    while( rc==SQLITE_OK ){
+      int i;
+      Bitmask mNext = ALLBITS;
+      assert( mNext>0 );
+      for(i=0; ia[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq
+        );
+        if( mThis>mPrev && mThisprereq==mPrereq ){
+        seenZero = 1;
+        if( bIn==0 ) seenZeroNoIN = 1;
+      }
+    }
+
+    /* If the calls to xBestIndex() in the above loop did not find a plan
+    ** that requires no source tables at all (i.e. one guaranteed to be
+    ** usable), make a call here with all source tables disabled */
+    if( rc==SQLITE_OK && seenZero==0 ){
+      WHERETRACE(0x40, ("  VirtualOne: all disabled\n"));
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn);
+      if( bIn==0 ) seenZeroNoIN = 1;
+    }
+
+    /* If the calls to xBestIndex() have so far failed to find a plan
+    ** that requires no source tables at all and does not use an IN(...)
+    ** operator, make a final call to obtain one here.  */
+    if( rc==SQLITE_OK && seenZeroNoIN==0 ){
+      WHERETRACE(0x40, ("  VirtualOne: all disabled and w/o IN\n"));
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn);
+    }
+  }
+
+  if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr);
+  sqlite3DbFree(pParse->db, p);
   return rc;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -2955,7 +3241,7 @@ whereLoopAddVtab_exit:
 */
 static int whereLoopAddOr(
   WhereLoopBuilder *pBuilder, 
-  Bitmask mExtra, 
+  Bitmask mPrereq, 
   Bitmask mUnusable
 ){
   WhereInfo *pWInfo = pBuilder->pWInfo;
@@ -3009,21 +3295,19 @@ static int whereLoopAddOr(
         WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", 
                    (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
         if( sqlite3WhereTrace & 0x400 ){
-          for(i=0; inTerm; i++){
-            whereTermPrint(&sSubBuild.pWC->a[i], i);
-          }
+          sqlite3WhereClausePrint(sSubBuild.pWC);
         }
 #endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
         if( IsVirtual(pItem->pTab) ){
-          rc = whereLoopAddVirtual(&sSubBuild, mExtra, mUnusable);
+          rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable);
         }else
 #endif
         {
-          rc = whereLoopAddBtree(&sSubBuild, mExtra);
+          rc = whereLoopAddBtree(&sSubBuild, mPrereq);
         }
         if( rc==SQLITE_OK ){
-          rc = whereLoopAddOr(&sSubBuild, mExtra, mUnusable);
+          rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable);
         }
         assert( rc==SQLITE_OK || sCur.n==0 );
         if( sCur.n==0 ){
@@ -3080,7 +3364,7 @@ static int whereLoopAddOr(
 */
 static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
   WhereInfo *pWInfo = pBuilder->pWInfo;
-  Bitmask mExtra = 0;
+  Bitmask mPrereq = 0;
   Bitmask mPrior = 0;
   int iTab;
   SrcList *pTabList = pWInfo->pTabList;
@@ -3101,9 +3385,10 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
     if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){
       /* This condition is true when pItem is the FROM clause term on the
       ** right-hand-side of a LEFT or CROSS JOIN.  */
-      mExtra = mPrior;
+      mPrereq = mPrior;
     }
     priorJointype = pItem->fg.jointype;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
     if( IsVirtual(pItem->pTab) ){
       struct SrcList_item *p;
       for(p=&pItem[1]; psMaskSet, p->iCursor);
         }
       }
-      rc = whereLoopAddVirtual(pBuilder, mExtra, mUnusable);
-    }else{
-      rc = whereLoopAddBtree(pBuilder, mExtra);
+      rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable);
+    }else
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+    {
+      rc = whereLoopAddBtree(pBuilder, mPrereq);
     }
     if( rc==SQLITE_OK ){
-      rc = whereLoopAddOr(pBuilder, mExtra, mUnusable);
+      rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable);
     }
     mPrior |= pNew->maskSelf;
     if( rc || db->mallocFailed ) break;
@@ -3147,7 +3434,7 @@ static i8 wherePathSatisfiesOrderBy(
   WhereInfo *pWInfo,    /* The WHERE clause */
   ExprList *pOrderBy,   /* ORDER BY or GROUP BY or DISTINCT clause to check */
   WherePath *pPath,     /* The WherePath to check */
-  u16 wctrlFlags,       /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */
+  u16 wctrlFlags,       /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */
   u16 nLoop,            /* Number of entries in pPath->aLoop[] */
   WhereLoop *pLast,     /* Add this WhereLoop to the end of pPath->aLoop[] */
   Bitmask *pRevMask     /* OUT: Mask of WhereLoops to run in reverse order */
@@ -3158,6 +3445,7 @@ static i8 wherePathSatisfiesOrderBy(
   u8 isOrderDistinct;   /* All prior WhereLoops are order-distinct */
   u8 distinctColumns;   /* True if the loop has UNIQUE NOT NULL columns */
   u8 isMatch;           /* iColumn matches a term of the ORDER BY clause */
+  u16 eqOpMask;         /* Allowed equality operators */
   u16 nKeyCol;          /* Number of key columns in pIndex */
   u16 nColumn;          /* Total number of ordered columns in the index */
   u16 nOrderBy;         /* Number terms in the ORDER BY clause */
@@ -3208,9 +3496,16 @@ static i8 wherePathSatisfiesOrderBy(
   obDone = MASKBIT(nOrderBy)-1;
   orderDistinctMask = 0;
   ready = 0;
+  eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
+  if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN;
   for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf;
-    pLoop = iLoopaLoop[iLoop] : pLast;
+    if( iLoopaLoop[iLoop];
+      if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
+    }else{
+      pLoop = pLast;
+    }
     if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
       if( pLoop->u.vtab.isOrdered ) obSat = obDone;
       break;
@@ -3228,8 +3523,16 @@ static i8 wherePathSatisfiesOrderBy(
       if( pOBExpr->op!=TK_COLUMN ) continue;
       if( pOBExpr->iTable!=iCur ) continue;
       pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
-                       ~ready, WO_EQ|WO_ISNULL|WO_IS, 0);
+                       ~ready, eqOpMask, 0);
       if( pTerm==0 ) continue;
+      if( pTerm->eOperator==WO_IN ){
+        /* IN terms are only valid for sorting in the ORDER BY LIMIT 
+        ** optimization, and then only if they are actually used
+        ** by the query plan */
+        assert( wctrlFlags & WHERE_ORDERBY_LIMIT );
+        for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
+        if( j>=pLoop->nLTerm ) continue;
+      }
       if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
         const char *z1, *z2;
         pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
@@ -3266,18 +3569,42 @@ static i8 wherePathSatisfiesOrderBy(
       rev = revSet = 0;
       distinctColumns = 0;
       for(j=0; ju.btree.nEq
-         && pLoop->nSkip==0
-         && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0
-        ){
-          if( i & WO_ISNULL ){
-            testcase( isOrderDistinct );
-            isOrderDistinct = 0;
+        assert( j>=pLoop->u.btree.nEq 
+            || (pLoop->aLTerm[j]==0)==(jnSkip)
+        );
+        if( ju.btree.nEq && j>=pLoop->nSkip ){
+          u16 eOp = pLoop->aLTerm[j]->eOperator;
+
+          /* Skip over == and IS and ISNULL terms.  (Also skip IN terms when
+          ** doing WHERE_ORDERBY_LIMIT processing). 
+          **
+          ** If the current term is a column of an ((?,?) IN (SELECT...)) 
+          ** expression for which the SELECT returns more than one column,
+          ** check that it is the only column used by this loop. Otherwise,
+          ** if it is one of two or more, none of the columns can be
+          ** considered to match an ORDER BY term.  */
+          if( (eOp & eqOpMask)!=0 ){
+            if( eOp & WO_ISNULL ){
+              testcase( isOrderDistinct );
+              isOrderDistinct = 0;
+            }
+            continue;  
+          }else if( ALWAYS(eOp & WO_IN) ){
+            /* ALWAYS() justification: eOp is an equality operator due to the
+            ** ju.btree.nEq constraint above.  Any equality other
+            ** than WO_IN is captured by the previous "if".  So this one
+            ** always has to be WO_IN. */
+            Expr *pX = pLoop->aLTerm[j]->pExpr;
+            for(i=j+1; iu.btree.nEq; i++){
+              if( pLoop->aLTerm[i]->pExpr==pX ){
+                assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
+                bOnce = 0;
+                break;
+              }
+            }
           }
-          continue;  
         }
 
         /* Get the column number in the table (iColumn) and sort order
@@ -3306,7 +3633,6 @@ static i8 wherePathSatisfiesOrderBy(
         /* Find the ORDER BY term that corresponds to the j-th column
         ** of the index and mark that ORDER BY term off 
         */
-        bOnce = 1;
         isMatch = 0;
         for(i=0; bOnce && i0 && 66==sqlite3LogEst(100) );
   rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66;
-  rSortCost = nRow + estLog(nRow) + rScale + 16;
+  rSortCost = nRow + rScale + 16;
+
+  /* Multiple by log(M) where M is the number of output rows.
+  ** Use the LIMIT for M if it is smaller */
+  if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit;
+  }
+  rSortCost += estLog(nRow);
   return rSortCost;
 }
 
@@ -3521,7 +3855,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
   nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2;
   nSpace += sizeof(LogEst) * nOrderBy;
   pSpace = sqlite3DbMallocRawNN(db, nSpace);
-  if( pSpace==0 ) return SQLITE_NOMEM;
+  if( pSpace==0 ) return SQLITE_NOMEM_BKPT;
   aTo = (WherePath*)pSpace;
   aFrom = aTo+mxChoice;
   memset(aFrom, 0, sizeof(aFrom[0]));
@@ -3576,6 +3910,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
 
         if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
         if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
+        if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){
+          /* Do not use an automatic index if the this loop is expected
+          ** to run less than 2 times. */
+          assert( 10==sqlite3LogEst(2) );
+          continue;
+        }
         /* At this point, pWLoop is a candidate to be the next loop. 
         ** Compute its cost */
         rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow);
@@ -3592,7 +3932,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         if( isOrdered>=0 && isOrderedpResultSet, pFrom,
+    int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pDistinctSet, pFrom,
                  WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used);
-    if( rc==pWInfo->pResultSet->nExpr ){
+    if( rc==pWInfo->pDistinctSet->nExpr ){
       pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
     }
   }
@@ -3781,8 +4121,26 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       }
     }else{
       pWInfo->nOBSat = pFrom->isOrdered;
-      if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0;
       pWInfo->revMask = pFrom->revLoop;
+      if( pWInfo->nOBSat<=0 ){
+        pWInfo->nOBSat = 0;
+        if( nLoop>0 ){
+          u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
+          if( (wsFlags & WHERE_ONEROW)==0 
+           && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
+          ){
+            Bitmask m = 0;
+            int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
+                      WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
+            testcase( wsFlags & WHERE_IPK );
+            testcase( wsFlags & WHERE_COLUMN_IN );
+            if( rc==pWInfo->pOrderBy->nExpr ){
+              pWInfo->bOrderedInnerLoop = 1;
+              pWInfo->revMask = m;
+            }
+          }
+        }
+      }
     }
     if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
         && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
@@ -3828,9 +4186,9 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   int j;
   Table *pTab;
   Index *pIdx;
-  
+
   pWInfo = pBuilder->pWInfo;
-  if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0;
+  if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
   assert( pWInfo->pTabList->nSrc>=1 );
   pItem = pWInfo->pTabList->a;
   pTab = pItem->pTab;
@@ -3977,7 +4335,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
 **
 ** The iIdxCur parameter is the cursor number of an index.  If 
-** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index
+** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
 ** to use for OR clause processing.  The WHERE clause should use this
 ** specific cursor.  If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
 ** the first cursor in an array of cursors for all indices.  iIdxCur should
@@ -3985,13 +4343,14 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
 ** used.
 */
 WhereInfo *sqlite3WhereBegin(
-  Parse *pParse,        /* The parser context */
-  SrcList *pTabList,    /* FROM clause: A list of all tables to be scanned */
-  Expr *pWhere,         /* The WHERE clause */
-  ExprList *pOrderBy,   /* An ORDER BY (or GROUP BY) clause, or NULL */
-  ExprList *pResultSet, /* Result set of the query */
-  u16 wctrlFlags,       /* One of the WHERE_* flags defined in sqliteInt.h */
-  int iIdxCur           /* If WHERE_ONETABLE_ONLY is set, index cursor number */
+  Parse *pParse,          /* The parser context */
+  SrcList *pTabList,      /* FROM clause: A list of all tables to be scanned */
+  Expr *pWhere,           /* The WHERE clause */
+  ExprList *pOrderBy,     /* An ORDER BY (or GROUP BY) clause, or NULL */
+  ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */
+  u16 wctrlFlags,         /* The WHERE_* flags defined in sqliteInt.h */
+  int iAuxArg             /* If WHERE_OR_SUBCLAUSE is set, index cursor number
+                          ** If WHERE_USE_LIMIT, then the limit amount */
 ){
   int nByteWInfo;            /* Num. bytes allocated for WhereInfo struct */
   int nTabList;              /* Number of elements in pTabList */
@@ -4009,9 +4368,13 @@ WhereInfo *sqlite3WhereBegin(
 
   assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
         (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
-     && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 
+     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 
   ));
 
+  /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
+  assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
+            || (wctrlFlags & WHERE_USE_LIMIT)==0 );
+
   /* Variable initialization */
   db = pParse->db;
   memset(&sWLB, 0, sizeof(sWLB));
@@ -4037,11 +4400,11 @@ WhereInfo *sqlite3WhereBegin(
   }
 
   /* This function normally generates a nested loop for all tables in 
-  ** pTabList.  But if the WHERE_ONETABLE_ONLY flag is set, then we should
+  ** pTabList.  But if the WHERE_OR_SUBCLAUSE flag is set, then we should
   ** only generate code for the first table in pTabList and assume that
   ** any cursors associated with subsequent tables are uninitialized.
   */
-  nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc;
+  nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
 
   /* Allocate and initialize the WhereInfo structure that will become the
   ** return value. A single allocation is used to store the WhereInfo
@@ -4051,21 +4414,25 @@ WhereInfo *sqlite3WhereBegin(
   ** some architectures. Hence the ROUND8() below.
   */
   nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel));
-  pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop));
+  pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
   if( db->mallocFailed ){
     sqlite3DbFree(db, pWInfo);
     pWInfo = 0;
     goto whereBeginError;
   }
-  pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
-  pWInfo->nLevel = nTabList;
   pWInfo->pParse = pParse;
   pWInfo->pTabList = pTabList;
   pWInfo->pOrderBy = pOrderBy;
-  pWInfo->pResultSet = pResultSet;
+  pWInfo->pDistinctSet = pDistinctSet;
+  pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
+  pWInfo->nLevel = nTabList;
   pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v);
   pWInfo->wctrlFlags = wctrlFlags;
+  pWInfo->iLimit = iAuxArg;
   pWInfo->savedNQueryLoop = pParse->nQueryLoop;
+  memset(&pWInfo->nOBSat, 0, 
+         offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
+  memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
   assert( pWInfo->eOnePass==ONEPASS_OFF );  /* ONEPASS defaults to OFF */
   pMaskSet = &pWInfo->sMaskSet;
   sWLB.pWInfo = pWInfo;
@@ -4116,7 +4483,7 @@ WhereInfo *sqlite3WhereBegin(
   ** Note that bitmasks are created for all pTabList->nSrc tables in
   ** pTabList, not just the first nTabList tables.  nTabList is normally
   ** equal to pTabList->nSrc but might be shortened to 1 if the
-  ** WHERE_ONETABLE_ONLY flag is set.
+  ** WHERE_OR_SUBCLAUSE flag is set.
   */
   for(ii=0; iinSrc; ii++){
     createMask(pMaskSet, pTabList->a[ii].iCursor);
@@ -4134,25 +4501,27 @@ WhereInfo *sqlite3WhereBegin(
   if( db->mallocFailed ) goto whereBeginError;
 
   if( wctrlFlags & WHERE_WANT_DISTINCT ){
-    if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){
+    if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){
       /* The DISTINCT marking is pointless.  Ignore it. */
       pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
     }else if( pOrderBy==0 ){
       /* Try to ORDER BY the result set to make distinct processing easier */
       pWInfo->wctrlFlags |= WHERE_DISTINCTBY;
-      pWInfo->pOrderBy = pResultSet;
+      pWInfo->pOrderBy = pDistinctSet;
     }
   }
 
   /* Construct the WhereLoop objects */
-  WHERETRACE(0xffff,("*** Optimizer Start *** (wctrlFlags: 0x%x)\n",
-             wctrlFlags));
 #if defined(WHERETRACE_ENABLED)
-  if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
-    int i;
-    for(i=0; inTerm; i++){
-      whereTermPrint(&sWLB.pWC->a[i], i);
+  if( sqlite3WhereTrace & 0xffff ){
+    sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags);
+    if( wctrlFlags & WHERE_USE_LIMIT ){
+      sqlite3DebugPrintf(", limit: %d", iAuxArg);
     }
+    sqlite3DebugPrintf(")\n");
+  }
+  if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
+    sqlite3WhereClausePrint(sWLB.pWC);
   }
 #endif
 
@@ -4181,7 +4550,7 @@ WhereInfo *sqlite3WhereBegin(
     }
   }
   if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){
-     pWInfo->revMask = (Bitmask)(-1);
+     pWInfo->revMask = ALLBITS;
   }
   if( pParse->nErr || NEVER(db->mallocFailed) ){
     goto whereBeginError;
@@ -4214,10 +4583,10 @@ WhereInfo *sqlite3WhereBegin(
 #endif
   /* Attempt to omit tables from the join that do not effect the result */
   if( pWInfo->nLevel>=2
-   && pResultSet!=0
+   && pDistinctSet!=0
    && OptimizationEnabled(db, SQLITE_OmitNoopJoin)
   ){
-    Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet);
+    Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet);
     if( sWLB.pOrderBy ){
       tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy);
     }
@@ -4294,7 +4663,7 @@ WhereInfo *sqlite3WhereBegin(
     }else
 #endif
     if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-         && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){
+         && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){
       int op = OP_OpenRead;
       if( pWInfo->eOnePass!=ONEPASS_OFF ){
         op = OP_OpenWrite;
@@ -4330,10 +4699,10 @@ WhereInfo *sqlite3WhereBegin(
       Index *pIx = pLoop->u.btree.pIndex;
       int iIndexCur;
       int op = OP_OpenRead;
-      /* iIdxCur is always set if to a positive value if ONEPASS is possible */
-      assert( iIdxCur!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
+      /* iAuxArg is always set if to a positive value if ONEPASS is possible */
+      assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
       if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
-       && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0
+       && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
       ){
         /* This is one term of an OR-optimization using the PRIMARY KEY of a
         ** WITHOUT ROWID table.  No need for a separate index */
@@ -4341,7 +4710,7 @@ WhereInfo *sqlite3WhereBegin(
         op = 0;
       }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
         Index *pJ = pTabItem->pTab->pIndex;
-        iIndexCur = iIdxCur;
+        iIndexCur = iAuxArg;
         assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
         while( ALWAYS(pJ) && pJ!=pIx ){
           iIndexCur++;
@@ -4349,9 +4718,9 @@ WhereInfo *sqlite3WhereBegin(
         }
         op = OP_OpenWrite;
         pWInfo->aiCurOnePass[1] = iIndexCur;
-      }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){
-        iIndexCur = iIdxCur;
-        if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx;
+      }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
+        iIndexCur = iAuxArg;
+        op = OP_ReopenIdx;
       }else{
         iIndexCur = pParse->nTab++;
       }
@@ -4413,7 +4782,7 @@ WhereInfo *sqlite3WhereBegin(
     pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
     notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady);
     pWInfo->iContinue = pLevel->addrCont;
-    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){
+    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
       sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
     }
   }
@@ -4467,10 +4836,12 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
       for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
         sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
-        sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
-        VdbeCoverage(v);
-        VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen);
-        VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen);
+        if( pIn->eEndLoopOp!=OP_Noop ){
+          sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
+          VdbeCoverage(v);
+          VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen);
+          VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen);
+        }
         sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
       }
     }
@@ -4483,24 +4854,21 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     }
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
     if( pLevel->addrLikeRep ){
-      int op;
-      if( sqlite3VdbeGetOp(v, pLevel->addrLikeRep-1)->p1 ){
-        op = OP_DecrJumpZero;
-      }else{
-        op = OP_JumpZeroIncr;
-      }
-      sqlite3VdbeAddOp2(v, op, pLevel->iLikeRepCntr, pLevel->addrLikeRep);
+      sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
+                        pLevel->addrLikeRep);
       VdbeCoverage(v);
     }
 #endif
     if( pLevel->iLeftJoin ){
+      int ws = pLoop->wsFlags;
       addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
-      assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-           || (pLoop->wsFlags & WHERE_INDEXED)!=0 );
-      if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){
+      assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
+      if( (ws & WHERE_IDX_ONLY)==0 ){
         sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
       }
-      if( pLoop->wsFlags & WHERE_INDEXED ){
+      if( (ws & WHERE_INDEXED) 
+       || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) 
+      ){
         sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
       }
       if( pLevel->op==OP_Return ){
@@ -4541,12 +4909,12 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
 
     /* Close all of the cursors that were opened by sqlite3WhereBegin.
     ** Except, do not close cursors that will be reused by the OR optimization
-    ** (WHERE_OMIT_OPEN_CLOSE).  And do not close the OP_OpenWrite cursors
+    ** (WHERE_OR_SUBCLAUSE).  And do not close the OP_OpenWrite cursors
     ** created for the ONEPASS optimization.
     */
     if( (pTab->tabFlags & TF_Ephemeral)==0
      && pTab->pSelect==0
-     && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0
+     && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
     ){
       int ws = pLoop->wsFlags;
       if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){
diff --git a/src/whereInt.h b/src/whereInt.h
index 1a189980..fd6ebe77 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -70,7 +70,7 @@ struct WhereLevel {
   int addrFirst;        /* First instruction of interior of the loop */
   int addrBody;         /* Beginning of the body of this loop */
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
-  int iLikeRepCntr;     /* LIKE range processing counter register */
+  u32 iLikeRepCntr;     /* LIKE range processing counter register (times 2) */
   int addrLikeRep;      /* LIKE range processing address */
 #endif
   u8 iFrom;             /* Which entry in the FROM clause */
@@ -122,6 +122,8 @@ struct WhereLoop {
   union {
     struct {               /* Information for internal btree tables */
       u16 nEq;               /* Number of equality constraints */
+      u16 nBtm;              /* Size of BTM vector */
+      u16 nTop;              /* Size of TOP vector */
       Index *pIndex;         /* Index used, or NULL */
     } btree;
     struct {               /* Information for virtual tables */
@@ -244,19 +246,20 @@ struct WherePath {
 */
 struct WhereTerm {
   Expr *pExpr;            /* Pointer to the subexpression that is this term */
+  WhereClause *pWC;       /* The clause this term is part of */
+  LogEst truthProb;       /* Probability of truth for this expression */
+  u16 wtFlags;            /* TERM_xxx bit flags.  See below */
+  u16 eOperator;          /* A WO_xx value describing  */
+  u8 nChild;              /* Number of children that must disable us */
+  u8 eMatchOp;            /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */
   int iParent;            /* Disable pWC->a[iParent] when this term disabled */
   int leftCursor;         /* Cursor number of X in "X  " */
+  int iField;             /* Field in (?,?,?) IN (SELECT...) vector */
   union {
     int leftColumn;         /* Column number of X in "X  " */
     WhereOrInfo *pOrInfo;   /* Extra information if (eOperator & WO_OR)!=0 */
     WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */
   } u;
-  LogEst truthProb;       /* Probability of truth for this expression */
-  u16 eOperator;          /* A WO_xx value describing  */
-  u16 wtFlags;            /* TERM_xxx bit flags.  See below */
-  u8 nChild;              /* Number of children that must disable us */
-  u8 eMatchOp;            /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */
-  WhereClause *pWC;       /* The clause this term is part of */
   Bitmask prereqRight;    /* Bitmask of tables used by pExpr->pRight */
   Bitmask prereqAll;      /* Bitmask of tables referenced by pExpr */
 };
@@ -408,24 +411,26 @@ struct WhereInfo {
   Parse *pParse;            /* Parsing and code generating context */
   SrcList *pTabList;        /* List of tables in the join */
   ExprList *pOrderBy;       /* The ORDER BY clause or NULL */
-  ExprList *pResultSet;     /* Result set. DISTINCT operates on these */
-  WhereLoop *pLoops;        /* List of all WhereLoop objects */
-  Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
-  LogEst nRowOut;           /* Estimated number of output rows */
+  ExprList *pDistinctSet;   /* DISTINCT over all these values */
+  LogEst iLimit;            /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */
+  int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
+  int iContinue;            /* Jump here to continue with next record */
+  int iBreak;               /* Jump here to break out of the loop */
+  int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
   u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
+  u8 nLevel;                /* Number of nested loop */
   i8 nOBSat;                /* Number of ORDER BY terms satisfied by indices */
   u8 sorted;                /* True if really sorted (not just grouped) */
   u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
   u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
-  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
-  u8 nLevel;                /* Number of nested loop */
+  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values */
+  u8 bOrderedInnerLoop;     /* True if only the inner-most loop is ordered */
   int iTop;                 /* The very beginning of the WHERE loop */
-  int iContinue;            /* Jump here to continue with next record */
-  int iBreak;               /* Jump here to break out of the loop */
-  int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
-  int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
-  WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
+  WhereLoop *pLoops;        /* List of all WhereLoop objects */
+  Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
+  LogEst nRowOut;           /* Estimated number of output rows */
   WhereClause sWC;          /* Decomposition of the WHERE clause */
+  WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
   WhereLevel a[1];          /* Information about each nest loop in WHERE */
 };
 
@@ -435,6 +440,9 @@ struct WhereInfo {
 ** where.c:
 */
 Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
+#ifdef WHERETRACE_ENABLED
+void sqlite3WhereClausePrint(WhereClause *pWC);
+#endif
 WhereTerm *sqlite3WhereFindTerm(
   WhereClause *pWC,     /* The WHERE clause to be searched */
   int iCur,             /* Cursor number of LHS */
@@ -491,6 +499,14 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*);
 ** operators that are of interest to the query planner.  An
 ** OR-ed combination of these values can be used when searching for
 ** particular WhereTerms within a WhereClause.
+**
+** Value constraints:
+**     WO_EQ    == SQLITE_INDEX_CONSTRAINT_EQ
+**     WO_LT    == SQLITE_INDEX_CONSTRAINT_LT
+**     WO_LE    == SQLITE_INDEX_CONSTRAINT_LE
+**     WO_GT    == SQLITE_INDEX_CONSTRAINT_GT
+**     WO_GE    == SQLITE_INDEX_CONSTRAINT_GE
+**     WO_MATCH == SQLITE_INDEX_CONSTRAINT_MATCH
 */
 #define WO_IN     0x0001
 #define WO_EQ     0x0002
diff --git a/src/wherecode.c b/src/wherecode.c
index bb48e5dc..d9e7db23 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -21,6 +21,17 @@
 #include "whereInt.h"
 
 #ifndef SQLITE_OMIT_EXPLAIN
+
+/*
+** Return the name of the i-th column of the pIdx index.
+*/
+static const char *explainIndexColumnName(Index *pIdx, int i){
+  i = pIdx->aiColumn[i];
+  if( i==XN_EXPR ) return "";
+  if( i==XN_ROWID ) return "rowid";
+  return pIdx->pTable->aCol[i].zName;
+}
+
 /*
 ** This routine is a helper for explainIndexRange() below
 **
@@ -31,24 +42,32 @@
 */
 static void explainAppendTerm(
   StrAccum *pStr,             /* The text expression being built */
-  int iTerm,                  /* Index of this term.  First is zero */
-  const char *zColumn,        /* Name of the column */
+  Index *pIdx,                /* Index to read column names from */
+  int nTerm,                  /* Number of terms */
+  int iTerm,                  /* Zero-based index of first term. */
+  int bAnd,                   /* Non-zero to append " AND " */
   const char *zOp             /* Name of the operator */
 ){
-  if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5);
-  sqlite3StrAccumAppendAll(pStr, zColumn);
-  sqlite3StrAccumAppend(pStr, zOp, 1);
-  sqlite3StrAccumAppend(pStr, "?", 1);
-}
+  int i;
 
-/*
-** Return the name of the i-th column of the pIdx index.
-*/
-static const char *explainIndexColumnName(Index *pIdx, int i){
-  i = pIdx->aiColumn[i];
-  if( i==XN_EXPR ) return "";
-  if( i==XN_ROWID ) return "rowid";
-  return pIdx->pTable->aCol[i].zName;
+  assert( nTerm>=1 );
+  if( bAnd ) sqlite3StrAccumAppend(pStr, " AND ", 5);
+
+  if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1);
+  for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1);
+
+  sqlite3StrAccumAppend(pStr, zOp, 1);
+
+  if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1);
+  for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1);
 }
 
 /*
@@ -81,12 +100,11 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){
 
   j = i;
   if( pLoop->wsFlags&WHERE_BTM_LIMIT ){
-    const char *z = explainIndexColumnName(pIndex, i);
-    explainAppendTerm(pStr, i++, z, ">");
+    explainAppendTerm(pStr, pIndex, pLoop->u.btree.nBtm, j, i, ">");
+    i = 1;
   }
   if( pLoop->wsFlags&WHERE_TOP_LIMIT ){
-    const char *z = explainIndexColumnName(pIndex, j);
-    explainAppendTerm(pStr, i, z, "<");
+    explainAppendTerm(pStr, pIndex, pLoop->u.btree.nTop, j, i, "<");
   }
   sqlite3StrAccumAppend(pStr, ")", 1);
 }
@@ -126,7 +144,7 @@ int sqlite3WhereExplainOneScan(
 
     pLoop = pLevel->pWLoop;
     flags = pLoop->wsFlags;
-    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0;
+    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0;
 
     isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
             || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
@@ -276,7 +294,7 @@ void sqlite3WhereAddScanStatus(
 */
 static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){
   int nLoop = 0;
-  while( pTerm
+  while( ALWAYS(pTerm!=0)
       && (pTerm->wtFlags & TERM_CODED)==0
       && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin))
       && (pLevel->notReady & pTerm->prereqAll)==0
@@ -332,16 +350,45 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){
   }
 }
 
+/*
+** Expression pRight, which is the RHS of a comparison operation, is 
+** either a vector of n elements or, if n==1, a scalar expression.
+** Before the comparison operation, affinity zAff is to be applied
+** to the pRight values. This function modifies characters within the
+** affinity string to SQLITE_AFF_BLOB if either:
+**
+**   * the comparison will be performed with no affinity, or
+**   * the affinity change in zAff is guaranteed not to change the value.
+*/
+static void updateRangeAffinityStr(
+  Expr *pRight,                   /* RHS of comparison */
+  int n,                          /* Number of vector elements in comparison */
+  char *zAff                      /* Affinity string to modify */
+){
+  int i;
+  for(i=0; ipVdbe;
   int iReg;                  /* Register holding results */
 
+  assert( pLevel->pWLoop->aLTerm[iEq]==pTerm );
   assert( iTarget>0 );
   if( pX->op==TK_EQ || pX->op==TK_IS ){
     iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget);
@@ -364,10 +412,13 @@ static int codeEqualityTerm(
     sqlite3VdbeAddOp2(v, OP_Null, 0, iReg);
 #ifndef SQLITE_OMIT_SUBQUERY
   }else{
-    int eType;
+    int eType = IN_INDEX_NOOP;
     int iTab;
     struct InLoop *pIn;
     WhereLoop *pLoop = pLevel->pWLoop;
+    int i;
+    int nEq = 0;
+    int *aiMap = 0;
 
     if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
       && pLoop->u.btree.pIndex!=0
@@ -379,7 +430,78 @@ static int codeEqualityTerm(
     }
     assert( pX->op==TK_IN );
     iReg = iTarget;
-    eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0);
+
+    for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){
+        disableTerm(pLevel, pTerm);
+        return iTarget;
+      }
+    }
+    for(i=iEq;inLTerm; i++){
+      if( ALWAYS(pLoop->aLTerm[i]) && pLoop->aLTerm[i]->pExpr==pX ) nEq++;
+    }
+
+    if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){
+      eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0);
+    }else{
+      Select *pSelect = pX->x.pSelect;
+      sqlite3 *db = pParse->db;
+      u16 savedDbOptFlags = db->dbOptFlags;
+      ExprList *pOrigRhs = pSelect->pEList;
+      ExprList *pOrigLhs = pX->pLeft->x.pList;
+      ExprList *pRhs = 0;         /* New Select.pEList for RHS */
+      ExprList *pLhs = 0;         /* New pX->pLeft vector */
+
+      for(i=iEq;inLTerm; i++){
+        if( pLoop->aLTerm[i]->pExpr==pX ){
+          int iField = pLoop->aLTerm[i]->iField - 1;
+          Expr *pNewRhs = sqlite3ExprDup(db, pOrigRhs->a[iField].pExpr, 0);
+          Expr *pNewLhs = sqlite3ExprDup(db, pOrigLhs->a[iField].pExpr, 0);
+
+          pRhs = sqlite3ExprListAppend(pParse, pRhs, pNewRhs);
+          pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs);
+        }
+      }
+      if( !db->mallocFailed ){
+        Expr *pLeft = pX->pLeft;
+
+        if( pSelect->pOrderBy ){
+          /* If the SELECT statement has an ORDER BY clause, zero the 
+          ** iOrderByCol variables. These are set to non-zero when an 
+          ** ORDER BY term exactly matches one of the terms of the 
+          ** result-set. Since the result-set of the SELECT statement may
+          ** have been modified or reordered, these variables are no longer 
+          ** set correctly.  Since setting them is just an optimization, 
+          ** it's easiest just to zero them here.  */
+          ExprList *pOrderBy = pSelect->pOrderBy;
+          for(i=0; inExpr; i++){
+            pOrderBy->a[i].u.x.iOrderByCol = 0;
+          }
+        }
+
+        /* Take care here not to generate a TK_VECTOR containing only a
+        ** single value. Since the parser never creates such a vector, some
+        ** of the subroutines do not handle this case.  */
+        if( pLhs->nExpr==1 ){
+          pX->pLeft = pLhs->a[0].pExpr;
+        }else{
+          pLeft->x.pList = pLhs;
+          aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int) * nEq);
+          testcase( aiMap==0 );
+        }
+        pSelect->pEList = pRhs;
+        db->dbOptFlags |= SQLITE_QueryFlattener;
+        eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap);
+        db->dbOptFlags = savedDbOptFlags;
+        testcase( aiMap!=0 && aiMap[0]!=0 );
+        pSelect->pEList = pOrigRhs;
+        pLeft->x.pList = pOrigLhs;
+        pX->pLeft = pLeft;
+      }
+      sqlite3ExprListDelete(pParse->db, pLhs);
+      sqlite3ExprListDelete(pParse->db, pRhs);
+    }
+
     if( eType==IN_INDEX_INDEX_DESC ){
       testcase( bRev );
       bRev = !bRev;
@@ -389,28 +511,45 @@ static int codeEqualityTerm(
     VdbeCoverageIf(v, bRev);
     VdbeCoverageIf(v, !bRev);
     assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 );
+
     pLoop->wsFlags |= WHERE_IN_ABLE;
     if( pLevel->u.in.nIn==0 ){
       pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
     }
-    pLevel->u.in.nIn++;
+
+    i = pLevel->u.in.nIn;
+    pLevel->u.in.nIn += nEq;
     pLevel->u.in.aInLoop =
        sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop,
                               sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn);
     pIn = pLevel->u.in.aInLoop;
     if( pIn ){
-      pIn += pLevel->u.in.nIn - 1;
-      pIn->iCur = iTab;
-      if( eType==IN_INDEX_ROWID ){
-        pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg);
-      }else{
-        pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg);
+      int iMap = 0;               /* Index in aiMap[] */
+      pIn += i;
+      for(i=iEq;inLTerm; i++){
+        if( pLoop->aLTerm[i]->pExpr==pX ){
+          int iOut = iReg + i - iEq;
+          if( eType==IN_INDEX_ROWID ){
+            testcase( nEq>1 );  /* Happens with a UNIQUE index on ROWID */
+            pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut);
+          }else{
+            int iCol = aiMap ? aiMap[iMap++] : 0;
+            pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut);
+          }
+          sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v);
+          if( i==iEq ){
+            pIn->iCur = iTab;
+            pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen;
+          }else{
+            pIn->eEndLoopOp = OP_Noop;
+          }
+          pIn++;
+        }
       }
-      pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen;
-      sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v);
     }else{
       pLevel->u.in.nIn = 0;
     }
+    sqlite3DbFree(pParse->db, aiMap);
 #endif
   }
   disableTerm(pLevel, pTerm);
@@ -536,9 +675,15 @@ static int codeAllEqualityTerms(
         sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j);
       }
     }
-    testcase( pTerm->eOperator & WO_ISNULL );
-    testcase( pTerm->eOperator & WO_IN );
-    if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){
+    if( pTerm->eOperator & WO_IN ){
+      if( pTerm->pExpr->flags & EP_xIsSelect ){
+        /* No affinity ever needs to be (or should be) applied to a value
+        ** from the RHS of an "? IN (SELECT ...)" expression. The 
+        ** sqlite3FindInIndex() routine has already ensured that the 
+        ** affinity of the comparison has been applied to the value.  */
+        if( zAff ) zAff[j] = SQLITE_AFF_BLOB;
+      }
+    }else if( (pTerm->eOperator & WO_ISNULL)==0 ){
       Expr *pRight = pTerm->pExpr->pRight;
       if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){
         sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
@@ -560,15 +705,16 @@ static int codeAllEqualityTerms(
 
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
 /*
-** If the most recently coded instruction is a constant range contraint
-** that originated from the LIKE optimization, then change the P3 to be
-** pLoop->iLikeRepCntr and set P5.
+** If the most recently coded instruction is a constant range constraint
+** (a string literal) that originated from the LIKE optimization, then 
+** set P3 and P5 on the OP_String opcode so that the string will be cast
+** to a BLOB at appropriate times.
 **
 ** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range
 ** expression: "x>='ABC' AND x<'abd'".  But this requires that the range
 ** scan loop run twice, once for strings and a second time for BLOBs.
 ** The OP_String opcodes on the second pass convert the upper and lower
-** bound string contants to blobs.  This routine makes the necessary changes
+** bound string constants to blobs.  This routine makes the necessary changes
 ** to the OP_String opcodes for that to happen.
 **
 ** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then
@@ -587,8 +733,8 @@ static void whereLikeOptimizationStringFixup(
     assert( pOp!=0 );
     assert( pOp->opcode==OP_String8 
             || pTerm->pWC->pWInfo->pParse->db->mallocFailed );
-    pOp->p3 = pLevel->iLikeRepCntr;
-    pOp->p5 = 1;
+    pOp->p3 = (int)(pLevel->iLikeRepCntr>>1);  /* Register holding counter */
+    pOp->p5 = (u8)(pLevel->iLikeRepCntr&1);    /* ASC or DESC */
   }
 }
 #else
@@ -625,6 +771,38 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
   return WRC_Continue;
 }
 
+/*
+** Test whether or not expression pExpr, which was part of a WHERE clause,
+** should be included in the cursor-hint for a table that is on the rhs
+** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the 
+** expression is not suitable.
+**
+** An expression is unsuitable if it might evaluate to non NULL even if
+** a TK_COLUMN node that does affect the value of the expression is set
+** to NULL. For example:
+**
+**   col IS NULL
+**   col IS NOT NULL
+**   coalesce(col, 1)
+**   CASE WHEN col THEN 0 ELSE 1 END
+*/
+static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_IS 
+   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT 
+   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE 
+  ){
+    pWalker->eCode = 1;
+  }else if( pExpr->op==TK_FUNCTION ){
+    int d1;
+    char d2[3];
+    if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){
+      pWalker->eCode = 1;
+    }
+  }
+
+  return WRC_Continue;
+}
+
 
 /*
 ** This function is called on every node of an expression tree used as an
@@ -677,6 +855,7 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
 ** Insert an OP_CursorHint instruction if it is appropriate to do so.
 */
 static void codeCursorHint(
+  struct SrcList_item *pTabItem,  /* FROM clause item */
   WhereInfo *pWInfo,    /* The where clause */
   WhereLevel *pLevel,   /* Which loop to provide hints for */
   WhereTerm *pEndRange  /* Hint this end-of-scan boundary term if not NULL */
@@ -707,7 +886,42 @@ static void codeCursorHint(
     pTerm = &pWC->a[i];
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( pTerm->prereqAll & pLevel->notReady ) continue;
-    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+
+    /* Any terms specified as part of the ON(...) clause for any LEFT 
+    ** JOIN for which the current table is not the rhs are omitted
+    ** from the cursor-hint. 
+    **
+    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms 
+    ** that were specified as part of the WHERE clause must be excluded.
+    ** This is to address the following:
+    **
+    **   SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL;
+    **
+    ** Say there is a single row in t2 that matches (t1.a=t2.b), but its
+    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is 
+    ** pushed down to the cursor, this row is filtered out, causing
+    ** SQLite to synthesize a row of NULL values. Which does match the
+    ** WHERE clause, and so the query returns a row. Which is incorrect.
+    **
+    ** For the same reason, WHERE terms such as:
+    **
+    **   WHERE 1 = (t2.c IS NULL)
+    **
+    ** are also excluded. See codeCursorHintIsOrFunction() for details.
+    */
+    if( pTabItem->fg.jointype & JT_LEFT ){
+      Expr *pExpr = pTerm->pExpr;
+      if( !ExprHasProperty(pExpr, EP_FromJoin) 
+       || pExpr->iRightJoinTable!=pTabItem->iCursor
+      ){
+        sWalker.eCode = 0;
+        sWalker.xExprCallback = codeCursorHintIsOrFunction;
+        sqlite3WalkExpr(&sWalker, pTerm->pExpr);
+        if( sWalker.eCode ) continue;
+      }
+    }else{
+      if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+    }
 
     /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize
     ** the cursor.  These terms are not needed as hints for a pure range
@@ -741,7 +955,7 @@ static void codeCursorHint(
   }
 }
 #else
-# define codeCursorHint(A,B,C)  /* No-op */
+# define codeCursorHint(A,B,C,D)  /* No-op */
 #endif /* SQLITE_ENABLE_CURSOR_HINTS */
 
 /*
@@ -775,7 +989,7 @@ static void codeDeferredSeek(
   assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 );
   
   sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur);
-  if( (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)
+  if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
    && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask)
   ){
     int i;
@@ -792,6 +1006,39 @@ static void codeDeferredSeek(
   }
 }
 
+/*
+** If the expression passed as the second argument is a vector, generate
+** code to write the first nReg elements of the vector into an array
+** of registers starting with iReg.
+**
+** If the expression is not a vector, then nReg must be passed 1. In
+** this case, generate code to evaluate the expression and leave the
+** result in register iReg.
+*/
+static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){
+  assert( nReg>0 );
+  if( sqlite3ExprIsVector(p) ){
+#ifndef SQLITE_OMIT_SUBQUERY
+    if( (p->flags & EP_xIsSelect) ){
+      Vdbe *v = pParse->pVdbe;
+      int iSelect = sqlite3CodeSubselect(pParse, p, 0, 0);
+      sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1);
+    }else
+#endif
+    {
+      int i;
+      ExprList *pList = p->x.pList;
+      assert( nReg<=pList->nExpr );
+      for(i=0; ia[i].pExpr, iReg+i);
+      }
+    }
+  }else{
+    assert( nReg==1 );
+    sqlite3ExprCode(pParse, p, iReg);
+  }
+}
+
 /*
 ** Generate code for the start of the iLevel-th loop in the WHERE clause
 ** implementation described by pWInfo.
@@ -830,7 +1077,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur);
   bRev = (pWInfo->revMask>>iLevel)&1;
   omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 
-           && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0;
+           && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0;
   VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName));
 
   /* Create labels for the "break" and "continue" instructions
@@ -874,6 +1121,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     int iReg;   /* P3 Value for OP_VFilter */
     int addrNotFound;
     int nConstraint = pLoop->nLTerm;
+    int iIn;    /* Counter for IN constraints */
 
     sqlite3ExprCachePush(pParse);
     iReg = sqlite3GetTempRange(pParse, nConstraint+2);
@@ -881,12 +1129,13 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     for(j=0; jaLTerm[j];
-      if( pTerm==0 ) continue;
+      if( NEVER(pTerm==0) ) continue;
       if( pTerm->eOperator & WO_IN ){
         codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget);
         addrNotFound = pLevel->addrNxt;
       }else{
-        sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget);
+        Expr *pRight = pTerm->pExpr->pRight;
+        codeExprOrVector(pParse, pRight, iTarget, 1);
       }
     }
     sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg);
@@ -896,15 +1145,57 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                       pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC);
     VdbeCoverage(v);
     pLoop->u.vtab.needFree = 0;
-    for(j=0; ju.vtab.omitMask>>j)&1 ){
-        disableTerm(pLevel, pLoop->aLTerm[j]);
-      }
-    }
     pLevel->p1 = iCur;
     pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext;
     pLevel->p2 = sqlite3VdbeCurrentAddr(v);
-    sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
+    iIn = pLevel->u.in.nIn;
+    for(j=nConstraint-1; j>=0; j--){
+      pTerm = pLoop->aLTerm[j];
+      if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){
+        disableTerm(pLevel, pTerm);
+      }else if( (pTerm->eOperator & WO_IN)!=0 ){
+        Expr *pCompare;  /* The comparison operator */
+        Expr *pRight;    /* RHS of the comparison */
+        VdbeOp *pOp;     /* Opcode to access the value of the IN constraint */
+
+        /* Reload the constraint value into reg[iReg+j+2].  The same value
+        ** was loaded into the same register prior to the OP_VFilter, but
+        ** the xFilter implementation might have changed the datatype or
+        ** encoding of the value in the register, so it *must* be reloaded. */
+        assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed );
+        if( !db->mallocFailed ){
+          assert( iIn>0 );
+          pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop);
+          assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid );
+          assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 );
+          assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 );
+          testcase( pOp->opcode==OP_Rowid );
+          sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3);
+        }
+
+        /* Generate code that will continue to the next row if 
+        ** the IN constraint is not satisfied */
+        pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0);
+        assert( pCompare!=0 || db->mallocFailed );
+        if( pCompare ){
+          pCompare->pLeft = pTerm->pExpr->pLeft;
+          pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0);
+          if( pRight ){
+            pRight->iTable = iReg+j+2;
+            sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0);
+          }
+          pCompare->pLeft = 0;
+          sqlite3ExprDelete(db, pCompare);
+        }
+      }
+    }
+    /* These registers need to be preserved in case there is an IN operator
+    ** loop.  So we could deallocate the registers here (and potentially
+    ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0.  But it seems
+    ** simpler and safer to simply not reuse the registers.
+    **
+    **    sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
+    */
     sqlite3ExprCachePop(pParse);
   }else
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -927,8 +1218,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg);
     if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg);
     addrNxt = pLevel->addrNxt;
-    sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v);
-    sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg);
+    sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
     VdbeCoverage(v);
     sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1);
     sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
@@ -955,10 +1245,11 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pStart = pEnd;
       pEnd = pTerm;
     }
-    codeCursorHint(pWInfo, pLevel, pEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pEnd);
     if( pStart ){
       Expr *pX;             /* The expression that defines the start bound */
       int r1, rTemp;        /* Registers for holding the start boundary */
+      int op;               /* Cursor seek operation */
 
       /* The following constant maps TK_xx codes into corresponding 
       ** seek opcodes.  It depends on a particular ordering of TK_xx
@@ -978,8 +1269,16 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pX = pStart->pExpr;
       assert( pX!=0 );
       testcase( pStart->leftCursor!=iCur ); /* transitive constraints */
-      r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
-      sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1);
+      if( sqlite3ExprIsVector(pX->pRight) ){
+        r1 = rTemp = sqlite3GetTempReg(pParse);
+        codeExprOrVector(pParse, pX->pRight, r1, 1);
+        op = aMoveOp[(pX->op - TK_GT) | 0x0001];
+      }else{
+        r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
+        disableTerm(pLevel, pStart);
+        op = aMoveOp[(pX->op - TK_GT)];
+      }
+      sqlite3VdbeAddOp3(v, op, iCur, addrBrk, r1);
       VdbeComment((v, "pk"));
       VdbeCoverageIf(v, pX->op==TK_GT);
       VdbeCoverageIf(v, pX->op==TK_LE);
@@ -987,7 +1286,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       VdbeCoverageIf(v, pX->op==TK_GE);
       sqlite3ExprCacheAffinityChange(pParse, r1, 1);
       sqlite3ReleaseTempReg(pParse, rTemp);
-      disableTerm(pLevel, pStart);
     }else{
       sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk);
       VdbeCoverageIf(v, bRev==0);
@@ -1001,13 +1299,17 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */
       testcase( pEnd->wtFlags & TERM_VIRTUAL );
       memEndValue = ++pParse->nMem;
-      sqlite3ExprCode(pParse, pX->pRight, memEndValue);
-      if( pX->op==TK_LT || pX->op==TK_GT ){
+      codeExprOrVector(pParse, pX->pRight, memEndValue, 1);
+      if( 0==sqlite3ExprIsVector(pX->pRight) 
+       && (pX->op==TK_LT || pX->op==TK_GT) 
+      ){
         testOp = bRev ? OP_Le : OP_Ge;
       }else{
         testOp = bRev ? OP_Lt : OP_Gt;
       }
-      disableTerm(pLevel, pEnd);
+      if( 0==sqlite3ExprIsVector(pX->pRight) ){
+        disableTerm(pLevel, pEnd);
+      }
     }
     start = sqlite3VdbeCurrentAddr(v);
     pLevel->op = bRev ? OP_Prev : OP_Next;
@@ -1074,6 +1376,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       OP_IdxLT,            /* 3: (end_constraints &&  bRev &&  endEq) */
     };
     u16 nEq = pLoop->u.btree.nEq;     /* Number of == or IN terms */
+    u16 nBtm = pLoop->u.btree.nBtm;   /* Length of BTM vector */
+    u16 nTop = pLoop->u.btree.nTop;   /* Length of TOP vector */
     int regBase;                 /* Base register holding constraint values */
     WhereTerm *pRangeStart = 0;  /* Inequality constraint at range start */
     WhereTerm *pRangeEnd = 0;    /* Inequality constraint at range end */
@@ -1086,7 +1390,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     int nExtraReg = 0;           /* Number of extra registers needed */
     int op;                      /* Instruction opcode */
     char *zStartAff;             /* Affinity for start of range constraint */
-    char cEndAff = 0;            /* Affinity for end of range constraint */
+    char *zEndAff = 0;           /* Affinity for end of range constraint */
     u8 bSeekPastNull = 0;        /* True to seek past initial nulls */
     u8 bStopAtNull = 0;          /* Add condition to terminate at NULLs */
 
@@ -1120,33 +1424,36 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     j = nEq;
     if( pLoop->wsFlags & WHERE_BTM_LIMIT ){
       pRangeStart = pLoop->aLTerm[j++];
-      nExtraReg = 1;
+      nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm);
       /* Like optimization range constraints always occur in pairs */
       assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || 
               (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 );
     }
     if( pLoop->wsFlags & WHERE_TOP_LIMIT ){
       pRangeEnd = pLoop->aLTerm[j++];
-      nExtraReg = 1;
+      nExtraReg = MAX(nExtraReg, pLoop->u.btree.nTop);
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
       if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){
         assert( pRangeStart!=0 );                     /* LIKE opt constraints */
         assert( pRangeStart->wtFlags & TERM_LIKEOPT );   /* occur in pairs */
-        pLevel->iLikeRepCntr = ++pParse->nMem;
-        testcase( bRev );
-        testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC );
-        sqlite3VdbeAddOp2(v, OP_Integer,
-                          bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC),
-                          pLevel->iLikeRepCntr);
+        pLevel->iLikeRepCntr = (u32)++pParse->nMem;
+        sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)pLevel->iLikeRepCntr);
         VdbeComment((v, "LIKE loop counter"));
         pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v);
+        /* iLikeRepCntr actually stores 2x the counter register number.  The
+        ** bottom bit indicates whether the search order is ASC or DESC. */
+        testcase( bRev );
+        testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC );
+        assert( (bRev & ~1)==0 );
+        pLevel->iLikeRepCntr <<=1;
+        pLevel->iLikeRepCntr |= bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC);
       }
 #endif
-      if( pRangeStart==0
-       && (j = pIdx->aiColumn[nEq])>=0 
-       && pIdx->pTable->aCol[j].notNull==0
-      ){
-        bSeekPastNull = 1;
+      if( pRangeStart==0 ){
+        j = pIdx->aiColumn[nEq];
+        if( (j>=0 && pIdx->pTable->aCol[j].notNull==0) || j==XN_EXPR ){
+          bSeekPastNull = 1;
+        }
       }
     }
     assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 );
@@ -1160,16 +1467,19 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ){
       SWAP(WhereTerm *, pRangeEnd, pRangeStart);
       SWAP(u8, bSeekPastNull, bStopAtNull);
+      SWAP(u8, nBtm, nTop);
     }
 
     /* Generate code to evaluate all constraint terms using == or IN
     ** and store the values of those terms in an array of registers
     ** starting at regBase.
     */
-    codeCursorHint(pWInfo, pLevel, pRangeEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd);
     regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff);
     assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq );
-    if( zStartAff ) cEndAff = zStartAff[nEq];
+    if( zStartAff && nTop ){
+      zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]);
+    }
     addrNxt = pLevel->addrNxt;
 
     testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 );
@@ -1184,7 +1494,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     nConstraint = nEq;
     if( pRangeStart ){
       Expr *pRight = pRangeStart->pExpr->pRight;
-      sqlite3ExprCode(pParse, pRight, regBase+nEq);
+      codeExprOrVector(pParse, pRight, regBase+nEq, nBtm);
       whereLikeOptimizationStringFixup(v, pLevel, pRangeStart);
       if( (pRangeStart->wtFlags & TERM_VNULL)==0
        && sqlite3ExprCanBeNull(pRight)
@@ -1193,18 +1503,16 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         VdbeCoverage(v);
       }
       if( zStartAff ){
-        if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_BLOB){
-          /* Since the comparison is to be performed with no conversions
-          ** applied to the operands, set the affinity to apply to pRight to 
-          ** SQLITE_AFF_BLOB.  */
-          zStartAff[nEq] = SQLITE_AFF_BLOB;
-        }
-        if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){
-          zStartAff[nEq] = SQLITE_AFF_BLOB;
-        }
+        updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]);
       }  
-      nConstraint++;
+      nConstraint += nBtm;
       testcase( pRangeStart->wtFlags & TERM_VIRTUAL );
+      if( sqlite3ExprIsVector(pRight)==0 ){
+        disableTerm(pLevel, pRangeStart);
+      }else{
+        startEq = 1;
+      }
+      bSeekPastNull = 0;
     }else if( bSeekPastNull ){
       sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
       nConstraint++;
@@ -1212,16 +1520,22 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       start_constraints = 1;
     }
     codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff);
-    op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
-    assert( op!=0 );
-    sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
-    VdbeCoverage(v);
-    VdbeCoverageIf(v, op==OP_Rewind);  testcase( op==OP_Rewind );
-    VdbeCoverageIf(v, op==OP_Last);    testcase( op==OP_Last );
-    VdbeCoverageIf(v, op==OP_SeekGT);  testcase( op==OP_SeekGT );
-    VdbeCoverageIf(v, op==OP_SeekGE);  testcase( op==OP_SeekGE );
-    VdbeCoverageIf(v, op==OP_SeekLE);  testcase( op==OP_SeekLE );
-    VdbeCoverageIf(v, op==OP_SeekLT);  testcase( op==OP_SeekLT );
+    if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){
+      /* The skip-scan logic inside the call to codeAllEqualityConstraints()
+      ** above has already left the cursor sitting on the correct row,
+      ** so no further seeking is needed */
+    }else{
+      op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
+      assert( op!=0 );
+      sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
+      VdbeCoverage(v);
+      VdbeCoverageIf(v, op==OP_Rewind);  testcase( op==OP_Rewind );
+      VdbeCoverageIf(v, op==OP_Last);    testcase( op==OP_Last );
+      VdbeCoverageIf(v, op==OP_SeekGT);  testcase( op==OP_SeekGT );
+      VdbeCoverageIf(v, op==OP_SeekGE);  testcase( op==OP_SeekGE );
+      VdbeCoverageIf(v, op==OP_SeekLE);  testcase( op==OP_SeekLE );
+      VdbeCoverageIf(v, op==OP_SeekLT);  testcase( op==OP_SeekLT );
+    }
 
     /* Load the value for the inequality constraint at the end of the
     ** range (if any).
@@ -1230,7 +1544,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( pRangeEnd ){
       Expr *pRight = pRangeEnd->pExpr->pRight;
       sqlite3ExprCacheRemove(pParse, regBase+nEq, 1);
-      sqlite3ExprCode(pParse, pRight, regBase+nEq);
+      codeExprOrVector(pParse, pRight, regBase+nEq, nTop);
       whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd);
       if( (pRangeEnd->wtFlags & TERM_VNULL)==0
        && sqlite3ExprCanBeNull(pRight)
@@ -1238,19 +1552,27 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
         VdbeCoverage(v);
       }
-      if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_BLOB
-       && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff)
-      ){
-        codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff);
+      if( zEndAff ){
+        updateRangeAffinityStr(pRight, nTop, zEndAff);
+        codeApplyAffinity(pParse, regBase+nEq, nTop, zEndAff);
+      }else{
+        assert( pParse->db->mallocFailed );
       }
-      nConstraint++;
+      nConstraint += nTop;
       testcase( pRangeEnd->wtFlags & TERM_VIRTUAL );
+
+      if( sqlite3ExprIsVector(pRight)==0 ){
+        disableTerm(pLevel, pRangeEnd);
+      }else{
+        endEq = 1;
+      }
     }else if( bStopAtNull ){
       sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
       endEq = 0;
       nConstraint++;
     }
     sqlite3DbFree(db, zStartAff);
+    sqlite3DbFree(db, zEndAff);
 
     /* Top of the loop body */
     pLevel->p2 = sqlite3VdbeCurrentAddr(v);
@@ -1266,12 +1588,10 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     }
 
     /* Seek the table cursor, if required */
-    disableTerm(pLevel, pRangeStart);
-    disableTerm(pLevel, pRangeEnd);
     if( omitTable ){
       /* pIdx is a covering index.  No need to access the main table. */
     }else if( HasRowid(pIdx->pTable) ){
-      if( pWInfo->eOnePass!=ONEPASS_OFF ){
+      if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)!=0 ){
         iRowidReg = ++pParse->nMem;
         sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
         sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
@@ -1291,9 +1611,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                            iRowidReg, pPk->nKeyCol); VdbeCoverage(v);
     }
 
-    /* Record the instruction used to terminate the loop. Disable 
-    ** WHERE clause terms made redundant by the index range scan.
-    */
+    /* Record the instruction used to terminate the loop. */
     if( pLoop->wsFlags & WHERE_ONEROW ){
       pLevel->op = OP_Noop;
     }else if( bRev ){
@@ -1370,7 +1688,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     u16 wctrlFlags;                    /* Flags for sub-WHERE clause */
     Expr *pAndExpr = 0;                /* An ".. AND (...)" expression */
     Table *pTab = pTabItem->pTab;
-   
+
     pTerm = pLoop->aLTerm[0];
     assert( pTerm!=0 );
     assert( pTerm->eOperator & WO_OR );
@@ -1464,10 +1782,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ** eliminating duplicates from other WHERE clauses, the action for each
     ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
     */
-    wctrlFlags =  WHERE_OMIT_OPEN_CLOSE
-                | WHERE_FORCE_TABLE
-                | WHERE_ONETABLE_ONLY
-                | WHERE_NO_AUTOINDEX;
+    wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
     for(ii=0; iinTerm; ii++){
       WhereTerm *pOrTerm = &pOrWc->a[ii];
       if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
@@ -1575,7 +1890,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
           ){
             assert( pSubWInfo->a[0].iIdxCur==iCovCur );
             pCov = pSubLoop->u.btree.pIndex;
-            wctrlFlags |= WHERE_REOPEN_IDX;
           }else{
             pCov = 0;
           }
@@ -1612,7 +1926,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
       pLevel->op = OP_Noop;
     }else{
-      codeCursorHint(pWInfo, pLevel, 0);
+      codeCursorHint(pTabItem, pWInfo, pLevel, 0);
       pLevel->op = aStep[bRev];
       pLevel->p1 = iCur;
       pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
@@ -1637,7 +1951,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( (pTerm->prereqAll & pLevel->notReady)!=0 ){
       testcase( pWInfo->untestedTerms==0
-               && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 );
+               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
       pWInfo->untestedTerms = 1;
       continue;
     }
@@ -1647,11 +1961,17 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       continue;
     }
     if( pTerm->wtFlags & TERM_LIKECOND ){
+      /* If the TERM_LIKECOND flag is set, that means that the range search
+      ** is sufficient to guarantee that the LIKE operator is true, so we
+      ** can skip the call to the like(A,B) function.  But this only works
+      ** for strings.  So do not skip the call to the function on the pass
+      ** that compares BLOBs. */
 #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS
       continue;
 #else
-      assert( pLevel->iLikeRepCntr>0 );
-      skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr);
+      u32 x = pLevel->iLikeRepCntr;
+      assert( x>0 );
+      skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1));
       VdbeCoverage(v);
 #endif
     }
@@ -1669,7 +1989,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   ** the implied "t1.a=123" constraint.
   */
   for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
-    Expr *pE, *pEAlt;
+    Expr *pE, sEAlt;
     WhereTerm *pAlt;
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue;
@@ -1687,13 +2007,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     testcase( pAlt->eOperator & WO_IS );
     testcase( pAlt->eOperator & WO_IN );
     VdbeModuleComment((v, "begin transitive constraint"));
-    pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt));
-    if( pEAlt ){
-      *pEAlt = *pAlt->pExpr;
-      pEAlt->pLeft = pE->pLeft;
-      sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL);
-      sqlite3StackFree(db, pEAlt);
-    }
+    sEAlt = *pAlt->pExpr;
+    sEAlt.pLeft = pE->pLeft;
+    sqlite3ExprIfFalse(pParse, &sEAlt, addrCont, SQLITE_JUMPIFNULL);
   }
 
   /* For a LEFT OUTER JOIN, generate code that will record the fact that
diff --git a/src/whereexpr.c b/src/whereexpr.c
index c84d2f23..13d56114 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -77,7 +77,6 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){
       sqlite3DbFree(db, pOld);
     }
     pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]);
-    memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm));
   }
   pTerm = &pWC->a[idx = pWC->nTerm++];
   if( p && ExprHasProperty(p, EP_Unlikely) ){
@@ -89,13 +88,15 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){
   pTerm->wtFlags = wtFlags;
   pTerm->pWC = pWC;
   pTerm->iParent = -1;
+  memset(&pTerm->eOperator, 0,
+         sizeof(WhereTerm) - offsetof(WhereTerm,eOperator));
   return idx;
 }
 
 /*
 ** Return TRUE if the given operator is one of the operators that is
 ** allowed for an indexable WHERE clause term.  The allowed operators are
-** "=", "<", ">", "<=", ">=", "IN", and "IS NULL"
+** "=", "<", ">", "<=", ">=", "IN", "IS", and "IS NULL"
 */
 static int allowedOp(int op){
   assert( TK_GT>TK_EQ && TK_GTwtFlags |= TERM_ORINFO;
   pOrWc = &pOrInfo->wc;
+  memset(pOrWc->aStatic, 0, sizeof(pOrWc->aStatic));
   sqlite3WhereClauseInit(pOrWc, pWInfo);
   sqlite3WhereSplit(pOrWc, pExpr, TK_OR);
   sqlite3WhereExprAnalyze(pSrc, pOrWc);
@@ -559,6 +561,7 @@ static void exprAnalyzeOrTerm(
         pOrTerm->wtFlags |= TERM_ANDINFO;
         pOrTerm->eOperator = WO_AND;
         pAndWC = &pAndInfo->wc;
+        memset(pAndWC->aStatic, 0, sizeof(pAndWC->aStatic));
         sqlite3WhereClauseInit(pAndWC, pWC->pWInfo);
         sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND);
         sqlite3WhereExprAnalyze(pSrc, pAndWC);
@@ -566,7 +569,9 @@ static void exprAnalyzeOrTerm(
         if( !db->mallocFailed ){
           for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){
             assert( pAndTerm->pExpr );
-            if( allowedOp(pAndTerm->pExpr->op) ){
+            if( allowedOp(pAndTerm->pExpr->op) 
+             || pAndTerm->eOperator==WO_MATCH 
+            ){
               b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
             }
           }
@@ -781,12 +786,10 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){
   pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight);
   if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1;
   pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
-  /* Since pLeft and pRight are both a column references, their collating
-  ** sequence should always be defined. */
-  zColl1 = ALWAYS(pColl) ? pColl->zName : 0;
+  zColl1 = pColl ? pColl->zName : 0;
   pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight);
-  zColl2 = ALWAYS(pColl) ? pColl->zName : 0;
-  return sqlite3StrICmp(zColl1, zColl2)==0;
+  zColl2 = pColl ? pColl->zName : 0;
+  return sqlite3_stricmp(zColl1, zColl2)==0;
 }
 
 /*
@@ -821,7 +824,8 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
 ** in any index.  Return TRUE (1) if pExpr is an indexed term and return
 ** FALSE (0) if not.  If TRUE is returned, also set *piCur to the cursor
 ** number of the table that is indexed and *piColumn to the column number
-** of the column that is indexed, or -2 if an expression is being indexed.
+** of the column that is indexed, or XN_EXPR (-2) if an expression is being
+** indexed.
 **
 ** If pExpr is a TK_COLUMN column reference, then this routine always returns
 ** true even if that particular column is not indexed, because the column
@@ -829,6 +833,7 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
 */
 static int exprMightBeIndexed(
   SrcList *pFrom,        /* The FROM clause */
+  int op,                /* The specific comparison operator */
   Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
   Expr *pExpr,           /* An operand of a comparison operator */
   int *piCur,            /* Write the referenced table cursor number here */
@@ -837,6 +842,17 @@ static int exprMightBeIndexed(
   Index *pIdx;
   int i;
   int iCur;
+
+  /* If this expression is a vector to the left or right of a 
+  ** inequality constraint (>, <, >= or <=), perform the processing 
+  ** on the first element of the vector.  */
+  assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE );
+  assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){
+    pExpr = pExpr->x.pList->a[0].pExpr;
+  }
+
   if( pExpr->op==TK_COLUMN ){
     *piCur = pExpr->iTable;
     *piColumn = pExpr->iColumn;
@@ -849,10 +865,10 @@ static int exprMightBeIndexed(
   for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
     if( pIdx->aColExpr==0 ) continue;
     for(i=0; inKeyCol; i++){
-      if( pIdx->aiColumn[i]!=(-2) ) continue;
+      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
       if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
         *piCur = iCur;
-        *piColumn = -2;
+        *piColumn = XN_EXPR;
         return 1;
       }
     }
@@ -909,6 +925,7 @@ static void exprAnalyze(
   op = pExpr->op;
   if( op==TK_IN ){
     assert( pExpr->pRight==0 );
+    if( sqlite3ExprCheckIN(pParse, pExpr) ) return;
     if( ExprHasProperty(pExpr, EP_xIsSelect) ){
       pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect);
     }else{
@@ -935,18 +952,26 @@ static void exprAnalyze(
     Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft);
     Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
     u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV;
-    if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){
+
+    if( pTerm->iField>0 ){
+      assert( op==TK_IN );
+      assert( pLeft->op==TK_VECTOR );
+      pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr;
+    }
+
+    if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){
       pTerm->leftCursor = iCur;
       pTerm->u.leftColumn = iColumn;
       pTerm->eOperator = operatorMask(op) & opMask;
     }
     if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
     if( pRight 
-     && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn)
+     && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn)
     ){
       WhereTerm *pNew;
       Expr *pDup;
       u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
+      assert( pTerm->iField==0 );
       if( pTerm->leftCursor>=0 ){
         int idxNew;
         pDup = sqlite3ExprDup(db, pExpr, 0);
@@ -1120,7 +1145,7 @@ static void exprAnalyze(
   ** virtual tables.  The native query optimizer does not attempt
   ** to do anything with MATCH functions.
   */
-  if( isMatchOfColumn(pExpr, &eOp2) ){
+  if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){
     int idxNew;
     Expr *pRight, *pLeft;
     WhereTerm *pNewTerm;
@@ -1150,6 +1175,60 @@ static void exprAnalyze(
   }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+  /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create
+  ** new terms for each component comparison - "a = ?" and "b = ?".  The
+  ** new terms completely replace the original vector comparison, which is
+  ** no longer used.
+  **
+  ** This is only required if at least one side of the comparison operation
+  ** is not a sub-select.  */
+  if( pWC->op==TK_AND 
+  && (pExpr->op==TK_EQ || pExpr->op==TK_IS)
+  && sqlite3ExprIsVector(pExpr->pLeft)
+  && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 
+    || (pExpr->pRight->flags & EP_xIsSelect)==0
+  )){
+    int nLeft = sqlite3ExprVectorSize(pExpr->pLeft);
+    int i;
+    assert( nLeft==sqlite3ExprVectorSize(pExpr->pRight) );
+    for(i=0; ipLeft, i);
+      Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i);
+
+      pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight, 0);
+      transferJoinMarkings(pNew, pExpr);
+      idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC);
+      exprAnalyze(pSrc, pWC, idxNew);
+    }
+    pTerm = &pWC->a[idxTerm];
+    pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL;  /* Disable the original */
+    pTerm->eOperator = 0;
+  }
+
+  /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create
+  ** a virtual term for each vector component. The expression object
+  ** used by each such virtual term is pExpr (the full vector IN(...) 
+  ** expression). The WhereTerm.iField variable identifies the index within
+  ** the vector on the LHS that the virtual term represents.
+  **
+  ** This only works if the RHS is a simple SELECT, not a compound
+  */
+  if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->iField==0
+   && pExpr->pLeft->op==TK_VECTOR
+   && pExpr->x.pSelect->pPrior==0
+  ){
+    int i;
+    for(i=0; ipLeft); i++){
+      int idxNew;
+      idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL);
+      pWC->a[idxNew].iField = i+1;
+      exprAnalyze(pSrc, pWC, idxNew);
+      markTermAsChild(pWC, idxNew, idxTerm);
+    }
+  }
+
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   /* When sqlite_stat3 histogram data is available an operator of the
   ** form "x IS NOT NULL" can sometimes be evaluated more efficiently
@@ -1170,7 +1249,7 @@ static void exprAnalyze(
 
     pNewExpr = sqlite3PExpr(pParse, TK_GT,
                             sqlite3ExprDup(db, pLeft, 0),
-                            sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0);
+                            sqlite3ExprAlloc(db, TK_NULL, 0, 0), 0);
 
     idxNew = whereClauseInsert(pWC, pNewExpr,
                               TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL);
@@ -1191,6 +1270,8 @@ static void exprAnalyze(
   /* Prevent ON clause terms of a LEFT JOIN from being used to drive
   ** an index for tables to the left of the join.
   */
+  testcase( pTerm!=&pWC->a[idxTerm] );
+  pTerm = &pWC->a[idxTerm];
   pTerm->prereqRight |= extraRight;
 }
 
@@ -1273,17 +1354,18 @@ void sqlite3WhereClauseClear(WhereClause *pWC){
 ** tree.
 */
 Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){
-  Bitmask mask = 0;
+  Bitmask mask;
   if( p==0 ) return 0;
   if( p->op==TK_COLUMN ){
     mask = sqlite3WhereGetMask(pMaskSet, p->iTable);
     return mask;
   }
-  mask = sqlite3WhereExprUsage(pMaskSet, p->pRight);
-  mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft);
+  assert( !ExprHasProperty(p, EP_TokenOnly) );
+  mask = p->pRight ? sqlite3WhereExprUsage(pMaskSet, p->pRight) : 0;
+  if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft);
   if( ExprHasProperty(p, EP_xIsSelect) ){
     mask |= exprSelectUsage(pMaskSet, p->x.pSelect);
-  }else{
+  }else if( p->x.pList ){
     mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList);
   }
   return mask;
@@ -1347,7 +1429,7 @@ void sqlite3WhereTabFuncArgs(
                       pTab->zName, j);
       return;
     }
-    pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0);
+    pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0);
     if( pColRef==0 ) return;
     pColRef->iTable = pItem->iCursor;
     pColRef->iColumn = k++;
diff --git a/test/alter3.test b/test/alter3.test
index f8ebe056..44b31c98 100644
--- a/test/alter3.test
+++ b/test/alter3.test
@@ -184,7 +184,7 @@ do_test alter3-3.2 {
 if {!$has_codec} {
   do_test alter3-3.3 {
     get_file_format
-  } {4}
+  } {3}
 }
 ifcapable schema_version {
   do_test alter3-3.4 {
@@ -220,7 +220,7 @@ do_test alter3-4.2 {
 if {!$has_codec} {
   do_test alter3-4.3 {
     get_file_format
-  } {4}
+  } {3}
 }
 ifcapable schema_version {
   do_test alter3-4.4 {
@@ -270,7 +270,7 @@ ifcapable attach {
   if {!$has_codec} {
     do_test alter3-5.5 {
       list [get_file_format test2.db] [get_file_format]
-    } {4 4}
+    } {3 3}
   }
   do_test alter3-5.6 {
     execsql {
@@ -347,19 +347,19 @@ if {!$has_codec} {
         ALTER TABLE abc ADD d DEFAULT NULL;
       }
       get_file_format
-    } {4}
+    } {3}
     do_test alter3-7.3 {
       execsql {
         ALTER TABLE abc ADD e DEFAULT 10;
       }
       get_file_format
-    } {4}
+    } {3}
     do_test alter3-7.4 {
       execsql {
         ALTER TABLE abc ADD f DEFAULT NULL;
       }
       get_file_format
-    } {4}
+    } {3}
     do_test alter3-7.5 {
       execsql {
         VACUUM;
diff --git a/test/alter4.test b/test/alter4.test
index ac39d614..2cd78777 100644
--- a/test/alter4.test
+++ b/test/alter4.test
@@ -355,4 +355,23 @@ do_execsql_test alter4-9.3 {
   SELECT typeof(c), c FROM t5;
 } {real 9.22337203685478e+18}
 
+# Confirm that doing an ALTER TABLE on a legacy format database
+# does not corrupt DESC indexes.
+#
+# Ticket https://www.sqlite.org/src/tktview/f68bf68513a1c
+#
+do_test alter4-10.1 {
+  db close
+  sqlite3 db :memory:
+  db eval {
+    PRAGMA legacy_file_format=on;
+    CREATE TABLE t1(a,b,c);
+    CREATE INDEX t1a ON t1(a DESC);
+    INSERT INTO t1 VALUES(1,2,3);
+    INSERT INTO t1 VALUES(2,3,4);
+    ALTER TABLE t1 ADD COLUMN d;
+    PRAGMA integrity_check;
+  }
+} {ok}
+
 finish_test
diff --git a/test/analyze9.test b/test/analyze9.test
index e24f3dd4..918577bd 100644
--- a/test/analyze9.test
+++ b/test/analyze9.test
@@ -1180,7 +1180,7 @@ do_execsql_test 26.1.3 {
 #
 # There should be no other samples that start with (x=10000). So it knows 
 # that (x=10000 AND y<50) must match somewhere between 0 and 99 rows, but
-# know more than that. Guessing less than 20 is therefore unreasonable.
+# no more than that. Guessing less than 20 is therefore unreasonable.
 #
 # At one point though, due to a problem in whereKeyStats(), the planner was
 # estimating that (x=10000 AND y<50) would match only 2 rows.
diff --git a/test/analyzeD.test b/test/analyzeD.test
index 7ef1094f..4588aaee 100644
--- a/test/analyzeD.test
+++ b/test/analyzeD.test
@@ -1,4 +1,4 @@
-# 2005 July 22
+# 2014-10-04
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
@@ -11,7 +11,6 @@
 # This file implements regression tests for SQLite library.
 # This file implements tests for the ANALYZE command.
 #
-# $Id: analyze.test,v 1.9 2008/08/11 18:44:58 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -75,7 +74,7 @@ do_test 1.3 {
 } {}
 
 # Without stat1, because 3001 is larger than all samples in the stat4
-# table, SQLite things that a=3001 matches just 1 row. So it (incorrectly)
+# table, SQLite thinks that a=3001 matches just 1 row. So it (incorrectly)
 # chooses it over the c=150 index (5 rows). Even with stat1 data, things
 # worked this way before commit [e6f7f97dbc].
 #
diff --git a/test/analyzer1.test b/test/analyzer1.test
index ac46704f..51b5f8b6 100644
--- a/test/analyzer1.test
+++ b/test/analyzer1.test
@@ -25,9 +25,12 @@ if {$tcl_platform(platform)=="windows"} {
   set PROG "./sqlite3_analyzer"
 }
 if {![file exe $PROG]} {
-  puts "analyzer1 cannot run because $PROG is not available"
-  finish_test
-  return
+  set PROG [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $PROG]]
+  if {![file exe $PROG]} {
+    puts "analyzer1 cannot run because $PROG is not available"
+    finish_test
+    return
+  }
 }
 db close
 forcedelete test.db test.db-journal test.db-wal
diff --git a/test/auth.test b/test/auth.test
index f3c2fa79..d9e12edc 100644
--- a/test/auth.test
+++ b/test/auth.test
@@ -2432,8 +2432,53 @@ do_test auth-6.3 {
   execsql {SELECT rowid, * FROM t6}
 } {101 1 2 3 4 5 6 7 8}
 
+#-------------------------------------------------------------------------
+# Test that view names are included as zArg4.
+#
+do_execsql_test auth-7.1 {
+  CREATE TABLE t7(a, b, c);
+  CREATE VIEW v7 AS SELECT * FROM t7;
+} {}
+set ::authargs [list]
+proc auth {args} {
+  eval lappend ::authargs [lrange $args 0 4]
+  return SQLITE_OK
+}
+
+do_test auth-7.2 {
+  execsql {SELECT a, c FROM v7}
+  set ::authargs
+} [list                          \
+  SQLITE_SELECT {} {} {} {}      \
+  SQLITE_READ t7 a main v7       \
+  SQLITE_READ t7 b main v7       \
+  SQLITE_READ t7 c main v7       \
+  SQLITE_READ v7 a main {}       \
+  SQLITE_READ v7 c main {}       \
+  SQLITE_SELECT {} {} {} v7      \
+]
+
+set ::authargs [list]
+do_test auth-7.3 {
+  execsql {SELECT a, c FROM t7}
+  set ::authargs
+} [list                          \
+  SQLITE_SELECT {} {} {} {}      \
+  SQLITE_READ t7 a main {}       \
+  SQLITE_READ t7 c main {}       \
+]
+
+set ::authargs [list]
+do_test auth-7.4 {
+  execsql {SELECT a, c FROM t7 AS v7}
+  set ::authargs
+} [list                          \
+  SQLITE_SELECT {} {} {} {}      \
+  SQLITE_READ t7 a main {}       \
+  SQLITE_READ t7 c main {}       \
+]
+
+
 rename proc {}
 rename proc_real proc
-
-
 finish_test
diff --git a/test/auth2.test b/test/auth2.test
index a9d64d08..08d46cac 100644
--- a/test/auth2.test
+++ b/test/auth2.test
@@ -98,12 +98,6 @@ SQLITE_UPDATE sqlite_master tbl_name main {}
 SQLITE_UPDATE sqlite_master rootpage main {}
 SQLITE_UPDATE sqlite_master sql main {}
 SQLITE_READ sqlite_master ROWID main {}
-SQLITE_READ sqlite_master name main {}
-SQLITE_READ sqlite_master rootpage main {}
-SQLITE_READ sqlite_master sql main {}
-SQLITE_READ sqlite_master tbl_name main {}
-SQLITE_READ sqlite_master type main {}
-SQLITE_READ sqlite_master ROWID main {}
 }
 do_test auth2-2.2 {
   set ::authargs {}
@@ -119,12 +113,6 @@ SQLITE_UPDATE sqlite_master tbl_name main {}
 SQLITE_UPDATE sqlite_master rootpage main {}
 SQLITE_UPDATE sqlite_master sql main {}
 SQLITE_READ sqlite_master ROWID main {}
-SQLITE_READ sqlite_master name main {}
-SQLITE_READ sqlite_master rootpage main {}
-SQLITE_READ sqlite_master sql main {}
-SQLITE_READ sqlite_master tbl_name main {}
-SQLITE_READ sqlite_master type main {}
-SQLITE_READ sqlite_master ROWID main {}
 }
 do_test auth2-2.3 {
   set ::authargs {}
diff --git a/test/auth3.test b/test/auth3.test
index eef10b39..ca7e16c8 100644
--- a/test/auth3.test
+++ b/test/auth3.test
@@ -12,8 +12,7 @@
 # Test that the truncate optimization is disabled if the SQLITE_DELETE
 # authorization callback returns SQLITE_IGNORE.
 #
-# $Id: auth3.test,v 1.2 2009/05/04 01:58:31 drh Exp $
-#
+# Test that authorizer is disabled during schema parsing.
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -108,4 +107,22 @@ do_test auth3-2.2 {
   set sqlite_search_count
 } {1}
 
+# 2016-07-28.  A problem report from a private client complaining about
+# an authorizer failure during an ALTER TABLE.  The solution (I think) is
+# to disable the authorizer during schema parsing.
+#
+proc auth {code args} {
+  if {$code=="SQLITE_READ" && [regexp {DoNotRead} $args]} {
+    return SQLITE_DENY
+  }
+  return SQLITE_OK
+}
+do_execsql_test auth3-3.0 {
+  CREATE TEMPORARY TABLE TempTable (
+      key TEXT NOT NULL ON CONFLICT FAIL UNIQUE ON CONFLICT REPLACE,
+      value TEXT NOT NULL ON CONFLICT FAIL);
+  ALTER TABLE TempTable RENAME TO DoNotRead;
+  SELECT name FROM sqlite_temp_master;
+} {DoNotRead sqlite_autoindex_DoNotRead_1}
+
 finish_test
diff --git a/test/autoinc.test b/test/autoinc.test
index 23960061..4c108ead 100644
--- a/test/autoinc.test
+++ b/test/autoinc.test
@@ -663,6 +663,18 @@ ifcapable trigger {
   } {1 124 2 10123}
 }
 
+# 2016-10-03 ticket https://www.sqlite.org/src/tktview/7b3328086a5c1
+# Make sure autoincrement plays nicely with the xfer optimization
+#
+do_execsql_test autoinc-10.1 {
+  DELETE FROM sqlite_sequence;
+  CREATE TABLE t10a(a INTEGER PRIMARY KEY AUTOINCREMENT, b UNIQUE);
+  INSERT INTO t10a VALUES(888,9999);
+  CREATE TABLE t10b(x INTEGER PRIMARY KEY AUTOINCREMENT, y UNIQUE);
+  INSERT INTO t10b SELECT * FROM t10a;
+  SELECT * FROM sqlite_sequence;
+} {t10a 888 t10b 888}
+
 
 
 finish_test
diff --git a/test/autoindex2.test b/test/autoindex2.test
index c8abdf41..afd4a666 100644
--- a/test/autoindex2.test
+++ b/test/autoindex2.test
@@ -218,10 +218,15 @@ do_execsql_test autoindex2-120 {
      AND t1.did = t2.did
      AND t2.uid = t3.uid
   ORDER BY t1.ptime desc LIMIT 500;
-} {0 0 0 {SEARCH TABLE t1 USING INDEX t1x1 (ptime>?)} 0 1 1 {SEARCH TABLE t2 USING INDEX t2x0 (did=?)} 0 2 2 {SEARCH TABLE t3 USING INDEX t3x0 (uid=?)}}
+} {~/AUTO/}
 #
 # ^^^--- Before being fixed, the above was using an automatic covering
 # on t3 and reordering the tables so that t3 was in the outer loop and
 # implementing the ORDER BY clause using a B-Tree.
+#
+# This test is sanitized data received from a user.  The original unsanitized
+# data and STAT4 data is found in the th3private test repository.  See one of
+# the th3private check-ins on 2016-02-25.  The test is much more accurate when
+# STAT4 data is used.
 
 finish_test
diff --git a/test/autovacuum.test b/test/autovacuum.test
index 9ee2cd0f..5c14ed85 100644
--- a/test/autovacuum.test
+++ b/test/autovacuum.test
@@ -269,7 +269,7 @@ do_test autovacuum-2.4.3 {
 } {3 4 5 6 7 8 9 10}
 
 # Right now there are 5 free pages in the database. Consume and then free
-# a 520 pages. Then create 520 tables. This ensures that at least some of the
+# all 520 pages. Then create 520 tables. This ensures that at least some of the
 # desired root-pages reside on the second free-list trunk page, and that the
 # trunk itself is required at some point.
 do_test autovacuum-2.4.4 {
@@ -280,9 +280,20 @@ do_test autovacuum-2.4.4 {
 } {}
 set root_page_list [list]
 set pending_byte_page [expr ($::sqlite_pending_byte / 1024) + 1]
+
+# unusable_pages
+# These are either the pending_byte page or the pointer map pages
+#
+unset -nocomplain unusable_page
+if {[sqlite3 -has-codec]} {
+  array set unusable_page {205 1 408 1}
+} else {
+  array set unusable_page {207 1 412 1}
+}
+set unusable_page($pending_byte_page) 1
+
 for {set i 3} {$i<=532} {incr i} {
-  # 207 and 412 are pointer-map pages.
-  if { $i!=207 && $i!=412 && $i != $pending_byte_page} {
+  if {![info exists unusable_page($i)]} {
     lappend root_page_list $i
   }
 }
diff --git a/test/backcompat.test b/test/backcompat.test
index ea7e6a9e..87ffc4b3 100644
--- a/test/backcompat.test
+++ b/test/backcompat.test
@@ -63,7 +63,7 @@ proc do_backcompat_test {rv bin1 bin2 script} {
       set v [split [db version] .]
       if {[llength $v]==3} {lappend v 0}
       set ::sqlite_libversion [format \
-        "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3]
+        "%d%.2d%.2d%.2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3]
       ]
     }
   }
@@ -85,7 +85,8 @@ proc do_allbackcompat_test {script} {
     set nErr [set_test_counter errors]
     foreach dir {0 1} {
 
-      set bintag [string map {testfixture {}} $bin]
+      set bintag $bin
+      regsub {.*testfixture\.} $bintag {} bintag
       set bintag [string map {\.exe {}} $bintag]
       if {$bintag == ""} {set bintag self}
       set ::bcname ".$bintag.$dir."
@@ -420,6 +421,12 @@ ifcapable fts3 {
       if {[code1 { set ::sqlite_libversion }] >=3071200 
        && [code2 { set ::sqlite_libversion }] >=3071200 
       } {
+        if {[code1 { set ::sqlite_libversion }]<3120000} {
+          set res {0 {0 1} 1 0}
+        } else {
+          set res {1 0}
+        }
+
         do_test backcompat-3.9 {
           sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
           sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); }
@@ -428,7 +435,7 @@ ifcapable fts3 {
           sql2 {
             SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level;
           }
-        } {0 {0 1} 1 0}
+        } $res
 
         do_test backcompat-3.10 {
           sql1 { INSERT INTO t2(t2) VALUES('integrity-check') }
diff --git a/test/backup.test b/test/backup.test
index 3b1e1db9..829b3245 100644
--- a/test/backup.test
+++ b/test/backup.test
@@ -164,7 +164,7 @@ foreach zOpenScript [list {
   set file_dest temp
 }] {
 foreach rows_dest {0 3 10} {
-foreach pgsz_dest {512 1024 2048} {
+foreach pgsz_dest {512 1024 2048 4096} {
 foreach nPagePerStep {1 200} {
 
   # Open the databases.
@@ -176,17 +176,16 @@ foreach nPagePerStep {1 200} {
   # in-memory destination is only possible if the initial destination
   # page size is the same as the source page size (in this case 1024 bytes).
   #
-  set isMemDest [expr {
-    $zDestFile eq ":memory:" || $file_dest eq "temp" && $TEMP_STORE>=2
-  }]
+  set isMemDest [expr { $zDestFile eq ":memory:" || $file_dest eq "temp" }]
 
-  if { $isMemDest==0 || $pgsz_dest == 1024 } {
-    if 0 {
-      puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile"
-      puts -nonewline " (as $db_dest.$file_dest)"
-      puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest"
-      puts ""
-    }
+  if 0 {
+    puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile"
+    puts -nonewline " (as $db_dest.$file_dest)"
+    puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest"
+    puts ""
+  }
+
+  if { $isMemDest==0 || $pgsz_dest==1024 || $rows_dest==0 } {
 
     # Set up the content of the source database.
     execsql {
diff --git a/test/backup4.test b/test/backup4.test
index 417df80e..2756995c 100644
--- a/test/backup4.test
+++ b/test/backup4.test
@@ -23,6 +23,11 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix backup4
 
+# The codec logic does not work for zero-length database files.  A database
+# file must contain at least one page in order to be recognized as an
+# encrypted database.
+do_not_use_codec
+
 #-------------------------------------------------------------------------
 # At one point this test was failing because [db] was using an out of
 # date schema in test case 1.2.
diff --git a/test/backup_malloc.test b/test/backup_malloc.test
index f556861f..5444a70b 100644
--- a/test/backup_malloc.test
+++ b/test/backup_malloc.test
@@ -84,4 +84,36 @@ do_malloc_test backup_malloc-2 -tclprep {
   db2 close
 }
 
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA page_size = 16384;
+  BEGIN;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep {
+  catch { db close }
+  catch { db2 close }
+
+  forcedelete test2.db
+  sqlite3 db2 test2.db
+  sqlite3 db test.db
+  sqlite3_backup B db2 main db main
+} -body {
+
+  set rc [B step 50]
+  if {$rc == "SQLITE_NOMEM" || $rc == "SQLITE_IOERR_NOMEM"} {
+    error "out of memory"
+  }
+
+} -test {
+  faultsim_test_result {0 {}} 
+  faultsim_integrity_check
+  
+  # Finalize the backup.
+  catch { B finish }
+}
+
 finish_test
diff --git a/test/bc_common.tcl b/test/bc_common.tcl
index 78010dfa..c47f9968 100644
--- a/test/bc_common.tcl
+++ b/test/bc_common.tcl
@@ -7,7 +7,7 @@ proc bc_find_binaries {zCaption} {
   # against.
   #
   set binaries [list]
-  set self [file tail [info nameofexec]]
+  set self [info nameofexec]
   set pattern "$self?*"
   if {$::tcl_platform(platform)=="windows"} {
     set pattern [string map {\.exe {}} $pattern]
@@ -52,7 +52,8 @@ proc do_bc_test {bin script} {
   code1 { sqlite3 db test.db }
   code2 { sqlite3 db test.db }
 
-  set bintag [string map {testfixture {}} $bin]
+  set bintag $bin
+  regsub {.*testfixture\.} $bintag {} bintag
   set bintag [string map {\.exe {}} $bintag]
   if {$bintag == ""} {set bintag self}
   set saved_prefix $::testprefix
diff --git a/test/bestindex1.test b/test/bestindex1.test
new file mode 100644
index 00000000..f90f96bf
--- /dev/null
+++ b/test/bestindex1.test
@@ -0,0 +1,275 @@
+# 2016-03-01
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex1
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+register_tcl_module db
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      set clist [lindex $args 0]
+      if {[llength $clist]!=1} { error "unexpected constraint list" }
+      catch { array unset C }
+      array set C [lindex $clist 0]
+      if {$C(usable)} {
+        return "omit 0 cost 0 rows 1 idxnum 555 idxstr eq!"
+      } else {
+        return "cost 1000000 rows 0 idxnum 0 idxstr scan..."
+      }
+    }
+
+  }
+
+  return {}
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+} {}
+
+do_eqp_test 1.1 {
+  SELECT * FROM x1 WHERE a = 'abc'
+} {
+  0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM x1 WHERE a IN ('abc', 'def');
+} {
+  0 0 0 {SCAN TABLE x1 VIRTUAL TABLE INDEX 555:eq!}
+  0 0 0 {EXECUTE LIST SUBQUERY 1}
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+register_tcl_module db
+
+# Parameter $mode may be one of:
+#
+#   "omit" - Implement filtering. Set the omit flag.
+#   "use"  - Implement filtering. Use the constraint, but do not set omit.
+#   "use2" - Do not implement filtering. Use the constraint anyway.
+#
+#   
+proc t1_vtab {mode method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b)"
+    }
+
+    xBestIndex {
+      set SQL_FILTER {SELECT * FROM t1x WHERE a='%1%'}
+      set SQL_SCAN   {SELECT * FROM t1x}
+
+      set clist [lindex $args 0]
+      set idx 0
+      for {set idx 0} {$idx < [llength $clist]} {incr idx} {
+        array unset C
+        array set C [lindex $clist $idx]
+        if {$C(column)==0 && $C(op)=="eq" && $C(usable)} {
+          switch -- $mode {
+            "omit" {
+              return [list omit $idx rows 10 cost 10 idxstr $SQL_FILTER]
+            }
+            "use" {
+              return [list use $idx rows 10 cost 10 idxstr $SQL_FILTER]
+            }
+            "use2" {
+              return [list use $idx rows 10 cost 10 idxstr $SQL_SCAN]
+            }
+            default {
+              error "Bad mode - $mode"
+            }
+          }
+        }
+      }
+
+      return [list idxstr {SELECT * FROM t1x}]
+    }
+
+    xFilter {
+      set map [list %1% [lindex $args 2 0]]
+      set sql [string map $map [lindex $args 1]]
+      return [list sql $sql]
+    }
+  }
+
+  return {}
+}
+
+do_execsql_test 2.1 {
+  CREATE TABLE t1x(i INTEGER PRIMARY KEY, a, b);
+  INSERT INTO t1x VALUES(1, 'one', 1);
+  INSERT INTO t1x VALUES(2, 'two', 2);
+  INSERT INTO t1x VALUES(3, 'three', 3);
+  INSERT INTO t1x VALUES(4, 'four', 4);
+}
+
+foreach {tn mode} {
+  1 use 2 omit 3 use2
+} {
+  do_execsql_test 2.2.$mode.1 "
+    DROP TABLE IF EXISTS t1;
+    CREATE VIRTUAL TABLE t1 USING tcl(t1_vtab $mode);
+  "
+
+  do_execsql_test 2.2.$mode.2 {SELECT * FROM t1} {one 1 two 2 three 3 four 4}
+  do_execsql_test 2.2.$mode.3 {SELECT rowid FROM t1} {1 2 3 4}
+  do_execsql_test 2.2.$mode.4 {SELECT rowid FROM t1 WHERE a='two'} {2} 
+
+  do_execsql_test 2.2.$mode.5 {
+    SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
+  } {1 4} 
+
+  set plan(use) {
+    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
+    0 0 0 {EXECUTE LIST SUBQUERY 1}
+    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  }
+  set plan(omit) {
+    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x WHERE a='%1%'}
+    0 0 0 {EXECUTE LIST SUBQUERY 1}
+    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  }
+  set plan(use2) {
+    0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:SELECT * FROM t1x}
+    0 0 0 {EXECUTE LIST SUBQUERY 1}
+    0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+  }
+
+  do_eqp_test 2.2.$mode.6 { 
+    SELECT rowid FROM t1 WHERE a IN ('one', 'four') ORDER BY +rowid
+  } $plan($mode)
+}
+
+# 2016-04-09.
+# Demonstrate a register overwrite problem when using two virtual
+# tables where the outer loop uses the IN operator.
+#
+set G(collist) [list PrimaryKey flagA columnA]
+set G(cols) [join $G(collist) ,]
+set G(nulls) "NULL"
+
+proc vtab_command {method args} {
+  global G
+
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1($G(cols))"
+    }
+
+    xBestIndex {
+      set clist [lindex $args 0]
+      #puts $clist
+      set W [list]
+      set U [list]
+
+      set i 0
+      for {set idx 0} {$idx < [llength $clist]} {incr idx} {
+        array set c [lindex $clist $idx]
+        if {$c(op)=="eq" && $c(usable)} {
+          lappend W "[lindex $G(collist) $c(column)] = %$i%"
+          lappend U use $idx
+          incr i
+        }
+      }
+
+      if {$W==""} {
+        set sql "SELECT rowid, * FROM t1"
+      } else {
+        set sql "SELECT rowid, * FROM t1 WHERE [join $W { AND }]"
+      }
+
+      return [concat [list idxstr $sql] $U]
+    }
+
+    xFilter {
+      foreach {idxnum idxstr vals} $args {}
+
+      set map [list]
+      for {set i 0} {$i < [llength $vals]} {incr i} {
+        lappend map "%$i%" 
+        set v [lindex $vals $i]
+        if {[string is integer $v]} { 
+          lappend map $v 
+        } else {
+          lappend map "'$v'"
+        }
+      }
+      set sql [string map $map $idxstr]
+
+      #puts "SQL: $sql"
+      return [list sql $sql]
+    }
+  }
+
+  return {}
+}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+register_tcl_module db
+
+do_execsql_test 3.1 "
+  CREATE TABLE t1($G(cols));
+  INSERT INTO t1 VALUES(1, 0, 'ValueA');
+  INSERT INTO t1 VALUES(2, 0, 'ValueA');
+  INSERT INTO t1 VALUES(3, 0, 'ValueB');
+  INSERT INTO t1 VALUES(4, 0, 'ValueB');
+"
+
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE VirtualTableA USING tcl(vtab_command);
+  CREATE VIRTUAL TABLE VirtualTableB USING tcl(vtab_command);
+}
+
+do_execsql_test 3.3 { SELECT primarykey FROM VirtualTableA } {1 2 3 4}
+
+do_execsql_test 3.4 {
+  SELECT * FROM 
+  VirtualTableA a CROSS JOIN VirtualTableB b ON b.PrimaryKey=a.PrimaryKey
+  WHERE a.ColumnA IN ('ValueA', 'ValueB') AND a.FlagA=0
+} {
+  1 0 ValueA 1 0 ValueA
+  2 0 ValueA 2 0 ValueA
+  3 0 ValueB 3 0 ValueB
+  4 0 ValueB 4 0 ValueB
+}
+
+do_execsql_test 3.5 {
+  SELECT * FROM 
+  VirtualTableA a CROSS JOIN VirtualTableB b ON b.PrimaryKey=a.PrimaryKey
+  WHERE a.FlagA=0 AND a.ColumnA IN ('ValueA', 'ValueB') 
+} {
+  1 0 ValueA 1 0 ValueA
+  2 0 ValueA 2 0 ValueA
+  3 0 ValueB 3 0 ValueB
+  4 0 ValueB 4 0 ValueB
+}
+
+
+finish_test
diff --git a/test/bestindex2.test b/test/bestindex2.test
new file mode 100644
index 00000000..8bc3fbc3
--- /dev/null
+++ b/test/bestindex2.test
@@ -0,0 +1,141 @@
+# 2016 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex2
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for table named $tbl, with the columns specified
+# by list argument $cols. e.g. if the function is invoked as:
+#
+#   vtab_cmd t1 {a b c} ...
+#
+# The table created is:
+#
+#      "CREATE TABLE t1 (a, b, c)"
+#
+# The tables xBestIndex method behaves as if all possible combinations of
+# "=" constraints (but no others) may be optimized. The cost of a full table
+# scan is:
+#
+#      "WHERE 1"                "cost 1000000 rows 1000000"
+#
+# If one or more "=" constraints are in use, the cost and estimated number
+# of rows returned are both is (11 - nCons)*1000, where nCons is the number
+# of constraints used. e.g.
+#
+#   "WHERE a=? AND b=?"    ->   "cost  900 rows  900"
+#   "WHERE c=? AND b   "cost 1000 rows 1000"
+#  
+proc vtab_cmd {tbl cols method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE $tbl ([join $cols ,])"
+    }
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set cons [list]
+      set used [list]
+
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(op)=="eq" && $C(usable) && [lsearch $cons $C(column)]<0} {
+          lappend used use $i
+          lappend cons $C(column)
+        }
+      }
+
+      set nCons [llength $cons]
+      if {$nCons==0} {
+        return "cost 1000000 rows 1000000"
+      } else {
+        set cost [expr (11-$nCons) * 1000]
+        set ret [concat $used "cost $cost rows $cost"]
+
+        set txt [list]
+        foreach c $cons { lappend txt "[lindex $cols $c]=?" }
+        lappend ret idxstr "indexed([join $txt { AND }])"
+
+        return $ret
+      }
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd t1 {a b}");
+  CREATE VIRTUAL TABLE t2 USING tcl("vtab_cmd t2 {c d}");
+  CREATE VIRTUAL TABLE t3 USING tcl("vtab_cmd t3 {e f}");
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a='abc'
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=?)}
+}
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a='abc' AND b='def'
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=? AND b=?)}
+}
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a='abc' AND a='def'
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:indexed(a=?)}
+}
+do_eqp_test 1.4 {
+  SELECT * FROM t1,t2 WHERE c=a
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:} 
+  0 1 1 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)}
+}
+
+do_eqp_test 1.5 {
+  SELECT * FROM t1, t2 CROSS JOIN t3 WHERE t2.c = +t1.b AND t3.e=t2.d
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:} 
+  0 1 1 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
+  0 2 2 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
+}
+
+do_eqp_test 1.6 {
+  SELECT * FROM t1, t2, t3 WHERE t2.c = +t1.b AND t3.e = t2.d
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:} 
+  0 1 1 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
+  0 2 2 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
+}
+
+do_execsql_test 1.7.1 {
+  CREATE TABLE x1(a, b);
+}
+do_eqp_test 1.7.2 {
+  SELECT * FROM x1 CROSS JOIN t1, t2, t3 
+    WHERE t1.a = t2.c AND t1.b = t3.e
+} {
+  0 0 0 {SCAN TABLE x1} 
+  0 1 1 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:}
+  0 2 2 {SCAN TABLE t2 VIRTUAL TABLE INDEX 0:indexed(c=?)} 
+  0 3 3 {SCAN TABLE t3 VIRTUAL TABLE INDEX 0:indexed(e=?)}
+}
+
+finish_test
diff --git a/test/bestindex3.test b/test/bestindex3.test
new file mode 100644
index 00000000..8baf8542
--- /dev/null
+++ b/test/bestindex3.test
@@ -0,0 +1,175 @@
+# 2016 May 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex3
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for a virtual table named $tbl.
+#
+# The table created is:
+#
+#      "CREATE TABLE t1 (a, b, c)"
+#
+# This virtual table supports both LIKE and = operators on all columns.
+#  
+proc vtab_cmd {bOmit method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set ret [list]
+      set use use
+      if {$bOmit} {set use omit}
+
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(usable) && ($C(op)=="like" || $C(op)=="eq")} {
+          lappend ret $use $i
+          lappend ret idxstr 
+          lappend ret "[lindex {a b c} $C(column)] [string toupper $C(op)] ?"
+          break
+        }
+      }
+
+      if {$ret==""} {
+        lappend ret cost 1000000 rows 1000000
+      } else {
+        lappend ret cost 100 rows 10
+      }
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr param} $args {}
+      set where ""
+      if {$bOmit && $idxstr != ""} {
+        set where " WHERE [string map [list ? '$param' EQ =] $idxstr]"
+      }
+      return [list sql "SELECT rowid, * FROM ttt$where"]
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd 0");
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a LIKE 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a = 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a = 'abc' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_execsql_test 1.5 {
+  CREATE TABLE ttt(a, b, c);
+
+  INSERT INTO ttt VALUES(1, 'two',   'three');
+  INSERT INTO ttt VALUES(2, 'one',   'two');
+  INSERT INTO ttt VALUES(3, 'three', 'one');
+  INSERT INTO ttt VALUES(4, 'y',     'one');
+  INSERT INTO ttt VALUES(5, 'x',     'two');
+  INSERT INTO ttt VALUES(6, 'y',     'three');
+}
+
+foreach omit {0 1} {
+  do_execsql_test 1.6.$omit.0 "
+    DROP TABLE t1;
+    CREATE VIRTUAL TABLE t1 USING tcl('vtab_cmd $omit');
+  "
+  do_execsql_test 1.6.$omit.1 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%'
+  } {3 4}
+
+  do_execsql_test 1.6.$omit.2 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%' OR b='y'
+  } {3 4 6}
+
+  do_execsql_test 1.6.$omit.3 { 
+    SELECT rowid FROM t1 WHERE c = 'three' OR c LIKE 'o%'
+  } {1 6 3 4}
+}
+
+#-------------------------------------------------------------------------
+# Test the same pattern works with ordinary tables.
+#
+# This test does not work if the ICU extension is enabled. ICU overrides
+# LIKE - and this optimization only works with the built-in LIKE function.
+#
+ifcapable !icu {
+  do_execsql_test 2.1 {
+    CREATE TABLE t2(x TEXT COLLATE nocase, y TEXT);
+    CREATE INDEX t2x ON t2(x COLLATE nocase);
+    CREATE INDEX t2y ON t2(y);
+  }
+
+  do_eqp_test 2.2 {
+    SELECT * FROM t2 WHERE x LIKE 'abc%' OR y = 'def'
+  } {
+    0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x>? AND x= 5 AND x() <= 5"   1  2
   4   5  "x() BETWEEN 5 AND 5"     1  1
+
+  5   9  "(x(),8) >= (9,7) AND (x(),8)<=(9,10)"  1 2
+  6   9  "(x(),8) BETWEEN (9,7) AND (9,10)"      1 1
 } {
   do_test e_expr-13.1.$tn {
     set ::xcount 0
@@ -1809,7 +1812,7 @@ do_expr_test e_expr-35.1.6 {
 # The following block tests that errors are returned in a bunch of cases
 # where a subquery returns more than one column.
 #
-set M {only a single result allowed for a SELECT that is part of an expression}
+set M {/1 {sub-select returns [23] columns - expected 1}/}
 foreach {tn sql} {
   1     { SELECT (SELECT * FROM t2 UNION SELECT a+1, b+1 FROM t2) }
   2     { SELECT (SELECT * FROM t2 UNION SELECT a+1, b+1 FROM t2 ORDER BY 1) }
@@ -1818,7 +1821,7 @@ foreach {tn sql} {
   5     { SELECT (SELECT * FROM t2) }
   6     { SELECT (SELECT * FROM (SELECT 1, 2, 3)) }
 } {
-  do_catchsql_test e_expr-35.2.$tn $sql [list 1 $M]
+  do_catchsql_test e_expr-35.2.$tn $sql $M
 }
 
 # EVIDENCE-OF: R-35764-28041 The result of the expression is the value
diff --git a/test/e_fkey.test b/test/e_fkey.test
index 09756505..99f1ae8e 100644
--- a/test/e_fkey.test
+++ b/test/e_fkey.test
@@ -52,11 +52,11 @@ ifcapable trigger&&foreignkey {
 #-------------------------------------------------------------------------
 # Test the effects of defining OMIT_TRIGGER but not OMIT_FOREIGN_KEY.
 #
-# EVIDENCE-OF: R-44697-61543 If SQLITE_OMIT_TRIGGER is defined but
+# EVIDENCE-OF: R-10109-20452 If SQLITE_OMIT_TRIGGER is defined but
 # SQLITE_OMIT_FOREIGN_KEY is not, then SQLite behaves as it did prior to
-# version 3.6.19 - foreign key definitions are parsed and may be queried
-# using PRAGMA foreign_key_list, but foreign key constraints are not
-# enforced.
+# version 3.6.19 (2009-10-14) - foreign key definitions are parsed and
+# may be queried using PRAGMA foreign_key_list, but foreign key
+# constraints are not enforced.
 #
 # Specifically, test that "PRAGMA foreign_keys" is a no-op in this case.
 # When using the pragma to query the current setting, 0 rows are returned.
diff --git a/test/e_uri.test b/test/e_uri.test
index 7a7f2559..95b53f1d 100644
--- a/test/e_uri.test
+++ b/test/e_uri.test
@@ -13,7 +13,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix e_uri
-
+do_not_use_codec
 db close
 
 proc parse_uri {uri} {
diff --git a/test/e_vacuum.test b/test/e_vacuum.test
index 99b31aac..3fd34afd 100644
--- a/test/e_vacuum.test
+++ b/test/e_vacuum.test
@@ -159,37 +159,39 @@ do_test e_vacuum-1.3.1.2 {
   execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
 } {1024 1}
 
-# EVIDENCE-OF: R-08570-19916 However, when not in write-ahead log mode,
-# the page_size and/or auto_vacuum properties of an existing database
-# may be changed by using the page_size and/or pragma auto_vacuum
-# pragmas and then immediately VACUUMing the database.
-#
-do_test e_vacuum-1.3.2.1 {
-  execsql { PRAGMA journal_mode = delete }
-  execsql { PRAGMA page_size = 2048 }
-  execsql { PRAGMA auto_vacuum = NONE }
-  execsql VACUUM
-  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
-} {2048 0}
-
-# EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the
-# auto_vacuum support property can be changed using VACUUM.
-#
-ifcapable wal {
-do_test e_vacuum-1.3.3.1 {
-  execsql { PRAGMA journal_mode = wal }
-  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
-} {2048 0}
-do_test e_vacuum-1.3.3.2 {
-  execsql { PRAGMA page_size = 1024 }
-  execsql { PRAGMA auto_vacuum = FULL }
-  execsql VACUUM
-  execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
-} {2048 1}
+if {![nonzero_reserved_bytes]} {
+  # EVIDENCE-OF: R-08570-19916 However, when not in write-ahead log mode,
+  # the page_size and/or auto_vacuum properties of an existing database
+  # may be changed by using the page_size and/or pragma auto_vacuum
+  # pragmas and then immediately VACUUMing the database.
+  #
+  do_test e_vacuum-1.3.2.1 {
+    execsql { PRAGMA journal_mode = delete }
+    execsql { PRAGMA page_size = 2048 }
+    execsql { PRAGMA auto_vacuum = NONE }
+    execsql VACUUM
+    execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
+  } {2048 0}
+  
+  # EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the
+  # auto_vacuum support property can be changed using VACUUM.
+  #
+  if {[wal_is_capable]} {
+    do_test e_vacuum-1.3.3.1 {
+      execsql { PRAGMA journal_mode = wal }
+      execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
+    } {2048 0}
+    do_test e_vacuum-1.3.3.2 {
+      execsql { PRAGMA page_size = 1024 }
+      execsql { PRAGMA auto_vacuum = FULL }
+      execsql VACUUM
+      execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
+    } {2048 1}
+  }
 }
-
-# EVIDENCE-OF: R-38001-03952 VACUUM only works on the main database. It
-# is not possible to VACUUM an attached database file.
+  
+# EVIDENCE-OF: R-55119-57913 By default, VACUUM only works only on the
+# main database.
 forcedelete test.db2
 create_db { PRAGMA auto_vacuum = NONE }
 do_execsql_test e_vacuum-2.1.1 {
@@ -200,14 +202,15 @@ do_execsql_test e_vacuum-2.1.1 {
 } {}
 set original_size [file size test.db2]
 
-# Try everything we can think of to get the aux database vacuumed:
+# Vacuuming the main database does not affect aux
 do_execsql_test e_vacuum-2.1.3 { VACUUM } {}
-do_execsql_test e_vacuum-2.1.4 { VACUUM aux } {}
-do_execsql_test e_vacuum-2.1.5 { VACUUM 'test.db2' } {}
-
-# Despite our efforts, space in the aux database has not been reclaimed:
 do_test e_vacuum-2.1.6 { expr {[file size test.db2]==$::original_size} } 1
 
+# EVIDENCE-OF: R-36598-60500 Attached databases can be vacuumed by
+# appending the appropriate schema-name to the VACUUM statement.
+do_execsql_test e_vacuum-2.1.7 { VACUUM aux; } {}
+do_test e_vacuum-2.1.8 { expr {[file size test.db2]<$::original_size} } 1
+
 # EVIDENCE-OF: R-17495-17419 The VACUUM command may change the ROWIDs of
 # entries in any tables that do not have an explicit INTEGER PRIMARY
 # KEY.
@@ -269,9 +272,9 @@ do_test e_vacuum-3.2.2.1 {
 } {1 {cannot VACUUM - SQL statements in progress}}
 
 
-# EVIDENCE-OF: R-38735-12540 As of SQLite version 3.1, an alternative to
-# using the VACUUM command to reclaim space after data has been deleted
-# is auto-vacuum mode, enabled using the auto_vacuum pragma.
+# EVIDENCE-OF: R-55138-13241 An alternative to using the VACUUM command
+# to reclaim space after data has been deleted is auto-vacuum mode,
+# enabled using the auto_vacuum pragma.
 #
 do_test e_vacuum-3.3.1 {
   create_db { PRAGMA auto_vacuum = FULL }
diff --git a/test/e_walauto.test b/test/e_walauto.test
index 093b13f9..7665b1bf 100644
--- a/test/e_walauto.test
+++ b/test/e_walauto.test
@@ -24,6 +24,11 @@ if {$tcl_platform(os) == "OpenBSD"} {
   return
 }
 
+# This module uses hard-coded offsets which do not work if the reserved_bytes
+# value is nonzero.
+if {[nonzero_reserved_bytes]} {finish_test; return;}
+
+
 proc read_nbackfill {} {
   seek $::shmfd 96
   binary scan [read $::shmfd 4] n nBackfill
diff --git a/test/e_walhook.test b/test/e_walhook.test
index c8c88194..c963524a 100644
--- a/test/e_walhook.test
+++ b/test/e_walhook.test
@@ -161,10 +161,10 @@ do_test 5.2 {
 
 
 
-# EVIDENCE-OF: R-42842-27162 Note that the sqlite3_wal_autocheckpoint()
+# EVIDENCE-OF: R-57445-43425 Note that the sqlite3_wal_autocheckpoint()
 # interface and the wal_autocheckpoint pragma both invoke
-# sqlite3_wal_hook() and will those overwrite any prior
-# sqlite3_wal_hook() settings.
+# sqlite3_wal_hook() and will overwrite any prior sqlite3_wal_hook()
+# settings.
 #
 set ::old_wal_hook 0
 proc my_old_wal_hook {args} { incr ::old_wal_hook ; return 0 }
diff --git a/test/eqp.test b/test/eqp.test
index 046088c9..c955a80c 100644
--- a/test/eqp.test
+++ b/test/eqp.test
@@ -516,45 +516,47 @@ det 5.13 "SELECT a FROM t1 EXCEPT SELECT d FROM t2 ORDER BY 1" {
 }
 
 
-#-------------------------------------------------------------------------
-# The following tests - eqp-6.* - test that the example C code on 
-# documentation page eqp.html works. The C code is duplicated in test1.c
-# and wrapped in Tcl command [print_explain_query_plan] 
-#
-set boilerplate {
-  proc explain_query_plan {db sql} {
-    set stmt [sqlite3_prepare_v2 db $sql -1 DUMMY]
-    print_explain_query_plan $stmt
-    sqlite3_finalize $stmt
+if {![nonzero_reserved_bytes]} {
+  #-------------------------------------------------------------------------
+  # The following tests - eqp-6.* - test that the example C code on 
+  # documentation page eqp.html works. The C code is duplicated in test1.c
+  # and wrapped in Tcl command [print_explain_query_plan] 
+  #
+  set boilerplate {
+    proc explain_query_plan {db sql} {
+      set stmt [sqlite3_prepare_v2 db $sql -1 DUMMY]
+      print_explain_query_plan $stmt
+      sqlite3_finalize $stmt
+    }
+    sqlite3 db test.db
+    explain_query_plan db {%SQL%}
+    db close
+    exit
   }
-  sqlite3 db test.db
-  explain_query_plan db {%SQL%}
-  db close
-  exit
-}
-
-# Do a "Print Explain Query Plan" test.
-proc do_peqp_test {tn sql res} {
-  set fd [open script.tcl w]
-  puts $fd [string map [list %SQL% $sql] $::boilerplate]
-  close $fd
-
-  uplevel do_test $tn [list {
-    set fd [open "|[info nameofexec] script.tcl"]
-    set data [read $fd]
+  
+  # Do a "Print Explain Query Plan" test.
+  proc do_peqp_test {tn sql res} {
+    set fd [open script.tcl w]
+    puts $fd [string map [list %SQL% $sql] $::boilerplate]
     close $fd
-    set data
-  }] [list $res]
-}
-
-do_peqp_test 6.1 {
-  SELECT a, b FROM t1 EXCEPT SELECT d, 99 FROM t2 ORDER BY 1
-} [string trimleft {
+  
+    uplevel do_test $tn [list {
+      set fd [open "|[info nameofexec] script.tcl"]
+      set data [read $fd]
+      close $fd
+      set data
+    }] [list $res]
+  }
+  
+  do_peqp_test 6.1 {
+    SELECT a, b FROM t1 EXCEPT SELECT d, 99 FROM t2 ORDER BY 1
+  } [string trimleft {
 1 0 0 SCAN TABLE t1 USING COVERING INDEX i2
 2 0 0 SCAN TABLE t2
 2 0 0 USE TEMP B-TREE FOR ORDER BY
 0 0 0 COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)
 }]
+}
 
 #-------------------------------------------------------------------------
 # The following tests - eqp-7.* - test that queries that use the OP_Count
diff --git a/test/exclusive.test b/test/exclusive.test
index c000dfef..45f93182 100644
--- a/test/exclusive.test
+++ b/test/exclusive.test
@@ -420,9 +420,10 @@ do_test exclusive-5.0 {
 } {}
 do_test exclusive-5.1 {
   # Three files are open: The db, journal and statement-journal.
+  # (2016-03-04) The statement-journal is now opened lazily
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 3 - ($TEMP_STORE>=2)]
+} {2}
 do_test exclusive-5.2 {
   execsql {
     COMMIT;
@@ -446,17 +447,19 @@ do_test exclusive-5.4 {
     INSERT INTO abc SELECT a+10, b+10, c+10 FROM abc;
   }
   # Three files are open: The db, journal and statement-journal.
+  # 2016-03-04: The statement-journal open is deferred
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 3 - ($TEMP_STORE>=2)]
+} {2}
 do_test exclusive-5.5 {
   execsql {
     COMMIT;
   }
   # Three files are still open: The db, journal and statement-journal.
+  # 2016-03-04: The statement-journal open is deferred
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 3 - ($TEMP_STORE>=2)]
+} {2}
 do_test exclusive-5.6 {
   execsql {
     PRAGMA locking_mode = normal;
diff --git a/test/exists.test b/test/exists.test
index fb73797d..5f9e36eb 100644
--- a/test/exists.test
+++ b/test/exists.test
@@ -19,6 +19,7 @@ source $testdir/lock_common.tcl
 
 
 foreach jm {rollback wal} {
+  if {![wal_is_capable] && $jm=="wal"} continue
 
   set testprefix exists-$jm
 
diff --git a/test/expr.test b/test/expr.test
index 7d7b8ce5..7a6d4772 100644
--- a/test/expr.test
+++ b/test/expr.test
@@ -308,6 +308,33 @@ ifcapable floatingpoint {if {[working_64bit_int]} {
   test_realnum_expr expr-1.257\
       {i1=-4294967296, i2=-2147483647} {i1*i2}    9223372032559808512
 
+  test_realnum_expr expr-1.260\
+      {i1=3037000500, i2=3037000500} {i1*i2}      9.22337203700025e+18
+  test_realnum_expr expr-1.261\
+      {i1=3037000500, i2=-3037000500} {i1*i2}     -9.22337203700025e+18
+  test_realnum_expr expr-1.262\
+      {i1=-3037000500, i2=3037000500} {i1*i2}     -9.22337203700025e+18
+  test_realnum_expr expr-1.263\
+      {i1=-3037000500, i2=-3037000500} {i1*i2}    9.22337203700025e+18
+
+  test_realnum_expr expr-1.264\
+      {i1=3037000500, i2=3037000499} {i1*i2}      9223372033963249500
+  test_realnum_expr expr-1.265\
+      {i1=3037000500, i2=-3037000499} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.266\
+      {i1=-3037000500, i2=3037000499} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.267\
+      {i1=-3037000500, i2=-3037000499} {i1*i2}    9223372033963249500
+
+  test_realnum_expr expr-1.268\
+      {i1=3037000499, i2=3037000500} {i1*i2}      9223372033963249500
+  test_realnum_expr expr-1.269\
+      {i1=3037000499, i2=-3037000500} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.270\
+      {i1=-3037000499, i2=3037000500} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.271\
+      {i1=-3037000499, i2=-3037000500} {i1*i2}    9223372033963249500
+
 }}
 
 ifcapable floatingpoint {
diff --git a/test/filectrl.test b/test/filectrl.test
index 28fecee9..cb469941 100644
--- a/test/filectrl.test
+++ b/test/filectrl.test
@@ -43,4 +43,30 @@ do_test filectrl-1.6 {
 } {/etilqs_/}
 db close
 forcedelete .test_control_lockproxy.db-conch test.proxy
+forcedelete test.db test2.db
+
+if {$tcl_platform(platform)=="windows"} {
+  do_test filectrl-2.1 {
+    sqlite3 db test2.db
+    set size [file size test2.db]
+    set handle [file_control_win32_get_handle db]
+    db close
+    forcedelete test2.db
+    list $size $handle [expr {$handle != 0}]
+  } {/^0 \{0 \d+\} 1$/}
+
+  do_test filectrl-2.2 {
+    sqlite3 db test2.db
+    execsql {
+      CREATE TABLE t1(x);
+      INSERT INTO t1 (x) VALUES(RANDOMBLOB(1048576));
+    }
+    set size [file size test2.db]
+    set handle [file_control_win32_get_handle db]
+    db close
+    forcedelete test2.db
+    list $size $handle [expr {$handle != 0}]
+  } {/^1\d+ \{0 \d+\} 1$/}
+}
+
 finish_test
diff --git a/test/filefmt.test b/test/filefmt.test
index 2df14244..087a955c 100644
--- a/test/filefmt.test
+++ b/test/filefmt.test
@@ -144,9 +144,11 @@ do_execsql_test filefmt-2.1.1 {
   CREATE TABLE t2(a);
   INSERT INTO t2 VALUES(1);
 } {}
-do_test filefmt-2.1.2 {
-  hexio_read test.db 28 4
-} {00000009}
+if {![nonzero_reserved_bytes]} {
+  do_test filefmt-2.1.2 {
+    hexio_read test.db 28 4
+  } {00000009}
+}
 
 do_test filefmt-2.1.3 {
   sql36231 { INSERT INTO t1 VALUES(a_string(3000)) }
@@ -170,9 +172,11 @@ do_execsql_test filefmt-2.2.1 {
   CREATE TABLE t2(a);
   INSERT INTO t2 VALUES(1);
 } {}
-do_test filefmt-2.2.2 {
-  hexio_read test.db 28 4
-} {00000009}
+if {![nonzero_reserved_bytes]} {
+  do_test filefmt-2.2.2 {
+    hexio_read test.db 28 4
+  } {00000009}
+}
 
 do_test filefmt-2.2.3 {
   sql36231 { INSERT INTO t1 VALUES(a_string(3000)) }
diff --git a/test/fkey6.test b/test/fkey6.test
index 6fc3de21..b658f20f 100644
--- a/test/fkey6.test
+++ b/test/fkey6.test
@@ -1,4 +1,4 @@
-# 2013-07-11
+# 2012 December 17
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
@@ -23,6 +23,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix fkey6
 
 ifcapable {!foreignkey} {
   finish_test
@@ -171,5 +172,58 @@ do_execsql_test fkey6-2.6 {
   PRAGMA defer_foreign_keys;
 } {0}
 
+#--------------------------------------------------------------------------
+# Test that defer_foreign_keys disables RESTRICT.
+#
+do_execsql_test 3.1 {
+  CREATE TABLE p2(a PRIMARY KEY, b);
+  CREATE TABLE c2(x, y REFERENCES p2 ON DELETE RESTRICT ON UPDATE RESTRICT);
+  INSERT INTO p2 VALUES(1, 'one');
+  INSERT INTO p2 VALUES(2, 'two');
+  INSERT INTO c2 VALUES('i', 1);
+}
+
+do_catchsql_test 3.2.1 {
+  BEGIN;
+    UPDATE p2 SET a=a-1;
+} {1 {FOREIGN KEY constraint failed}}
+do_execsql_test 3.2.2 { COMMIT }
+
+do_execsql_test 3.2.3 {
+  BEGIN;
+    PRAGMA defer_foreign_keys = 1;
+    UPDATE p2 SET a=a-1;
+  COMMIT;
+}
+
+do_execsql_test 3.2.4 {
+  BEGIN;
+    PRAGMA defer_foreign_keys = 1;
+    UPDATE p2 SET a=a-1;
+}
+do_catchsql_test 3.2.5 {
+  COMMIT;
+} {1 {FOREIGN KEY constraint failed}}
+do_execsql_test 3.2.6 { ROLLBACK }
+
+do_execsql_test 3.3.1 {
+  CREATE TRIGGER p2t AFTER DELETE ON p2 BEGIN
+    INSERT INTO p2 VALUES(old.a, 'deleted!');
+  END;
+}
+do_catchsql_test 3.3.2 {
+  BEGIN;
+    DELETE FROM p2 WHERE a=1;
+} {1 {FOREIGN KEY constraint failed}}
+do_execsql_test 3.3.3 { COMMIT }
+
+do_execsql_test 3.3.4 {
+  BEGIN;
+    PRAGMA defer_foreign_keys = 1;
+    DELETE FROM p2 WHERE a=1;
+  COMMIT;
+  SELECT * FROM p2;
+} {0 one 1 deleted!}
+
 
 finish_test
diff --git a/test/fts3atoken.test b/test/fts3atoken.test
index 2cdea79a..4ce38762 100644
--- a/test/fts3atoken.test
+++ b/test/fts3atoken.test
@@ -56,40 +56,41 @@ proc escape_string {str} {
 #
 #   5: Test that the table created to use tokenizer 'blah' is usable.
 #
-ifcapable fts3_tokenizer {
-  do_test fts3atoken-1.1 {
-    catchsql {
-      CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
-    }
-  } {1 {unknown tokenizer: blah}}
-  do_test fts3atoken-1.2 {
-    execsql {
-      SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
-    }
-  } {0}
-  do_test fts3atoken-1.3 {
-    execsql {
-      SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
-    }
-  } {1}
-  do_test fts3atoken-1.4 {
-    catchsql {
-      CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
-    }
-  } {0 {}}
-  do_test fts3atoken-1.5 {
-    execsql {
-      INSERT INTO t1(content) VALUES('There was movement at the station');
-      INSERT INTO t1(content) VALUES('For the word has passed around');
-      INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
-      SELECT content FROM t1 WHERE content MATCH 'movement'
-    }
-  } {{There was movement at the station}}
-} else {
-  do_catchsql_test 1.6 {
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
+do_test fts3atoken-1.1 {
+  catchsql {
+    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
+  }
+} {1 {unknown tokenizer: blah}}
+do_test fts3atoken-1.2 {
+  execsql {
     SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
-  } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}}
-}
+  }
+} {0}
+do_test fts3atoken-1.3 {
+  execsql {
+    SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
+  }
+} {1}
+do_test fts3atoken-1.4 {
+  catchsql {
+    CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
+  }
+} {0 {}}
+do_test fts3atoken-1.5 {
+  execsql {
+    INSERT INTO t1(content) VALUES('There was movement at the station');
+    INSERT INTO t1(content) VALUES('For the word has passed around');
+    INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
+    SELECT content FROM t1 WHERE content MATCH 'movement'
+  }
+} {{There was movement at the station}}
+
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0
+do_catchsql_test 1.6 {
+  SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
+} {1 {fts3tokenize disabled}}
+
 
 #--------------------------------------------------------------------------
 # Test cases fts3atoken-2.* test error cases in the scalar function based
@@ -212,14 +213,14 @@ do_catchsql_test 6.1.3 {
 do_catchsql_test 6.2.1 {
   SELECT fts3_tokenizer(NULL);
 } {1 {unknown tokenizer: }}
-ifcapable fts3_tokenizer {
-  do_catchsql_test 6.2.2 {
-    SELECT fts3_tokenizer(NULL, X'1234567812345678');
-  } {1 {argument type mismatch}}
-  do_catchsql_test 6.2.3 {
-    SELECT fts3_tokenizer(NULL, X'12345678');
-  } {1 {argument type mismatch}}
-}
+
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
+do_catchsql_test 6.2.2 {
+  SELECT fts3_tokenizer(NULL, X'1234567812345678');
+} {1 {argument type mismatch}}
+do_catchsql_test 6.2.3 {
+  SELECT fts3_tokenizer(NULL, X'12345678');
+} {1 {argument type mismatch}}
 
 
 finish_test
diff --git a/test/fts3aux1.test b/test/fts3aux1.test
index d17ac85d..b6111f84 100644
--- a/test/fts3aux1.test
+++ b/test/fts3aux1.test
@@ -117,12 +117,12 @@ do_test 2.1.2.1 {
   set cnt 0
   execsql { SELECT * FROM terms_v WHERE rec('cnt', term) AND term='braid' }
   set cnt
-} {2}
+} {1}
 do_test 2.1.2.2 {
   set cnt 0
   execsql { SELECT * FROM terms_v WHERE rec('cnt', term) AND +term='braid' }
   set cnt
-} {38}
+} {19}
 
 # Similar to the test immediately above, but using a term ("breakfast") that 
 # is not featured in the dataset.
@@ -136,7 +136,7 @@ do_test 2.1.3.2 {
   set cnt 0
   execsql { SELECT * FROM terms_v WHERE rec('cnt', term) AND +term='breakfast' }
   set cnt
-} {38}
+} {19}
 
 do_execsql_test 2.1.4.1 { SELECT * FROM terms_v WHERE term='braid' } {braid 1 1}
 do_execsql_test 2.1.4.2 { SELECT * FROM terms_v WHERE +term='braid'} {braid 1 1}
diff --git a/test/fts3conf.test b/test/fts3conf.test
index f9698770..6766a954 100644
--- a/test/fts3conf.test
+++ b/test/fts3conf.test
@@ -211,4 +211,42 @@ do_execsql_test 4.2.2 {
   INSERT INTO t01(t01) VALUES('integrity-check');
 } {}
 
+do_execsql_test 4.3.1 {
+  CREATE VIRTUAL TABLE t02 USING fts4;
+  INSERT INTO t01 VALUES('1 1 1');
+  INSERT INTO t02 VALUES('2 2 2');
+  BEGIN;
+    SAVEPOINT abc;
+      INSERT INTO t01 VALUES('a b c');
+      INSERT INTO t02 VALUES('a b c');
+    ROLLBACK TO abc;
+  COMMIT;
+}
+do_execsql_test 4.3.2 {
+  SELECT * FROM t01 WHERE t01 MATCH 'b';
+  INSERT INTO t01(t01) VALUES('integrity-check');
+} {}
+
+do_execsql_test 4.4.1 {
+  CREATE TABLE A(ID INTEGER PRIMARY KEY, AnotherID INTEGER, Notes TEXT);
+  CREATE VIRTUAL TABLE AFTS USING FTS4 (Notes);
+  CREATE TRIGGER A_DeleteTrigger AFTER DELETE ON A FOR EACH ROW BEGIN 
+    DELETE FROM AFTS WHERE rowid=OLD.ID; 
+  END;
+  CREATE TABLE B(ID INTEGER PRIMARY KEY,Notes TEXT);
+  CREATE VIRTUAL TABLE BFTS USING FTS3 (Notes);
+  CREATE TRIGGER B_DeleteTrigger AFTER DELETE ON B FOR EACH ROW BEGIN 
+    DELETE FROM BFTS WHERE rowid=OLD.ID; 
+  END;
+}
+		
+do_execsql_test 4.4.2 {
+  BEGIN TRANSACTION;
+    DELETE FROM A WHERE AnotherID=1;
+    DELETE FROM B WHERE ID=1;
+  COMMIT;
+}
+		
+
+
 finish_test
diff --git a/test/fts3expr.test b/test/fts3expr.test
index 6e23faf6..b186a157 100644
--- a/test/fts3expr.test
+++ b/test/fts3expr.test
@@ -514,4 +514,8 @@ do_test fts3expr-9.1 {
   test_fts3expr "f (e NEAR/2 a)"
 } {AND {PHRASE 3 0 f} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}}
 
+do_test fts3expr-10.1 { test_fts3expr "abc *" } {PHRASE 3 0 abc}
+do_test fts3expr-10.2 { test_fts3expr "*" } {}
+do_test fts3expr-10.3 { test_fts3expr "abc*" } {PHRASE 3 0 abc+}
+
 finish_test
diff --git a/test/fts4growth.test b/test/fts4growth.test
index e4b5f19e..4dc22124 100644
--- a/test/fts4growth.test
+++ b/test/fts4growth.test
@@ -59,11 +59,11 @@ do_test 1.4 {
     INSERT INTO x1(x1) VALUES('merge=4,4');
     SELECT level, end_block, length(root) FROM x1_segdir;
   }
-} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2}
+} {1 {224 921} 2}
 
 do_execsql_test 1.5 {
   SELECT length(block) FROM x1_segments;
-} {658 {}}
+} {921 {}}
 
 do_test 1.6 {
   foreach L {
@@ -71,27 +71,33 @@ do_test 1.6 {
     {That perched above Dead Man's Creek, beside the mountain road.}
     {He turned the cycle down the hill and mounted for the fray,}
     {But 'ere he'd gone a dozen yards it bolted clean away.}
+
     {It left the track, and through the trees, just like a silver steak,}
     {It whistled down the awful slope towards the Dead Man's Creek.}
     {It shaved a stump by half an inch, it dodged a big white-box:}
     {The very wallaroos in fright went scrambling up the rocks,}
+
     {The wombats hiding in their caves dug deeper underground,}
     {As Mulga Bill, as white as chalk, sat tight to every bound.}
     {It struck a stone and gave a spring that cleared a fallen tree,}
     {It raced beside a precipice as close as close could be;}
+
     {And then as Mulga Bill let out one last despairing shriek}
     {It made a leap of twenty feet into the Dead Man's Creek.}
+    {It shaved a stump by half an inch, it dodged a big white-box:}
+    {The very wallaroos in fright went scrambling up the rocks,}
+    {The wombats hiding in their caves dug deeper underground,}
   } {
     execsql { INSERT INTO x1 VALUES($L) }
   }
   execsql { 
     SELECT level, end_block, length(root) FROM x1_segdir;
   }
-} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117}
+} {1 {224 921} 2 1 {226 1230} 7 0 {0 98} 98}
 
 do_execsql_test 1.7 {
-  SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130);
-} {1377}
+  SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (224,225,226)
+} {1230}
 
 #-------------------------------------------------------------------------
 #
@@ -131,24 +137,24 @@ do_execsql_test 2.5 {
   SELECT end_block FROM x2_segdir WHERE level=3;
   INSERT INTO x2(x2) VALUES('merge=4,4');
   SELECT end_block FROM x2_segdir WHERE level=3;
-} {{3828 -3430} {3828 -10191} {3828 -14109}}
+} {{5588 -3950} {5588 -11766} {5588 -15541}}
 
 do_execsql_test 2.6 {
   SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE 
     blockid BETWEEN start_block AND leaves_end_block
     AND level=3
-} {14109}
+} {15541}
 
 do_execsql_test 2.7 { 
   INSERT INTO x2(x2) VALUES('merge=1000,4');
   SELECT end_block FROM x2_segdir WHERE level=3;
-} {{3828 86120}}
+} {{5588 127563}}
 
 do_execsql_test 2.8 {
   SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE 
     blockid BETWEEN start_block AND leaves_end_block
     AND level=3
-} {86120}
+} {127563}
 
 #--------------------------------------------------------------------------
 # Test that delete markers are removed from FTS segments when possible.
@@ -391,7 +397,7 @@ do_execsql_test 7.2 {
 } {
   0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
   0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
-  1 0 {16014 -51226}
+  1 0 {23694 -69477}
 }
 
 do_execsql_test 7.3 {
@@ -400,7 +406,7 @@ do_execsql_test 7.3 {
 } {
   0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
   0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
-  1 0 16014
+  1 0 23694
 }
 
 do_execsql_test 7.4 {
@@ -409,28 +415,26 @@ do_execsql_test 7.4 {
 } {
   0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 
   0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006}
-  1 0 16014
+  1 0 23694
 }
 
 do_execsql_test 7.5 {
   INSERT INTO x6(x6) VALUES('merge=2500,4');
-  SELECT level, idx, end_block FROM x6_segdir;
+  SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
 } {
-  0 0 {598 118006} 0 1 {718 118006} 1 0 16014
+  1 0 719 1171 23694
 }
 
 do_execsql_test 7.6 {
   INSERT INTO x6(x6) VALUES('merge=2500,2');
   SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir;
 } {
-  2 0 23695 24147 {41262 633507}
+  1 0 719 1171 23694
 }
 
 do_execsql_test 7.7 {
   SELECT sum(length(block)) FROM x6_segments 
-  WHERE blockid BETWEEN 23695 AND 24147
-} {633507}
-
+} {635247}
 
 
 finish_test
diff --git a/test/fts4langid.test b/test/fts4langid.test
index a3059931..fdb1876b 100644
--- a/test/fts4langid.test
+++ b/test/fts4langid.test
@@ -358,31 +358,30 @@ proc build_multilingual_db_2 {db} {
   }
 }
 
-ifcapable fts3_tokenizer {
-  do_test 4.1.0 {
-    reset_db
-    set ptr [fts3_test_tokenizer]
-    execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
-    build_multilingual_db_2 db
-  } {}
-  do_execsql_test 4.1.1 {
-    SELECT docid FROM t4 WHERE t4 MATCH 'quick';
-  } {0}
-  do_execsql_test 4.1.2 {
-    SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
-  } {}
-  do_execsql_test 4.1.3 {
-    SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
-  } {1}
-  for {set i 0} {$i < 50} {incr i} {
-    do_execsql_test 4.1.4.$i {
-      SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
-    } [expr 0==($i%2)]
-  }
-  do_catchsql_test 4.1.5 {
-    INSERT INTO t4(content, lid) VALUES('hello world', 101)
-  } {1 {SQL logic error or missing database}}
+do_test 4.1.0 {
+  reset_db
+  set ptr [fts3_test_tokenizer]
+  sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
+  execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
+  build_multilingual_db_2 db
+} {}
+do_execsql_test 4.1.1 {
+  SELECT docid FROM t4 WHERE t4 MATCH 'quick';
+} {0}
+do_execsql_test 4.1.2 {
+  SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
+} {}
+do_execsql_test 4.1.3 {
+  SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
+} {1}
+for {set i 0} {$i < 50} {incr i} {
+  do_execsql_test 4.1.4.$i {
+    SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
+  } [expr 0==($i%2)]
 }
+do_catchsql_test 4.1.5 {
+  INSERT INTO t4(content, lid) VALUES('hello world', 101)
+} {1 {SQL logic error or missing database}}
 
 #-------------------------------------------------------------------------
 # Test cases 5.*
@@ -482,6 +481,6 @@ foreach lid [list 4 [expr 1<<30]] {
   do_execsql_test 5.4.$lid.5 {
     SELECT count(*) FROM t6_segdir;
     SELECT count(*) FROM t6_segments;
-  } {4 4}
+  } {1 2}
 }
 finish_test
diff --git a/test/fts4merge.test b/test/fts4merge.test
index fabb651e..eac18311 100644
--- a/test/fts4merge.test
+++ b/test/fts4merge.test
@@ -55,8 +55,6 @@ foreach mod {fts3 fts4} {
   do_execsql_test 1.3 { 
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
   } {
-    0 {0 1 2 3} 
-    1 {0 1 2 3 4 5 6} 
     2 {0 1 2 3}
   }
   
@@ -71,7 +69,6 @@ foreach mod {fts3 fts4} {
   do_execsql_test 1.5 { 
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
   } {
-    2 {0 1}
     3 0
   }
   
@@ -89,7 +86,6 @@ foreach mod {fts3 fts4} {
     5   {merge=6,%}
     6   {merge=6,six}
     7   {merge=6,1}
-    8   {merge=6,0}
   } {
     do_catchsql_test 2.$tn { 
       INSERT INTO t2(t2) VALUES($arg);
@@ -119,11 +115,7 @@ foreach mod {fts3 fts4} {
     INSERT INTO t2(t2) VALUES('merge=1000000,2');
     SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
   } {
-    0 0 
-    2 0
-    3 0 
     4 0
-    6 0
   }
   
   #-------------------------------------------------------------------------
@@ -203,28 +195,27 @@ foreach mod {fts3 fts4} {
     INSERT INTO t1(t1) VALUES('merge=1,5');
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
   } {
-    0 {0 1 2}
     1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 
     2 {0 1 2 3}
   }
   
-  do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
+  do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'010F'}
   do_test 5.5 {
     foreach docid [execsql {SELECT docid FROM t1}] {
       execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
     }
   } {}
   
-  do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
+  do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'010F'}
   
   do_execsql_test 5.7 {
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
     SELECT quote(value) from t1_stat WHERE rowid=1;
   } {
-    0 {0 1 2 3 4 5 6 7 8 9 10} 
+    0 {0 1 2 3 4 5 6 7} 
     1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 
-    2 {0 1 2 3 4 5 6 7}
-    X'0105'
+    2 {0 1 2 3 4 5 6 7} 
+    X'010F'
   }
   
   do_execsql_test 5.8 {
@@ -233,9 +224,8 @@ foreach mod {fts3 fts4} {
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
     SELECT quote(value) from t1_stat WHERE rowid=1;
   } {
-    0 {0 1 2 3 4} 
     1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
-    2 {0 1 2 3 4 5 6 7 8} X'0106'
+    2 {0 1 2 3 4 5 6 7 8} X'010E'
   }
   
   do_test 5.8.1 { fts3_integrity_check t1 } ok
@@ -253,7 +243,7 @@ foreach mod {fts3 fts4} {
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
     SELECT quote(value) from t1_stat WHERE rowid=1;
   } {
-    0 0 1 {0 1} 2 0 3 0 X'0106'
+    0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0 2 0 3 0 X'010E'
   }
   
   do_execsql_test 5.11 {
@@ -261,7 +251,7 @@ foreach mod {fts3 fts4} {
     SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
     SELECT quote(value) from t1_stat WHERE rowid=1;
   } {
-    0 0 1 {0 1} 2 0 3 0 X''
+    1 {0 1} 2 0 3 0 X'010E'
   }
   
   #-------------------------------------------------------------------------
diff --git a/test/fts4merge3.test b/test/fts4merge3.test
index 329b4d2c..08b68b97 100644
--- a/test/fts4merge3.test
+++ b/test/fts4merge3.test
@@ -62,7 +62,7 @@ do_all_bc_test {
 
     do_test 1.7 { sql2 { 
       SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
-    } } [list  0 1  2 18  3 5]
+    } } {2 15 3 5}
 
     # Using the old connection, insert many rows. 
     do_test 1.8 {
@@ -73,7 +73,7 @@ do_all_bc_test {
 
     do_test 1.9 { sql2 { 
       SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
-    } } [list  0 13  1 13  2 5  3 6]
+    } } [list  0 12  1 13  2 4  3 6]
 
     # Run a big incr-merge operation on the db.
     do_test 1.10 { sql1 { INSERT INTO t2(t2) VALUES('merge=2000,2') } } {}
@@ -97,7 +97,7 @@ do_all_bc_test {
 
     do_test 1.15 { sql2 { 
       SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
-    } } {6 1}
+    } } {4 1}
   }
 }
 
diff --git a/test/fts4opt.test b/test/fts4opt.test
new file mode 100644
index 00000000..0d0ed9f4
--- /dev/null
+++ b/test/fts4opt.test
@@ -0,0 +1,213 @@
+# 2016 March 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/fts3_common.tcl
+set ::testprefix fts4opt
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+# Create the fts_kjv_genesis procedure which fills and FTS3/4 table 
+# with the complete text of the Book of Genesis.
+#
+source $testdir/genesis.tcl
+
+do_execsql_test 1.0 { CREATE TABLE t1(docid, words) }
+fts_kjv_genesis
+
+#-------------------------------------------------------------------------
+# Argument $db is an open database handle. $tbl is the name of an FTS3/4
+# table with the database. This command rearranges the contents of the
+# %_segdir table so that all segments within each index are on the same
+# level. This means that the 'merge' command can then be used for an
+# incremental optimize routine.
+#
+proc prepare_for_optimize {db tbl} {
+  $db eval [string map [list % $tbl] {
+    BEGIN;
+      CREATE TEMP TABLE tmp_segdir(
+        level, idx, start_block, leaves_end_block, end_block, root
+      );
+
+      INSERT INTO temp.tmp_segdir 
+        SELECT 
+        1024*(o.level / 1024) + 32,                                -- level
+        sum(o.leveli.idx)), -- idx
+        o.start_block, o.leaves_end_block, o.end_block, o.root     -- other
+        FROM %_segdir o, %_segdir i 
+        WHERE (o.level / 1024) = (i.level / 1024)
+        GROUP BY o.level, o.idx;
+  
+      DELETE FROM %_segdir;
+      INSERT INTO %_segdir SELECT * FROM temp.tmp_segdir;
+      DROP TABLE temp.tmp_segdir;
+  
+    COMMIT;
+  }]
+}
+
+do_test 1.1 {
+  execsql { CREATE VIRTUAL TABLE t2 USING fts4(words, prefix="1,2,3") }
+  foreach {docid words} [db eval { SELECT * FROM t1 }] {
+    execsql { INSERT INTO t2(docid, words) VALUES($docid, $words) }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  SELECT level, count(*) FROM t2_segdir GROUP BY level
+} {
+  0    13    1 15    2 5 
+  1024 13 1025 15 1026 5 
+  2048 13 2049 15 2050 5 
+  3072 13 3073 15 3074 5
+}
+
+do_execsql_test 1.3 { INSERT INTO t2(t2) VALUES('integrity-check') }
+prepare_for_optimize db t2
+do_execsql_test 1.4 { INSERT INTO t2(t2) VALUES('integrity-check') }
+
+do_execsql_test 1.5 {
+  SELECT level, count(*) FROM t2_segdir GROUP BY level
+} {
+  32   33 
+  1056 33 
+  2080 33 
+  3104 33
+}
+
+do_test 1.6 {
+  while 1 {
+    set tc1 [db total_changes]
+    execsql { INSERT INTO t2(t2) VALUES('merge=5,2') }
+    set tc2 [db total_changes]
+    if {($tc2 - $tc1) < 2} break
+  }
+  execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level }
+} {33 1 1057 1 2081 1 3105 1}
+do_execsql_test 1.7 { INSERT INTO t2(t2) VALUES('integrity-check') }
+
+do_execsql_test 1.8 {
+  INSERT INTO t2(words) SELECT words FROM t1;
+  SELECT level, count(*) FROM t2_segdir GROUP BY level;
+} {0 2 1024 2 2048 2 3072 2}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  DELETE FROM t2;
+}
+do_test 2.1 {
+  foreach {docid words} [db eval { SELECT * FROM t1 }] {
+    execsql { INSERT INTO t2(docid, words) VALUES($docid, $words) }
+  }
+
+  set i 0
+  foreach {docid words} [db eval { SELECT * FROM t1 }] {
+    if {[incr i] % 2} { execsql { DELETE FROM t2 WHERE docid = $docid } }
+  }
+
+  set i 0
+  foreach {docid words} [db eval { SELECT * FROM t1 }] {
+    if {[incr i] % 3} {
+      execsql { INSERT OR REPLACE INTO t2(docid, words) VALUES($docid, $words) }
+    }
+  }
+} {}
+
+do_execsql_test 2.2 {
+  SELECT level, count(*) FROM t2_segdir GROUP BY level
+} {
+  0    10    1 15    2 12 
+  1024 10 1025 15 1026 12 
+  2048 10 2049 15 2050 12 
+  3072 10 3073 15 3074 12
+}
+
+do_execsql_test 2.3 { INSERT INTO t2(t2) VALUES('integrity-check') }
+prepare_for_optimize db t2
+do_execsql_test 2.4 { INSERT INTO t2(t2) VALUES('integrity-check') }
+
+do_execsql_test 2.5 {
+  SELECT level, count(*) FROM t2_segdir GROUP BY level
+} {
+    32 37 
+  1056 37 
+  2080 37 
+  3104 37
+}
+
+do_test 2.6 {
+  while 1 {
+    set tc1 [db total_changes]
+    execsql { INSERT INTO t2(t2) VALUES('merge=5,2') }
+    set tc2 [db total_changes]
+    if {($tc2 - $tc1) < 2} break
+  }
+  execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level }
+} {33 1 1057 1 2081 1 3105 1}
+do_execsql_test 2.7 { INSERT INTO t2(t2) VALUES('integrity-check') }
+
+do_execsql_test 2.8 {
+  INSERT INTO t2(words) SELECT words FROM t1;
+  SELECT level, count(*) FROM t2_segdir GROUP BY level;
+} {0 2 1024 2 2048 2 3072 2}
+
+#-------------------------------------------------------------------------
+# Check that 'optimize' works when there is data in the in-memory hash
+# table, but no segments at all on disk.
+#
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE fts USING fts4 (t);
+  INSERT INTO fts (fts) VALUES ('optimize');
+}
+do_execsql_test 3.2 {
+  INSERT INTO fts(fts) VALUES('integrity-check');
+  SELECT count(*) FROM fts_segdir;
+} {0}
+do_execsql_test 3.3 {
+  BEGIN;
+  INSERT INTO fts (rowid, t) VALUES (2, 'test');
+  INSERT INTO fts (fts) VALUES ('optimize');
+  COMMIT;
+  SELECT level, idx FROM fts_segdir;
+} {0 0}
+do_execsql_test 3.4 {
+  INSERT INTO fts(fts) VALUES('integrity-check');
+  SELECT rowid FROM fts WHERE fts MATCH 'test';
+} {2}
+do_execsql_test 3.5 {
+  INSERT INTO fts (fts) VALUES ('optimize');
+  INSERT INTO fts(fts) VALUES('integrity-check');
+}
+do_test 3.6 {
+  set c1 [db total_changes]
+  execsql { INSERT INTO fts (fts) VALUES ('optimize') }
+  expr {[db total_changes] - $c1}
+} {1}
+do_test 3.7 {
+  execsql { INSERT INTO fts (rowid, t) VALUES (3, 'xyz') }
+  set c1 [db total_changes]
+  execsql { INSERT INTO fts (fts) VALUES ('optimize') }
+  expr {([db total_changes] - $c1) > 1}
+} {1}
+do_test 3.8 {
+  set c1 [db total_changes]
+  execsql { INSERT INTO fts (fts) VALUES ('optimize') }
+  expr {[db total_changes] - $c1}
+} {1}
+
+finish_test
diff --git a/test/fuzz-oss1.test b/test/fuzz-oss1.test
index 08bc6708..e77b7ed0 100644
--- a/test/fuzz-oss1.test
+++ b/test/fuzz-oss1.test
@@ -1997,5 +1997,9 @@ NULL AND ("9_u" COLLATE NOCASE = ? COLLATE NOCASE))) FROM (SELECT
   }
 } {/.* Goto .*/}
 
+# Crash reported by OSS-FUZZ on 2016-11-10
+do_catchsql_test fuzz-oss1-detach {
+  DETACH x IS #1;
+} {1 {near "#1": syntax error}}
 
 finish_test
diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c
index 03776f1e..fec65881 100644
--- a/test/fuzzcheck.c
+++ b/test/fuzzcheck.c
@@ -70,6 +70,7 @@
 #include 
 #include 
 #include "sqlite3.h"
+#include 
 #define ISSPACE(X) isspace((unsigned char)(X))
 #define ISDIGIT(X) isdigit((unsigned char)(X))
 
@@ -188,7 +189,7 @@ static int progressHandler(void *pVdbeLimitFlag){
 ** Reallocate memory.  Show and error and quit if unable.
 */
 static void *safe_realloc(void *pOld, int szNew){
-  void *pNew = realloc(pOld, szNew);
+  void *pNew = realloc(pOld, szNew<=0 ? 1 : szNew);
   if( pNew==0 ) fatalError("unable to realloc for %d bytes", szNew);
   return pNew;
 }
@@ -255,8 +256,9 @@ static VFile *createVFile(const char *zName, int sz, unsigned char *pData){
   if( i>=MX_FILE ) return 0;
   pNew = &g.aFile[i];
   if( zName ){
-    pNew->zFilename = safe_realloc(0, strlen(zName)+1);
-    memcpy(pNew->zFilename, zName, strlen(zName)+1);
+    int nName = (int)strlen(zName)+1;
+    pNew->zFilename = safe_realloc(0, nName);
+    memcpy(pNew->zFilename, zName, nName);
   }else{
     pNew->zFilename = 0;
   }
@@ -620,12 +622,14 @@ static void inmemVfsRegister(void){
 */
 static void runSql(sqlite3 *db, const char *zSql, unsigned  runFlags){
   const char *zMore;
+  const char *zEnd = &zSql[strlen(zSql)];
   sqlite3_stmt *pStmt;
 
   while( zSql && zSql[0] ){
     zMore = 0;
     pStmt = 0;
     sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zMore);
+    assert( zMore<=zEnd );
     if( zMore==zSql ) break;
     if( runFlags & SQL_TRACE ){
       const char *z = zSql;
@@ -778,25 +782,23 @@ static void showHelp(void){
 "Read databases and SQL scripts from SOURCE-DB and execute each script against\n"
 "each database, checking for crashes and memory leaks.\n"
 "Options:\n"
-"  --cell-size-check     Set the PRAGMA cell_size_check=ON\n"
-"  --dbid N              Use only the database where dbid=N\n"
-"  --export-db DIR       Write databases to files(s) in DIR. Works with --dbid\n"
-"  --export-sql DIR      Write SQL to file(s) in DIR. Also works with --sqlid\n"
-"  --help                Show this help text\n"
-"  -q                    Reduced output\n"
-"  --quiet               Reduced output\n"
-"  --limit-mem N         Limit memory used by test SQLite instance to N bytes\n"
-"  --limit-vdbe          Panic if an sync SQL runs for more than 100,000 cycles\n"
-"  --load-sql ARGS...    Load SQL scripts fro files into SOURCE-DB\n"
-"  --load-db ARGS...     Load template databases from files into SOURCE_DB\n"
-"  -m TEXT               Add a description to the database\n"
-"  --native-vfs          Use the native VFS for initially empty database files\n"
-"  --rebuild             Rebuild and vacuum the database file\n"
-"  --result-trace        Show the results of each SQL command\n"
-"  --sqlid N             Use only SQL where sqlid=N\n"
-"  --timeout N           Abort if any single test case needs more than N seconds\n"
-"  -v                    Increased output\n"
-"  --verbose             Increased output\n"
+"  --cell-size-check    Set the PRAGMA cell_size_check=ON\n"
+"  --dbid N             Use only the database where dbid=N\n"
+"  --export-db DIR      Write databases to files(s) in DIR. Works with --dbid\n"
+"  --export-sql DIR     Write SQL to file(s) in DIR. Also works with --sqlid\n"
+"  --help               Show this help text\n"
+"  -q|--quiet           Reduced output\n"
+"  --limit-mem N        Limit memory used by test SQLite instance to N bytes\n"
+"  --limit-vdbe         Panic if any test runs for more than 100,000 cycles\n"
+"  --load-sql ARGS...   Load SQL scripts fro files into SOURCE-DB\n"
+"  --load-db ARGS...    Load template databases from files into SOURCE_DB\n"
+"  -m TEXT              Add a description to the database\n"
+"  --native-vfs         Use the native VFS for initially empty database files\n"
+"  --rebuild            Rebuild and vacuum the database file\n"
+"  --result-trace       Show the results of each SQL command\n"
+"  --sqlid N            Use only SQL where sqlid=N\n"
+"  --timeout N          Abort if any single test needs more than N seconds\n"
+"  -v|--verbose         Increased output.  Repeat for more output.\n"
   );
 }
 
@@ -919,8 +921,8 @@ int main(int argc, char **argv){
       }else
       if( strcmp(z,"verbose")==0 || strcmp(z,"v")==0 ){
         quietFlag = 0;
-        verboseFlag = 1;
-        runFlags |= SQL_TRACE;
+        verboseFlag++;
+        if( verboseFlag>1 ) runFlags |= SQL_TRACE;
       }else
       {
         fatalError("unknown option: %s", argv[i]);
@@ -1055,7 +1057,7 @@ int main(int argc, char **argv){
     /* Print the description, if there is one */
     if( !quietFlag ){
       zDbName = azSrcDb[iSrcDb];
-      i = strlen(zDbName) - 1;
+      i = (int)strlen(zDbName) - 1;
       while( i>0 && zDbName[i-1]!='/' && zDbName[i-1]!='\\' ){ i--; }
       zDbName += i;
       sqlite3_prepare_v2(db, "SELECT msg FROM readme", -1, &pStmt, 0);
diff --git a/test/hook.test b/test/hook.test
index de6fbdd2..12e5bc06 100644
--- a/test/hook.test
+++ b/test/hook.test
@@ -21,6 +21,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set ::testprefix hook
 
 do_test hook-1.2 {
   db commit_hook
@@ -438,4 +439,413 @@ do_test hook-6.2 {
 } {COMMIT ROLLBACK}
 unset ::hooks
 
+#----------------------------------------------------------------------------
+# The following tests - hook-7.* - test the pre-update hook.
+#
+ifcapable !preupdate {
+  finish_test
+  return
+}
+#
+# 7.1.1 - INSERT statement.
+# 7.1.2 - INSERT INTO ... SELECT statement.
+# 7.1.3 - REPLACE INTO ... (rowid conflict)
+# 7.1.4 - REPLACE INTO ... (other index conflicts)
+# 7.1.5 - REPLACE INTO ... (both rowid and other index conflicts)
+#
+# 7.2.1 - DELETE statement.
+# 7.2.2 - DELETE statement that uses the truncate optimization.
+#
+# 7.3.1 - UPDATE statement.
+# 7.3.2 - UPDATE statement that modifies the rowid.
+# 7.3.3 - UPDATE OR REPLACE ... (rowid conflict).
+# 7.3.4 - UPDATE OR REPLACE ... (other index conflicts)
+# 7.3.4 - UPDATE OR REPLACE ... (both rowid and other index conflicts)
+#
+# 7.4.1 - Test that the pre-update-hook is invoked only once if a row being
+#         deleted is removed by a BEFORE trigger.
+#
+# 7.4.2 - Test that the pre-update-hook is invoked if a BEFORE trigger 
+#         removes a row being updated. In this case the update hook should
+#         be invoked with SQLITE_INSERT as the opcode when inserting the
+#         new version of the row.
+#         
+# TODO: Short records (those created before a column is added to a table 
+#       using ALTER TABLE)
+#
+
+proc do_preupdate_test {tn sql x} {
+  set X [list]
+  foreach elem $x {lappend X $elem}
+  uplevel do_test $tn [list "
+    set ::preupdate \[list\]
+    execsql { $sql }
+    set ::preupdate
+  "] [list $X]
+}
+
+proc preupdate_hook {args} {
+  set type [lindex $args 0]
+  eval lappend ::preupdate $args
+  if {$type != "INSERT"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      lappend ::preupdate [db preupdate old $i]
+    }
+  }
+  if {$type != "DELETE"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      set rc [catch { db preupdate new $i } v]
+      lappend ::preupdate $v
+    }
+  }
+}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+db preupdate hook preupdate_hook
+
+# Set up a schema to use for tests 7.1.* to 7.3.*.
+do_execsql_test 7.0 { 
+  CREATE TABLE t1(a, b); 
+  CREATE TABLE t2(x, y); 
+  CREATE TABLE t3(i, j, UNIQUE(i));
+
+  INSERT INTO t2 VALUES('a', 'b');
+  INSERT INTO t2 VALUES('c', 'd');
+
+  INSERT INTO t3 VALUES(4, 16);
+  INSERT INTO t3 VALUES(5, 25);
+  INSERT INTO t3 VALUES(6, 36);
+} 
+
+do_preupdate_test 7.1.1 {
+  INSERT INTO t1 VALUES('x', 'y')
+} {INSERT main t1 1 1  x y}
+
+# 7.1.2.1 does not use the xfer optimization. 7.1.2.2 does.
+do_preupdate_test 7.1.2.1 {
+  INSERT INTO t1 SELECT y, x FROM t2;
+} {INSERT main t1 2 2 b a   INSERT main t1 3 3 d c}
+do_preupdate_test 7.1.2.2 {
+  INSERT INTO t1 SELECT * FROM t2;
+} {INSERT main t1 4 4 a b   INSERT main t1 5 5 c d}
+
+do_preupdate_test 7.1.3 {
+  REPLACE INTO t1(rowid, a, b) VALUES(1, 1, 1);
+} {
+  DELETE main t1 1 1   x y
+  INSERT main t1 1 1   1 1
+}
+
+do_preupdate_test 7.1.4 {
+  REPLACE INTO t3 VALUES(4, NULL);
+} {
+  DELETE main t3 1 1   4 16
+  INSERT main t3 4 4   4 {}
+}
+
+do_preupdate_test 7.1.5 {
+  REPLACE INTO t3(rowid, i, j) VALUES(2, 6, NULL);
+} {
+  DELETE main t3 2 2  5 25
+  DELETE main t3 3 3  6 36
+  INSERT main t3 2 2  6 {}
+}
+
+do_execsql_test 7.2.0 { SELECT rowid FROM t1 } {1 2 3 4 5}
+
+do_preupdate_test 7.2.1 {
+  DELETE FROM t1 WHERE rowid = 3
+} {
+  DELETE main t1 3 3  d c
+}
+do_preupdate_test 7.2.2 {
+  DELETE FROM t1
+} {
+  DELETE main t1 1 1   1 1
+  DELETE main t1 2 2   b a
+  DELETE main t1 4 4   a b
+  DELETE main t1 5 5   c d
+}
+
+do_execsql_test 7.3.0 { 
+  DELETE FROM t1;
+  DELETE FROM t2;
+  DELETE FROM t3;
+
+  INSERT INTO t2 VALUES('a', 'b');
+  INSERT INTO t2 VALUES('c', 'd');
+
+  INSERT INTO t3 VALUES(4, 16);
+  INSERT INTO t3 VALUES(5, 25);
+  INSERT INTO t3 VALUES(6, 36);
+}
+
+do_preupdate_test 7.3.1 {
+  UPDATE t2 SET y = y||y;
+} {
+  UPDATE main t2 1 1   a b  a bb
+  UPDATE main t2 2 2   c d  c dd
+}
+
+do_preupdate_test 7.3.2 {
+  UPDATE t2 SET rowid = rowid-1;
+} {
+  UPDATE main t2 1 0   a bb  a bb
+  UPDATE main t2 2 1   c dd  c dd
+}
+
+do_preupdate_test 7.3.3 {
+  UPDATE OR REPLACE t2 SET rowid = 1 WHERE x = 'a'
+} {
+  DELETE main t2 1 1   c dd
+  UPDATE main t2 0 1   a bb  a bb
+}
+
+do_preupdate_test 7.3.4.1 {
+  UPDATE OR REPLACE t3 SET i = 5 WHERE i = 6
+} {
+  DELETE main t3 2 2   5 25
+  UPDATE main t3 3 3   6 36  5 36
+}
+
+do_execsql_test 7.3.4.2 {
+  INSERT INTO t3 VALUES(10, 100);
+  SELECT rowid, * FROM t3;
+} {1 4 16   3 5 36   4 10 100}
+
+do_preupdate_test 7.3.5 {
+  UPDATE OR REPLACE t3 SET rowid = 1, i = 5 WHERE j = 100;
+} {
+  DELETE main t3 1 1    4  16
+  DELETE main t3 3 3    5  36
+  UPDATE main t3 4 1   10 100  5 100
+}
+
+do_execsql_test 7.4.1.0 {
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES('a', 1);
+  INSERT INTO t4 VALUES('b', 2);
+  INSERT INTO t4 VALUES('c', 3);
+
+  CREATE TRIGGER t4t BEFORE DELETE ON t4 BEGIN
+    DELETE FROM t4 WHERE b = 1;
+  END;
+}
+
+do_preupdate_test 7.4.1.1 {
+  DELETE FROM t4 WHERE b = 3
+} {
+  DELETE main t4 1 1   a 1
+  DELETE main t4 3 3   c 3
+}
+
+do_execsql_test 7.4.1.2 {
+  INSERT INTO t4(rowid, a, b) VALUES(1, 'a', 1);
+  INSERT INTO t4(rowid, a, b) VALUES(3, 'c', 3);
+}
+do_preupdate_test 7.4.1.3 {
+  DELETE FROM t4 WHERE b = 1
+} {
+  DELETE main t4 1 1   a 1
+}
+
+do_execsql_test 7.4.2.0 {
+  CREATE TABLE t5(a, b);
+  INSERT INTO t5 VALUES('a', 1);
+  INSERT INTO t5 VALUES('b', 2);
+  INSERT INTO t5 VALUES('c', 3);
+
+  CREATE TRIGGER t5t BEFORE UPDATE ON t5 BEGIN
+    DELETE FROM t5 WHERE b = 1;
+  END;
+}
+do_preupdate_test 7.4.2.1 {
+  UPDATE t5 SET b = 4 WHERE a = 'c'
+} {
+  DELETE main t5 1 1   a 1
+  UPDATE main t5 3 3   c 3  c 4
+}
+
+do_execsql_test 7.4.2.2 {
+  INSERT INTO t5(rowid, a, b) VALUES(1, 'a', 1);
+}
+
+do_preupdate_test 7.4.2.3 {
+  UPDATE t5 SET b = 5 WHERE a = 'a'
+} {
+  DELETE main t5 1 1   a 1
+}
+
+do_execsql_test 7.5.1.0 {
+  CREATE TABLE t7(a, b);
+  INSERT INTO t7 VALUES('one', 'two');
+  INSERT INTO t7 VALUES('three', 'four');
+  ALTER TABLE t7 ADD COLUMN c DEFAULT NULL;
+}
+
+do_preupdate_test 7.5.1.1 {
+  DELETE FROM t7 WHERE a = 'one'
+} {
+  DELETE main t7 1 1   one two {}
+}
+
+do_preupdate_test 7.5.1.2 {
+  UPDATE t7 SET b = 'five'
+} {
+  UPDATE main t7 2 2   three four {}  three five {}
+}
+
+do_execsql_test 7.5.2.0 {
+  CREATE TABLE t8(a, b);
+  INSERT INTO t8 VALUES('one', 'two');
+  INSERT INTO t8 VALUES('three', 'four');
+  ALTER TABLE t8 ADD COLUMN c DEFAULT 'xxx';
+}
+
+if 0 {
+  # At time of writing, these two are broken. They demonstrate that the
+  # sqlite3_preupdate_old() method does not handle the case where ALTER TABLE
+  # has been used to add a column with a default value other than NULL.
+  #
+  do_preupdate_test 7.5.2.1 {
+    DELETE FROM t8 WHERE a = 'one'
+  } {
+    DELETE main t8 1 1   one two xxx
+  }
+  do_preupdate_test 7.5.2.2 {
+    UPDATE t8 SET b = 'five'
+  } {
+    UPDATE main t8 2 2   three four xxx  three five xxx
+  }
+}
+
+# This block of tests verifies that IPK values are correctly reported
+# by the sqlite3_preupdate_old() and sqlite3_preupdate_new() functions.
+#
+do_execsql_test 7.6.1 { CREATE TABLE t9(a, b INTEGER PRIMARY KEY, c) }
+do_preupdate_test 7.6.2 {
+  INSERT INTO t9 VALUES(1, 2, 3);
+  UPDATE t9 SET b = b+1, c = c+1;
+  DELETE FROM t9 WHERE a = 1;
+} {
+  INSERT main t9 2 2   1 2 3
+  UPDATE main t9 2 3   1 2 3   1 3 4
+  DELETE main t9 3 3   1 3 4
+}
+
+#--------------------------------------------------------------------------
+# Test that the sqlite3_preupdate_depth() API seems to work.
+#
+proc preupdate_hook {args} {
+  set type [lindex $args 0]
+  eval lappend ::preupdate $args
+  eval lappend ::preupdate [db preupdate depth]
+
+  if {$type != "INSERT"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      lappend ::preupdate [db preupdate old $i]
+    }
+  }
+  if {$type != "DELETE"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      set rc [catch { db preupdate new $i } v]
+      lappend ::preupdate $v
+    }
+  }
+}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+db preupdate hook preupdate_hook
+
+do_execsql_test 7.6.1 { 
+  CREATE TABLE t1(x PRIMARY KEY);
+  CREATE TABLE t2(x PRIMARY KEY);
+  CREATE TABLE t3(x PRIMARY KEY);
+  CREATE TABLE t4(x PRIMARY KEY);
+
+  CREATE TRIGGER a AFTER INSERT ON t1 BEGIN INSERT INTO t2 VALUES(new.x); END;
+  CREATE TRIGGER b AFTER INSERT ON t2 BEGIN INSERT INTO t3 VALUES(new.x); END;
+  CREATE TRIGGER c AFTER INSERT ON t3 BEGIN INSERT INTO t4 VALUES(new.x); END;
+
+  CREATE TRIGGER d AFTER UPDATE ON t1 BEGIN UPDATE t2 SET x = new.x; END;
+  CREATE TRIGGER e AFTER UPDATE ON t2 BEGIN UPDATE t3 SET x = new.x; END;
+  CREATE TRIGGER f AFTER UPDATE ON t3 BEGIN UPDATE t4 SET x = new.x; END;
+
+  CREATE TRIGGER g AFTER DELETE ON t1 BEGIN DELETE FROM t2 WHERE 1; END;
+  CREATE TRIGGER h AFTER DELETE ON t2 BEGIN DELETE FROM t3 WHERE 1; END;
+  CREATE TRIGGER i AFTER DELETE ON t3 BEGIN DELETE FROM t4 WHERE 1; END;
+}
+
+do_preupdate_test 7.6.2 {
+  INSERT INTO t1 VALUES('xyz');
+} {
+  INSERT main t1 1 1   0      xyz
+  INSERT main t2 1 1   1      xyz
+  INSERT main t3 1 1   2      xyz
+  INSERT main t4 1 1   3      xyz
+}
+do_preupdate_test 7.6.3 {
+  UPDATE t1 SET x = 'abc';
+} {
+  UPDATE main t1 1 1   0      xyz abc
+  UPDATE main t2 1 1   1      xyz abc
+  UPDATE main t3 1 1   2      xyz abc
+  UPDATE main t4 1 1   3      xyz abc
+}
+do_preupdate_test 7.6.4 {
+  DELETE FROM t1 WHERE 1;
+} {
+  DELETE main t1 1 1   0      abc
+  DELETE main t2 1 1   1      abc
+  DELETE main t3 1 1   2      abc
+  DELETE main t4 1 1   3      abc
+}
+
+do_execsql_test 7.6.5 { 
+  DROP TRIGGER a; DROP TRIGGER b; DROP TRIGGER c;
+  DROP TRIGGER d; DROP TRIGGER e; DROP TRIGGER f;
+  DROP TRIGGER g; DROP TRIGGER h; DROP TRIGGER i;
+
+  CREATE TRIGGER a BEFORE INSERT ON t1 BEGIN INSERT INTO t2 VALUES(new.x); END;
+  CREATE TRIGGER b BEFORE INSERT ON t2 BEGIN INSERT INTO t3 VALUES(new.x); END;
+  CREATE TRIGGER c BEFORE INSERT ON t3 BEGIN INSERT INTO t4 VALUES(new.x); END;
+
+  CREATE TRIGGER d BEFORE UPDATE ON t1 BEGIN UPDATE t2 SET x = new.x; END;
+  CREATE TRIGGER e BEFORE UPDATE ON t2 BEGIN UPDATE t3 SET x = new.x; END;
+  CREATE TRIGGER f BEFORE UPDATE ON t3 BEGIN UPDATE t4 SET x = new.x; END;
+
+  CREATE TRIGGER g BEFORE DELETE ON t1 BEGIN DELETE FROM t2 WHERE 1; END;
+  CREATE TRIGGER h BEFORE DELETE ON t2 BEGIN DELETE FROM t3 WHERE 1; END;
+  CREATE TRIGGER i BEFORE DELETE ON t3 BEGIN DELETE FROM t4 WHERE 1; END;
+}
+
+do_preupdate_test 7.6.6 {
+  INSERT INTO t1 VALUES('xyz');
+} {
+  INSERT main t4 1 1   3      xyz
+  INSERT main t3 1 1   2      xyz
+  INSERT main t2 1 1   1      xyz
+  INSERT main t1 1 1   0      xyz
+}
+do_preupdate_test 7.6.3 {
+  UPDATE t1 SET x = 'abc';
+} {
+  UPDATE main t4 1 1   3      xyz abc
+  UPDATE main t3 1 1   2      xyz abc
+  UPDATE main t2 1 1   1      xyz abc
+  UPDATE main t1 1 1   0      xyz abc
+}
+do_preupdate_test 7.6.4 {
+  DELETE FROM t1 WHERE 1;
+} {
+  DELETE main t4 1 1   3      abc
+  DELETE main t3 1 1   2      abc
+  DELETE main t2 1 1   1      abc
+  DELETE main t1 1 1   0      abc
+}
+
 finish_test
diff --git a/test/icu.test b/test/icu.test
index 73cb9b91..743bcfae 100644
--- a/test/icu.test
+++ b/test/icu.test
@@ -72,6 +72,10 @@ test_expr icu-2.6 {i1=$::OGRAVE} {upper(i1)}     $::OGRAVE
 test_expr icu-2.7 {i1=$::szlig} {upper(i1)}      "SS"
 test_expr icu-2.8 {i1='SS'} {lower(i1)}          "ss"
 
+do_execsql_test icu-2.9 {
+  SELECT upper(char(0xfb04,0xfb04,0xfb04,0xfb04));
+} {FFLFFLFFLFFL}
+
 # In turkish (locale="tr_TR"), the lower case version of I
 # is "small dotless i" (code point 0x131 (decimal 305)).
 #
@@ -133,4 +137,9 @@ do_catchsql_test icu-5.4 {
 do_catchsql_test icu-5.4 { SELECT 'abc' REGEXP }    {1 {near " ": syntax error}}
 do_catchsql_test icu-5.5 { SELECT 'abc' REGEXP, 1 } {1 {near ",": syntax error}}
 
+
+do_malloc_test icu-6.10 -sqlbody {
+  SELECT upper(char(0xfb04,0xdf,0xfb04,0xe8,0xfb04));
+}
+
 finish_test
diff --git a/test/in.test b/test/in.test
index 3a42e84b..4e108004 100644
--- a/test/in.test
+++ b/test/in.test
@@ -314,7 +314,7 @@ do_test in-9.4 {
   catchsql {
     SELECT b FROM t1 WHERE a NOT IN tb;
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 
 # IN clauses in CHECK constraints.  Ticket #1645
 #
@@ -391,28 +391,28 @@ do_test in-12.2 {
       SELECT a, b FROM t3 UNION ALL SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.3 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
       SELECT a, b FROM t3 UNION SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.4 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
       SELECT a, b FROM t3 EXCEPT SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.5 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
       SELECT a, b FROM t3 INTERSECT SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.6 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
@@ -478,7 +478,7 @@ do_test in-12.14 {
       SELECT a, b FROM t3 UNION ALL SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.15 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
@@ -629,15 +629,27 @@ do_test in-13.14 {
   }
 } {}
 
+breakpoint
 do_test in-13.15 {
   catchsql {
     SELECT 0 WHERE (SELECT 0,0) OR (0 IN (1,2));
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 
 
 do_test in-13.X {
   db nullvalue ""
 } {}
 
+# At one point the following was causing valgrind to report a "jump
+# depends on unitialized location" problem.
+#
+do_execsql_test in-14.0 {
+  CREATE TABLE c1(a);
+  INSERT INTO c1 VALUES(1), (2), (4), (3);
+}
+do_execsql_test in-14.1 {
+  SELECT * FROM c1 WHERE a IN (SELECT a FROM c1) ORDER BY 1
+} {1 2 3 4}
+
 finish_test
diff --git a/test/in5.test b/test/in5.test
index 67d21258..ac60cfe5 100644
--- a/test/in5.test
+++ b/test/in5.test
@@ -183,4 +183,58 @@ do_execsql_test 6.3.1 {
   SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2);
 } {2}
 
+#-------------------------------------------------------------------------
+# Test to confirm that bug [5e3c886796e5] is fixed.
+#
+do_execsql_test 7.1 {
+  CREATE TABLE y1(a, b);
+  CREATE TABLE y2(c);
+
+  INSERT INTO y1 VALUES(1,     'one');
+  INSERT INTO y1 VALUES('two', 'two');
+  INSERT INTO y1 VALUES(3,     'three');
+
+  INSERT INTO y2 VALUES('one');
+  INSERT INTO y2 VALUES('two');
+  INSERT INTO y2 VALUES('three');
+} {}
+
+do_execsql_test 7.2.1 {
+  SELECT a FROM y1 WHERE b NOT IN (SELECT a FROM y2);
+} {1 3}
+do_execsql_test 7.2.2 {
+  SELECT a FROM y1 WHERE b IN (SELECT a FROM y2);
+} {two}
+
+do_execsql_test 7.3.1 {
+  CREATE INDEX y2c ON y2(c);
+  SELECT a FROM y1 WHERE b NOT IN (SELECT a FROM y2);
+} {1 3}
+do_execsql_test 7.3.2 {
+  SELECT a FROM y1 WHERE b IN (SELECT a FROM y2);
+} {two}
+
+#-------------------------------------------------------------------------
+# Tests to confirm that indexes on the rowid column do not confuse
+# the query planner. See ticket [0eab1ac7591f511d].
+#
+do_execsql_test 8.0 {
+  CREATE TABLE n1(a INTEGER PRIMARY KEY, b VARCHAR(500));
+  CREATE UNIQUE INDEX n1a ON n1(a);
+}
+
+do_execsql_test 8.1 {
+  SELECT count(*) FROM n1 WHERE a IN (1, 2, 3)
+} 0
+do_execsql_test 8.2 {
+  SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
+} 0
+do_execsql_test 8.3 {
+  INSERT INTO n1 VALUES(1, NULL), (2, NULL), (3, NULL);
+  SELECT count(*) FROM n1 WHERE a IN (1, 2, 3)
+} 3
+do_execsql_test 8.4 {
+  SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
+} 3
+
 finish_test
diff --git a/test/incrblob.test b/test/incrblob.test
index 4277e5c4..c56689ee 100644
--- a/test/incrblob.test
+++ b/test/incrblob.test
@@ -126,6 +126,11 @@ foreach AutoVacuumMode [list 0 1] {
   execsql "PRAGMA mmap_size = 0"
   execsql "PRAGMA auto_vacuum = $AutoVacuumMode"
 
+  # Extra value added to size answers
+  set ib2_extra 0
+  if {$AutoVacuumMode} {incr ib2_extra}
+  if {[nonzero_reserved_bytes]} {incr ib2_extra}
+
   do_test incrblob-2.$AutoVacuumMode.1 {
     set ::str [string repeat abcdefghij 2900]
     execsql {
@@ -136,7 +141,7 @@ foreach AutoVacuumMode [list 0 1] {
       COMMIT;
     }
     expr [file size test.db]/1024
-  } [expr 31 + $AutoVacuumMode]
+  } [expr 31 + $ib2_extra]
 
   ifcapable autovacuum {
     do_test incrblob-2.$AutoVacuumMode.2 {
@@ -163,7 +168,7 @@ foreach AutoVacuumMode [list 0 1] {
     # sqlite uses the ptrmap pages to avoid reading the other pages.
     #
     nRead db
-  } [expr $AutoVacuumMode ? 4 : 30]
+  } [expr $AutoVacuumMode ? 4 : 30+$ib2_extra]
 
   do_test incrblob-2.$AutoVacuumMode.4 {
     string range [db one {SELECT v FROM blobs}] end-19 end
@@ -187,7 +192,7 @@ foreach AutoVacuumMode [list 0 1] {
     # sqlite uses the ptrmap pages to avoid reading the other pages.
     #
     nRead db
-  } [expr $AutoVacuumMode ? 4 : 30]
+  } [expr $AutoVacuumMode ? 4 : 30 + $ib2_extra]
 
   # Pages 1 (the write-counter) and 32 (the blob data) were written.
   do_test incrblob-2.$AutoVacuumMode.6 {
@@ -210,7 +215,7 @@ foreach AutoVacuumMode [list 0 1] {
 
   do_test incrblob-2.$AutoVacuumMode.9 {
     nRead db
-  } [expr $AutoVacuumMode ? 4 : 30]
+  } [expr $AutoVacuumMode ? 4 : 30 + $ib2_extra]
 }
 sqlite3_soft_heap_limit $cmdlinearg(soft-heap-limit)
 
@@ -384,7 +389,7 @@ ifcapable vtab {
 ifcapable attach {
   do_test incrblob-5.1 {
     forcedelete test2.db test2.db-journal
-    set ::size [expr [file size [info script]]]
+    set ::size [expr [file size $::cmdlinearg(INFO_SCRIPT)]]
     execsql {
       ATTACH 'test2.db' AS aux;
       CREATE TABLE aux.files(name, text);
@@ -392,16 +397,16 @@ ifcapable attach {
     }
     set fd  [db incrblob aux files text 1]
     fconfigure $fd -translation binary
-    set fd2 [open [info script]]
+    set fd2 [open $::cmdlinearg(INFO_SCRIPT)]
     fconfigure $fd2 -translation binary
     puts -nonewline $fd [read $fd2]
     close $fd
     close $fd2
     set ::text [db one {select text from aux.files}]
     string length $::text
-  } [file size [info script]]
+  } [file size $::cmdlinearg(INFO_SCRIPT)]
   do_test incrblob-5.2 {
-    set fd2 [open [info script]]
+    set fd2 [open $::cmdlinearg(INFO_SCRIPT)]
     fconfigure $fd2 -translation binary
     set ::data [read $fd2]
     close $fd2
@@ -576,7 +581,7 @@ foreach {tn arg} {1 "" 2 -readonly} {
 
 }
 
-set fd [open [info script]]
+set fd [open $::cmdlinearg(INFO_SCRIPT)]
 fconfigure $fd -translation binary
 set ::data [read $fd 14000]
 close $fd
diff --git a/test/incrblob_err.test b/test/incrblob_err.test
index a08bea3e..0db8b0dc 100644
--- a/test/incrblob_err.test
+++ b/test/incrblob_err.test
@@ -24,12 +24,12 @@ ifcapable {!incrblob  || !memdebug || !tclvar} {
 source $testdir/malloc_common.tcl
 
 unset -nocomplain ::fd ::data
-set ::fd [open [info script]]
+set ::fd [open $::cmdlinearg(INFO_SCRIPT)]
 set ::data [read $::fd]
 close $::fd
 
 do_malloc_test 1 -tclprep {
-  set bytes [file size [info script]]
+  set bytes [file size $::cmdlinearg(INFO_SCRIPT)]
   execsql {
     CREATE TABLE blobs(k, v BLOB);
     INSERT INTO blobs VALUES(1, zeroblob($::bytes));
diff --git a/test/incrvacuum2.test b/test/incrvacuum2.test
index 2219d541..6ec66c9d 100644
--- a/test/incrvacuum2.test
+++ b/test/incrvacuum2.test
@@ -134,7 +134,7 @@ do_test incrvacuum2-3.2 {
 
 integrity_check incrvacuum2-3.3
 
-ifcapable wal {
+if {[wal_is_capable]} {
   # At one point, when a specific page was being extracted from the b-tree
   # free-list (e.g. during an incremental-vacuum), all trunk pages that
   # occurred before the specific page in the free-list trunk were being
diff --git a/test/index6.test b/test/index6.test
index 33ae3d1c..f6177b44 100644
--- a/test/index6.test
+++ b/test/index6.test
@@ -65,12 +65,15 @@ do_test index6-1.5 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a!=random();
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {1 {non-deterministic functions prohibited in partial index WHERE clauses}}
 do_test index6-1.6 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a NOT LIKE 'abc%';
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {0 {}}
+do_execsql_test index6-1.7 {
+  DROP INDEX IF EXISTS bad1;
+}
 
 do_test index6-1.10 {
   execsql {
@@ -376,4 +379,15 @@ do_execsql_test index6-10.3eqp {
   SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC;
 } {~/USING INDEX t10x/}
 
+# A partial index will be used for a full table scan, where possible
+do_execsql_test index6-11.1 {
+  CREATE TABLE t11(a,b,c);
+  CREATE INDEX t11x ON t11(a) WHERE b<>99;
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
+} {/USING INDEX t11x/}
+do_execsql_test index6-11.2 {
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
+} {/USING INDEX t11x/}
+  
+
 finish_test
diff --git a/test/index7.test b/test/index7.test
index 557fe213..0037a8a4 100644
--- a/test/index7.test
+++ b/test/index7.test
@@ -99,12 +99,25 @@ do_test index7-1.5 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a!=random();
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {1 {non-deterministic functions prohibited in partial index WHERE clauses}}
 do_test index7-1.6 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a NOT LIKE 'abc%';
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {0 {}}
+do_execsql_test index7-1.7 {
+  INSERT INTO t1(a,b,c)
+     VALUES('abcde',1,101),('abdef',2,102),('xyz',3,103),('abcz',4,104);
+  SELECT c FROM t1 WHERE a NOT LIKE 'abc%' AND a=7 ORDER BY +b;
+} {7}
+do_execsql_test index7-1.7eqp {
+  EXPLAIN QUERY PLAN
+  SELECT b FROM t1 WHERE a NOT LIKE 'abc%' AND a=7 ORDER BY +b;
+} {/SEARCH TABLE t1 USING COVERING INDEX bad1 /}
+do_execsql_test index7-1.8 {
+  DELETE FROM t1 WHERE c>=101;
+  DROP INDEX IF EXISTS bad1;
+} {}
 
 do_test index7-1.10 {
   execsql {
diff --git a/test/index8.test b/test/index8.test
new file mode 100644
index 00000000..bb582285
--- /dev/null
+++ b/test/index8.test
@@ -0,0 +1,60 @@
+# 2016-07-27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for ORDER BY and LIMIT on an index scan.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Performance regression reported at
+# http://www.mail-archive.com/sqlite-users@mailinglists.sqlite.org/msg98615.html
+#
+# Caused by the ORDER BY LIMIT optionation for check-in
+# https://sqlite.org/src/info/bf46179d44843769
+#
+# Fixed on approximately 2016-07-27 by changes that compute a better score
+# for index scans by taking into account WHERE clause constraints that can
+# be handled by the index and do not require a table lookup.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a,b,c,d);
+  WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b,c,d)
+     SELECT x/10, x%10, x%19, x FROM c;
+  CREATE INDEX t1abc ON t1(a,b,c);
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {0 4 4 4 2 3 4 23}
+
+# Prior to the fix, the following EQP would show a table scan and a sort
+# rather than an index scan.
+#
+do_execsql_test 1.0eqp {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {/SCAN TABLE t1 USING INDEX t1abc/}
+
+# If we change the index so that it no longer covers the WHERE clause,
+# then we should (correctly) revert to using a table scan.
+#
+do_execsql_test 1.1 {
+  DROP INDEX t1abc;
+  CREATE INDEX t1abd ON t1(a,b,d);
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {0 4 4 4 2 3 4 23}
+do_execsql_test 1.1eqp {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {~/USING INDEX/}
+
+
+finish_test
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index a8a74f25..cd72430d 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -323,5 +323,51 @@ do_execsql_test indexexpr1-1010 {
   SELECT *, '|' FROM t0 ORDER BY +a;
 } {0 88 2 | 2 99 4 | 5 99 7 |}
 
+# 2016-10-10
+# Make sure indexes on expressions skip over initial NULL values in the
+# index as they are suppose to do.
+# Ticket https://www.sqlite.org/src/tktview/4baa46491212947
+#
+do_execsql_test indexexpr1-1100 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a);
+  INSERT INTO t1 VALUES(NULL),(1);
+  SELECT '1:', typeof(a), a FROM t1 WHERE a<10;
+  SELECT '2:', typeof(a), a FROM t1 WHERE a+0<10;
+  CREATE INDEX t1x1 ON t1(a);
+  CREATE INDEX t1x2 ON t1(a+0);
+  SELECT '3:', typeof(a), a FROM t1 WHERE a<10;
+  SELECT '4:', typeof(a), a FROM t1 WHERE a+0<10;
+} {1: integer 1 2: integer 1 3: integer 1 4: integer 1}
+
+do_execsql_test indexexpr1-1200 {
+  CREATE TABLE t10(a int, b int, c int, d int);
+  INSERT INTO t10(a, b, c, d) VALUES(0, 0, 2, 2);
+  INSERT INTO t10(a, b, c, d) VALUES(0, 0, 0, 0);
+  INSERT INTO t10(a, b, c, d) VALUES(0, 0, 1, 1);
+  INSERT INTO t10(a, b, c, d) VALUES(1, 1, 1, 1);
+  INSERT INTO t10(a, b, c, d) VALUES(1, 1, 0, 0);
+  INSERT INTO t10(a, b, c, d) VALUES(2, 2, 0, 0);
+
+  SELECT a+b, c+d FROM t10 ORDER BY a+b, c+d;
+} {
+  0 0 0 2 0 4 2 0 2 2 4 0
+}
+do_execsql_test indexexpr1-1200.1 {
+  CREATE INDEX t10_ab ON t10(a+b);
+}
+do_execsql_test indexexpr1-1200.2 {
+  SELECT a+b, c+d FROM t10 ORDER BY a+b, c+d;
+} {
+  0 0 0 2 0 4 2 0 2 2 4 0
+}
+do_execsql_test indexexpr1-1200.3 {
+  CREATE INDEX t10_abcd ON t10(a+b,c+d);
+}
+do_execsql_test indexexpr1-1200.4 {
+  SELECT a+b, c+d FROM t10 ORDER BY a+b, c+d;
+} {
+  0 0 0 2 0 4 2 0 2 2 4 0
+}
 
 finish_test
diff --git a/test/instrfault.test b/test/instrfault.test
new file mode 100644
index 00000000..e2d93e74
--- /dev/null
+++ b/test/instrfault.test
@@ -0,0 +1,68 @@
+# 2016 November 4
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing OOM error handling within the built-in 
+# INSTR() function.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix instrfault
+
+# Use big NEEDLE and HAYSTACK strings. Strings so large they cannot
+# use lookaside buffers.
+#
+set ::NEEDLE [string repeat "abcdefghijklmnopqrstuvwxyz" 10]
+set ::HAYSTACK "[string repeat 123 10]$NEEDLE[string repeat 456 10]"
+
+foreach {enc} {
+  utf8
+  utf16
+} {
+  reset_db
+  execsql "PRAGMA encoding = $enc"
+  do_execsql_test 1.$enc.1 {
+    CREATE TABLE t1(n, h);
+    INSERT INTO t1 VALUES($::NEEDLE, $::HAYSTACK);
+  } {}
+
+  do_faultsim_test 1.$enc.1 -faults oom-t* -prep {
+    execsql { SELECT instr(h, n) FROM t1 }
+  } -body {
+    execsql { SELECT instr(h, n) FROM t1 }
+  } -test {
+    faultsim_test_result {0 31}
+  }
+
+  do_faultsim_test 1.$enc.2 -faults oom-t* -prep {
+    execsql { SELECT instr($::HAYSTACK, $::NEEDLE) FROM t1 }
+  } -body {
+    execsql { SELECT instr($::HAYSTACK, $::NEEDLE) FROM t1 }
+  } -test {
+    faultsim_test_result {0 31}
+  }
+
+  do_faultsim_test 1.$enc.3 -faults oom-t* -prep {
+    set ::stmt [sqlite3_prepare_v2 db "SELECT instr(?, ?)" -1 dummy]
+    sqlite3_bind_text $::stmt 1 $::HAYSTACK [string length $::HAYSTACK]
+    sqlite3_bind_text $::stmt 2 $::NEEDLE [string length $::NEEDLE]
+  } -body {
+    set rc [sqlite3_step $::stmt]
+    if {$rc=="SQLITE_NOMEM"} { error "out of memory" }
+    sqlite3_column_int $::stmt 0
+  } -test {
+    faultsim_test_result {0 31}
+    sqlite3_finalize $::stmt
+  }
+}
+
+finish_test
diff --git a/test/intpkey.test b/test/intpkey.test
index 41858e5d..a56cab4c 100644
--- a/test/intpkey.test
+++ b/test/intpkey.test
@@ -296,7 +296,7 @@ do_test intpkey-3.8 {
   count {
     SELECT * FROM t1 WHERE c=='world' AND a>7;
   }
-} {11 hello world 4}
+} {11 hello world 3}
 do_test intpkey-3.9 {
   count {
     SELECT * FROM t1 WHERE 7 "');
+} {0}
+do_execsql_test json-10.32 {
+  SELECT json_valid('" \? "');
+} {0}
+do_execsql_test json-10.33 {
+  SELECT json_valid('" \@ "');
+} {0}
+do_execsql_test json-10.34 {
+  SELECT json_valid('" \A "');
+} {0}
+do_execsql_test json-10.35 {
+  SELECT json_valid('" \B "');
+} {0}
+do_execsql_test json-10.36 {
+  SELECT json_valid('" \C "');
+} {0}
+do_execsql_test json-10.37 {
+  SELECT json_valid('" \D "');
+} {0}
+do_execsql_test json-10.38 {
+  SELECT json_valid('" \E "');
+} {0}
+do_execsql_test json-10.39 {
+  SELECT json_valid('" \F "');
+} {0}
+do_execsql_test json-10.40 {
+  SELECT json_valid('" \G "');
+} {0}
+do_execsql_test json-10.41 {
+  SELECT json_valid('" \H "');
+} {0}
+do_execsql_test json-10.42 {
+  SELECT json_valid('" \I "');
+} {0}
+do_execsql_test json-10.43 {
+  SELECT json_valid('" \J "');
+} {0}
+do_execsql_test json-10.44 {
+  SELECT json_valid('" \K "');
+} {0}
+do_execsql_test json-10.45 {
+  SELECT json_valid('" \L "');
+} {0}
+do_execsql_test json-10.46 {
+  SELECT json_valid('" \M "');
+} {0}
+do_execsql_test json-10.47 {
+  SELECT json_valid('" \N "');
+} {0}
+do_execsql_test json-10.48 {
+  SELECT json_valid('" \O "');
+} {0}
+do_execsql_test json-10.49 {
+  SELECT json_valid('" \P "');
+} {0}
+do_execsql_test json-10.50 {
+  SELECT json_valid('" \Q "');
+} {0}
+do_execsql_test json-10.51 {
+  SELECT json_valid('" \R "');
+} {0}
+do_execsql_test json-10.52 {
+  SELECT json_valid('" \S "');
+} {0}
+do_execsql_test json-10.53 {
+  SELECT json_valid('" \T "');
+} {0}
+do_execsql_test json-10.54 {
+  SELECT json_valid('" \U "');
+} {0}
+do_execsql_test json-10.55 {
+  SELECT json_valid('" \V "');
+} {0}
+do_execsql_test json-10.56 {
+  SELECT json_valid('" \W "');
+} {0}
+do_execsql_test json-10.57 {
+  SELECT json_valid('" \X "');
+} {0}
+do_execsql_test json-10.58 {
+  SELECT json_valid('" \Y "');
+} {0}
+do_execsql_test json-10.59 {
+  SELECT json_valid('" \Z "');
+} {0}
+do_execsql_test json-10.60 {
+  SELECT json_valid('" \[ "');
+} {0}
+do_execsql_test json-10.61 {
+  SELECT json_valid('" \\ "');
+} {1}
+do_execsql_test json-10.62 {
+  SELECT json_valid('" \] "');
+} {0}
+do_execsql_test json-10.63 {
+  SELECT json_valid('" \^ "');
+} {0}
+do_execsql_test json-10.64 {
+  SELECT json_valid('" \_ "');
+} {0}
+do_execsql_test json-10.65 {
+  SELECT json_valid('" \` "');
+} {0}
+do_execsql_test json-10.66 {
+  SELECT json_valid('" \a "');
+} {0}
+do_execsql_test json-10.67 {
+  SELECT json_valid('" \b "');
+} {1}
+do_execsql_test json-10.68 {
+  SELECT json_valid('" \c "');
+} {0}
+do_execsql_test json-10.69 {
+  SELECT json_valid('" \d "');
+} {0}
+do_execsql_test json-10.70 {
+  SELECT json_valid('" \e "');
+} {0}
+do_execsql_test json-10.71 {
+  SELECT json_valid('" \f "');
+} {1}
+do_execsql_test json-10.72 {
+  SELECT json_valid('" \g "');
+} {0}
+do_execsql_test json-10.73 {
+  SELECT json_valid('" \h "');
+} {0}
+do_execsql_test json-10.74 {
+  SELECT json_valid('" \i "');
+} {0}
+do_execsql_test json-10.75 {
+  SELECT json_valid('" \j "');
+} {0}
+do_execsql_test json-10.76 {
+  SELECT json_valid('" \k "');
+} {0}
+do_execsql_test json-10.77 {
+  SELECT json_valid('" \l "');
+} {0}
+do_execsql_test json-10.78 {
+  SELECT json_valid('" \m "');
+} {0}
+do_execsql_test json-10.79 {
+  SELECT json_valid('" \n "');
+} {1}
+do_execsql_test json-10.80 {
+  SELECT json_valid('" \o "');
+} {0}
+do_execsql_test json-10.81 {
+  SELECT json_valid('" \p "');
+} {0}
+do_execsql_test json-10.82 {
+  SELECT json_valid('" \q "');
+} {0}
+do_execsql_test json-10.83 {
+  SELECT json_valid('" \r "');
+} {1}
+do_execsql_test json-10.84 {
+  SELECT json_valid('" \s "');
+} {0}
+do_execsql_test json-10.85 {
+  SELECT json_valid('" \t "');
+} {1}
+do_execsql_test json-10.86.0 {
+  SELECT json_valid('" \u "');
+} {0}
+do_execsql_test json-10.86.1 {
+  SELECT json_valid('" \ua "');
+} {0}
+do_execsql_test json-10.86.2 {
+  SELECT json_valid('" \uab "');
+} {0}
+do_execsql_test json-10.86.3 {
+  SELECT json_valid('" \uabc "');
+} {0}
+do_execsql_test json-10.86.4 {
+  SELECT json_valid('" \uabcd "');
+} {1}
+do_execsql_test json-10.86.5 {
+  SELECT json_valid('" \uFEDC "');
+} {1}
+do_execsql_test json-10.86.6 {
+  SELECT json_valid('" \u1234 "');
+} {1}
+do_execsql_test json-10.87 {
+  SELECT json_valid('" \v "');
+} {0}
+do_execsql_test json-10.88 {
+  SELECT json_valid('" \w "');
+} {0}
+do_execsql_test json-10.89 {
+  SELECT json_valid('" \x "');
+} {0}
+do_execsql_test json-10.90 {
+  SELECT json_valid('" \y "');
+} {0}
+do_execsql_test json-10.91 {
+  SELECT json_valid('" \z "');
+} {0}
+do_execsql_test json-10.92 {
+  SELECT json_valid('" \{ "');
+} {0}
+do_execsql_test json-10.93 {
+  SELECT json_valid('" \| "');
+} {0}
+do_execsql_test json-10.94 {
+  SELECT json_valid('" \} "');
+} {0}
+do_execsql_test json-10.95 {
+  SELECT json_valid('" \~ "');
+} {0}
+
 
 finish_test
diff --git a/test/limit2.test b/test/limit2.test
new file mode 100644
index 00000000..f6eaefb8
--- /dev/null
+++ b/test/limit2.test
@@ -0,0 +1,153 @@
+# 2016-05-20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the LIMIT in combination with ORDER BY
+# and in particular, the optimizations in the inner loop that cause an
+# early exit of the inner loop when the LIMIT is reached and the inner
+# loop is emitting rows in ORDER BY order.
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix limit2
+
+do_execsql_test limit2-100 {
+  CREATE TABLE t1(a,b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t1(a,b) SELECT 1, (x*17)%1000 + 1000 FROM c;
+  INSERT INTO t1(a,b) VALUES(2,2),(3,1006),(4,4),(5,9999);
+  CREATE INDEX t1ab ON t1(a,b);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-100.1 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-100.2 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+do_test limit2-100.3 {
+  set slow_count $sqlite_search_count
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+do_execsql_test limit2-110 {
+  CREATE TABLE t2(x,y);
+  INSERT INTO t2(x,y) VALUES('a',1),('a',2),('a',3),('a',4);
+  INSERT INTO t2(x,y) VALUES('b',1),('c',2),('d',3),('e',4);
+  CREATE INDEX t2xy ON t2(x,y);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-110.1 {
+  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY t1.b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-110.2 {
+  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY +t1.b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set slow_count $sqlite_search_count
+do_test limit2-110.3 {
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+do_execsql_test limit2-120 {
+  DROP INDEX t1ab;
+  CREATE INDEX t1ab ON t1(a,b DESC);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-120.1 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b DESC LIMIT 5;
+} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-120.2 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b DESC LIMIT 5;
+} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
+do_test limit2-120.3 {
+  set slow_count $sqlite_search_count
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+# Bug report against the new ORDER BY LIMIT optimization just prior to
+# release.  (Unreleased so there is no ticket).
+#
+# Make sure the optimization is not applied if the inner loop can only
+# provide a single row of output.
+#
+do_execsql_test limit2-200 {
+  CREATE TABLE t200(a, b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t200(a,b) SELECT x, x FROM c;
+  CREATE TABLE t201(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t201(x,y) VALUES(2,12345);
+
+  SELECT *, '|' FROM t200, t201 WHERE x=b ORDER BY y LIMIT 3;
+} {2 2 2 12345 |}
+do_execsql_test limit2-210 {
+  SELECT *, '|' FROM t200 LEFT JOIN t201 ON x=b ORDER BY y LIMIT 3;
+} {1 1 {} {} | 3 3 {} {} | 4 4 {} {} |}
+
+# Bug in the ORDER BY LIMIT optimization reported on 2016-09-06.
+# Ticket https://www.sqlite.org/src/info/559733b09e96
+#
+do_execsql_test limit2-300 {
+  CREATE TABLE t300(a,b,c);
+  CREATE INDEX t300x ON t300(a,b,c);
+  INSERT INTO t300 VALUES(0,1,99),(0,1,0),(0,0,0);
+  SELECT *,'.' FROM t300 WHERE a=0 AND (c=0 OR c=99) ORDER BY c DESC;
+} {0 1 99 . 0 0 0 . 0 1 0 .}
+do_execsql_test limit2-310 {
+  SELECT *,'.' FROM t300 WHERE a=0 AND (c=0 OR c=99) ORDER BY c DESC LIMIT 1;
+} {0 1 99 .}
+
+# Make sure the SELECT loop is ordered correctly for the direction of
+# the ORDER BY
+#
+do_execsql_test limit2-400 {
+  CREATE TABLE t400(a,b);
+  CREATE INDEX t400_ab ON t400(a,b);
+  INSERT INTO t400(a,b) VALUES(1,90),(1,40),(2,80),(2,30),(3,70),(3,20);
+  SELECT *,'x' FROM t400 WHERE a IN (1,2,3) ORDER BY b DESC LIMIT 3;
+  SELECT *,'y' FROM t400 WHERE a IN (1,2,3) ORDER BY +b DESC LIMIT 3;
+} {1 90 x 2 80 x 3 70 x 1 90 y 2 80 y 3 70 y}
+
+
+do_execsql_test 500 {
+  CREATE TABLE t500(i INTEGER PRIMARY KEY, j);
+  INSERT INTO t500 VALUES(1, 1);
+  INSERT INTO t500 VALUES(2, 2);
+  INSERT INTO t500 VALUES(3, 3);
+  INSERT INTO t500 VALUES(4, 0);
+  INSERT INTO t500 VALUES(5, 5);
+  SELECT j FROM t500 WHERE i IN (1,2,3,4,5) ORDER BY j DESC LIMIT 3;
+} {5 3 2}
+do_execsql_test 501 {
+  CREATE TABLE t501(i INTEGER PRIMARY KEY, j);
+  INSERT INTO t501 VALUES(1, 5);
+  INSERT INTO t501 VALUES(2, 4);
+  INSERT INTO t501 VALUES(3, 3);
+  INSERT INTO t501 VALUES(4, 6);
+  INSERT INTO t501 VALUES(5, 1);
+  SELECT j FROM t501 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3;
+} {1 3 4}
+do_execsql_test 502 {
+  CREATE TABLE t502(i INT PRIMARY KEY, j);
+  INSERT INTO t502 VALUES(1, 5);
+  INSERT INTO t502 VALUES(2, 4);
+  INSERT INTO t502 VALUES(3, 3);
+  INSERT INTO t502 VALUES(4, 6);
+  INSERT INTO t502 VALUES(5, 1);
+  SELECT j FROM t502 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3;
+} {1 3 4}
+
+finish_test
diff --git a/test/loadext.test b/test/loadext.test
index 7ba4c0cf..24ee7a68 100644
--- a/test/loadext.test
+++ b/test/loadext.test
@@ -81,7 +81,7 @@ if {![file exists $testextension]} {
   set testextsrc $srcdir/test_loadext.c
 
   set cmdline [concat exec gcc $gcc_shared]
-  lappend cmdline -Wall -I$srcdir -I. -g $testextsrc -o $testextension
+  lappend cmdline -Wall -I$srcdir -I. -I.. -g $testextsrc -o $testextension
   
   if {[catch $cmdline msg]} {
     puts "Skipping loadext tests: Test extension not built..."
@@ -111,7 +111,7 @@ do_test loadext-1.2 {
 #
 do_test loadext-1.3 {
   sqlite3 db2 test.db
-  sqlite3_enable_load_extension db2 1
+  sqlite3_db_config db2 SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1
   catchsql {
     SELECT half(1.0);
   } db2
@@ -256,6 +256,7 @@ do_test loadext-4.2 {
   }
 } {0 {{}}}
 
+# disable all extension loading
 do_test loadext-4.3 {
   sqlite3_enable_load_extension db 0
   catchsql {
@@ -263,6 +264,15 @@ do_test loadext-4.3 {
   }
 } {1 {not authorized}}
 
+# enable C-api extension loading only.  Show that the SQL function
+# still does not work.
+do_test loadext-4.4 {
+  sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1
+  catchsql {
+    SELECT load_extension($::testextension,'testloadext_init')
+  }
+} {1 {not authorized}}
+
 source $testdir/malloc_common.tcl
 
 
diff --git a/test/lock.test b/test/lock.test
index f785c4bb..534aa3b9 100644
--- a/test/lock.test
+++ b/test/lock.test
@@ -423,8 +423,9 @@ do_test lock-6.5 {
 #   * there exists one or more active read-only statements, and
 #   * a transaction that modified zero database pages is committed.
 # 
-set temp_status unlocked
-if {$TEMP_STORE>=2} {set temp_status unknown}
+#set temp_status unlocked
+#if {$TEMP_STORE>=2} {set temp_status unknown}
+set temp_status unknown
 do_test lock-7.1 {
   set STMT [sqlite3_prepare $DB "SELECT * FROM sqlite_master" -1 TAIL]
   sqlite3_step $STMT
diff --git a/test/main.test b/test/main.test
index 3f35afe2..9346cf6c 100644
--- a/test/main.test
+++ b/test/main.test
@@ -319,7 +319,7 @@ do_test main-3.1 {
   sqlite3 db testdb
   set v [catch {execsql {SELECT * from T1 where x!!5}} msg]
   lappend v $msg
-} {1 {unrecognized token: "!!"}}
+} {1 {unrecognized token: "!"}}
 do_test main-3.2 {
   catch {db close}
   foreach f [glob -nocomplain testdb/*] {forcedelete $f}
diff --git a/test/mallocI.test b/test/mallocI.test
index 12294555..ee4a603d 100644
--- a/test/mallocI.test
+++ b/test/mallocI.test
@@ -61,4 +61,15 @@ do_malloc_test mallocI-4 -tclprep {
 }
 catch { db2 close }
 
+do_faultsim_test mallocI-5 -faults oom* -prep {
+  catch { db close }
+  sqlite3 db test.db
+  sqlite3_db_config_lookaside db 0 0 0
+} -body {
+  db eval { Select CAST(1 AS blob) }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+
 finish_test
diff --git a/test/memsubsys1.test b/test/memsubsys1.test
index 8265ce63..36427f9b 100644
--- a/test/memsubsys1.test
+++ b/test/memsubsys1.test
@@ -100,10 +100,12 @@ reset_highwater_marks
 build_test_db memsubsys1-2 {PRAGMA page_size=1024; PRAGMA mmap_size=0}
 #show_memstats
 set MEMORY_MANAGEMENT $sqlite_options(memorymanage)
-ifcapable !malloc_usable_size {
- do_test memsubsys1-2.3 {
-    set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
-  } [expr ($TEMP_STORE>1 || $MEMORY_MANAGEMENT==0)*1024]
+ifcapable pagecache_overflow_stats {
+  ifcapable !malloc_usable_size {
+    do_test memsubsys1-2.3 {
+      set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
+    } [expr ($TEMP_STORE>1 || $MEMORY_MANAGEMENT==0)*1024]
+  }
 }
 do_test memsubsys1-2.4 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
@@ -255,7 +257,7 @@ do_test memsubsys1-7.4 {
 } 0
 do_test memsubsys1-7.5 {
   set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
-  expr {$maxreq<4100}
+  expr {$maxreq<(4100 + 8200*[nonzero_reserved_bytes])}
 } 1
 do_test memsubsys1-7.6 {
   set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
diff --git a/test/minmax.test b/test/minmax.test
index fb9bbb38..04a3f06d 100644
--- a/test/minmax.test
+++ b/test/minmax.test
@@ -628,5 +628,19 @@ do_test_13_noopt 13.7 {
   SELECT min(c), count(c) FROM t1 WHERE a='a';
 } {1 5}
 
+# 2016-07-26.  https://www.sqlite.org/src/info/a0bac8b3c3d1bb75
+# Incorrect result on a min() query after a CREATE INDEX.
+#
+do_execsql_test 14.1 {
+  CREATE TABLE t14(a INTEGER, b INTEGER);
+  INSERT INTO t14(a,b) VALUES(100,2),(200,2),(300,2),(400,1),(500,2);
+  SELECT min(a) FROM t14 WHERE b='2' AND a>'50';
+} {100}
+do_execsql_test 14.2 {
+  CREATE INDEX t14ba ON t14(b,a);
+  SELECT min(a) FROM t14 WHERE b='2' AND a>'50';
+} {100}
+
+
 
 finish_test
diff --git a/test/misc1.test b/test/misc1.test
index 400a4517..e646bfd0 100644
--- a/test/misc1.test
+++ b/test/misc1.test
@@ -699,7 +699,7 @@ do_catchsql_test misc1-23.3 {
 #
 do_test misc1-24.0 {
   list [catch { sqlite3_prepare_v2 db ! -1 dummy } msg] $msg
-} {1 {(1) unrecognized token: "!}}
+} {1 {(1) unrecognized token: "!"}}
 
 # The following query (provided by Kostya Serebryany) used to take 25
 # minutes to prepare.  This has been speeded up to about 250 milliseconds.
diff --git a/test/misc5.test b/test/misc5.test
index 30176b80..0e7e34dd 100644
--- a/test/misc5.test
+++ b/test/misc5.test
@@ -571,6 +571,8 @@ ifcapable subquery&&compound {
 # Overflow the lemon parser stack by providing an overly complex
 # expression.  Make sure that the overflow is detected and reported.
 #
+# This test fails when building with -DYYSTACKDEPTH=0
+#
 do_test misc5-7.1 {
   execsql {CREATE TABLE t1(x)}
   set sql "INSERT INTO t1 VALUES("
diff --git a/test/misc8.test b/test/misc8.test
index 3ff52e56..b639dd14 100644
--- a/test/misc8.test
+++ b/test/misc8.test
@@ -94,5 +94,44 @@ do_execsql_test misc8-2.1 {
   0 10 {} 10 {} {}
 }
 
+# 2016-02-26: An assertion fault found by the libFuzzer project
+#
+do_execsql_test misc8-3.0 {
+  SELECT *
+    FROM
+         (
+           (SELECT 0 AS i) AS x1,
+           (SELECT 1) AS x2
+         ) AS x3,
+         (SELECT 6 AS j UNION ALL SELECT 7) AS x4
+   WHERE i0} {
+        faultsim_restore_and_reopen $prefix
+        foreach f [glob ${prefix}-mj*] { forcedelete $f }
+      } else {
+        db close
+        sqlite3 db $prefix
+      }
       execsql "ATTACH '${prefix}2' AS aux"
+      glob -nocomplain ${prefix}-mj*
     } {}
     do_execsql_test pager1-4.4.$tn.8 {
       SELECT * FROM a
@@ -678,7 +735,6 @@ db close
 tv delete
 forcedelete $dirname
 
-
 # Set up a VFS to make a copy of the file-system just before deleting a
 # journal file to commit a transaction. The transaction modifies exactly
 # two database pages (and page 1 - the change counter).
@@ -1340,26 +1396,47 @@ do_test pager1-9.3.1 {
   execsql { PRAGMA page_size = 1024 }
   for {set ii 0} {$ii < 4} {incr ii} { execsql "CREATE TABLE t${ii}(a, b)" }
 } {}
-do_test pager1-9.3.2 {
-  sqlite3 db2 test.db2
-
-  execsql {
-    PRAGMA page_size = 4096;
-    PRAGMA synchronous = OFF;
-    CREATE TABLE t1(a, b);
-    CREATE TABLE t2(a, b);
-  } db2
-
-  sqlite3_backup B db2 main db main
-  B step 30
-  list [B step 10000] [B finish]
-} {SQLITE_DONE SQLITE_OK}
-do_test pager1-9.3.3 {
-  db2 close
-  db close
-  tv delete
-  file size test.db2
-} [file size test.db]
+if {[nonzero_reserved_bytes]} {
+  # backup with a page size changes is not possible with the codec
+  #
+  do_test pager1-9.3.2codec {
+    sqlite3 db2 test.db2
+    execsql {
+      PRAGMA page_size = 4096;
+      PRAGMA synchronous = OFF;
+      CREATE TABLE t1(a, b);
+      CREATE TABLE t2(a, b);
+    } db2
+    sqlite3_backup B db2 main db main
+    B step 30
+    list [B step 10000] [B finish]
+  } {SQLITE_READONLY SQLITE_READONLY}
+  do_test pager1-9.3.3codec {
+    db2 close
+    db close
+    tv delete
+    file size test.db2
+  } [file size test.db2]
+} else {
+  do_test pager1-9.3.2 {
+    sqlite3 db2 test.db2
+    execsql {
+      PRAGMA page_size = 4096;
+      PRAGMA synchronous = OFF;
+      CREATE TABLE t1(a, b);
+      CREATE TABLE t2(a, b);
+    } db2
+    sqlite3_backup B db2 main db main
+    B step 30
+    list [B step 10000] [B finish]
+  } {SQLITE_DONE SQLITE_OK}
+  do_test pager1-9.3.3 {
+    db2 close
+    db close
+    tv delete
+    file size test.db2
+  } [file size test.db]
+}
 
 do_test pager1-9.4.1 {
   faultsim_delete_and_reopen
@@ -2391,13 +2468,23 @@ do_test pager1-29.1 {
   }
   file size test.db
 } [expr 1024*3]
-do_test pager1-29.2 {
-  execsql {
-    PRAGMA page_size = 4096;
-    VACUUM;
-  }
-  file size test.db
-} [expr 4096*3]
+if {[nonzero_reserved_bytes]} {
+  # VACUUM with size changes is not possible with the codec.
+  do_test pager1-29.2 {
+    catchsql {
+      PRAGMA page_size = 4096;
+      VACUUM;
+    }
+  } {1 {attempt to write a readonly database}}
+} else {
+  do_test pager1-29.2 {
+    execsql {
+      PRAGMA page_size = 4096;
+      VACUUM;
+    }
+    file size test.db
+  } [expr 4096*3]
+}
 
 #-------------------------------------------------------------------------
 # Test that if an empty database file (size 0 bytes) is opened in 
diff --git a/test/pagerfault.test b/test/pagerfault.test
index c0f5de69..2e70171b 100644
--- a/test/pagerfault.test
+++ b/test/pagerfault.test
@@ -684,22 +684,24 @@ do_faultsim_test pagerfault-14a -prep {
 # is not possible to change the page-size of an in-memory database. Even
 # using the backup API.
 #
-if {$TEMP_STORE<2} {
-  do_faultsim_test pagerfault-14b -prep {
-    catch { db2 close }
-    faultsim_restore_and_reopen
+# Update: It is no longer possible to change the page size of any temp
+# database after it has been created.
+#
+do_faultsim_test pagerfault-14b -prep {
+  catch { db2 close }
+  faultsim_restore_and_reopen
     sqlite3 db2 ""
     db2 eval { PRAGMA page_size = 4096; CREATE TABLE xx(a) }
-  } -body {
-    sqlite3_backup B db2 main db main
-    B step 200
-    set rc [B finish]
-    if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR}
-    if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] }
-    set {} {}
-  } -test {
-    faultsim_test_result {0 {}} {1 {sqlite3_backup_init() failed}}
-  }
+} -body {
+  sqlite3_backup B db2 main db main
+  B step 200
+  set rc [B finish]
+  if {[string match SQLITE_IOERR_* $rc]} {set rc SQLITE_IOERR}
+  if {$rc != "SQLITE_OK"} { error [sqlite3_test_errstr $rc] }
+  set {} {}
+} -test {
+  faultsim_test_result {1 {attempt to write a readonly database}} \
+                       {1 {sqlite3_backup_init() failed}}
 }
 
 do_faultsim_test pagerfault-14c -prep {
diff --git a/test/pageropt.test b/test/pageropt.test
index 7191661b..f58b8ee7 100644
--- a/test/pageropt.test
+++ b/test/pageropt.test
@@ -16,12 +16,17 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+do_not_use_codec
 
 ifcapable {!pager_pragmas||secure_delete||direct_read} {
   finish_test
   return
 }
 
+# A non-zero reserved_bytes value changes the number of pages in the 
+# database file, which messes up the results in this test.
+if {[nonzero_reserved_bytes]} {finish_test; return;}
+
 # Run the SQL statement supplied by the argument and return
 # the results.  Prepend four integers to the beginning of the
 # result which are
diff --git a/test/parser1.test b/test/parser1.test
index 78c1a40c..c708dded 100644
--- a/test/parser1.test
+++ b/test/parser1.test
@@ -76,4 +76,27 @@ do_catchsql_test parser1-2.2 {
   SELECT x FROM c;
 } {1 {syntax error after column name "x"}}
 
+# Verify that the comma between multiple table constraints is
+# optional.
+#
+# The missing comma is technically a syntax error.  But we have to support
+# it because there might be legacy databases that omit the commas in their
+# sqlite_master tables.
+#
+do_execsql_test parser1-3.1 {
+  CREATE TABLE t300(id INTEGER PRIMARY KEY);
+  CREATE TABLE t301(
+    id INTEGER PRIMARY KEY,
+    c1 INTEGER NOT NULL,
+    c2 INTEGER NOT NULL,
+    c3 BOOLEAN NOT NULL DEFAULT 0,
+    FOREIGN KEY(c1) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
+        /* no comma */
+    FOREIGN KEY(c2) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT
+        /* no comma */
+    UNIQUE(c1, c2)
+  );
+  PRAGMA foreign_key_list(t301);
+} {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE}
+
 finish_test
diff --git a/test/permutations.test b/test/permutations.test
index c0998890..f8880da2 100644
--- a/test/permutations.test
+++ b/test/permutations.test
@@ -86,7 +86,13 @@ proc test_set {args} {
 #
 set alltests [list]
 foreach f [glob $testdir/*.test] { lappend alltests [file tail $f] }
-foreach f [glob -nocomplain $testdir/../ext/rtree/*.test] { 
+foreach f [glob -nocomplain       \
+    $testdir/../ext/rtree/*.test  \
+    $testdir/../ext/fts5/test/*.test   \
+] { 
+  lappend alltests $f 
+}
+foreach f [glob -nocomplain $testdir/../ext/session/*.test] { 
   lappend alltests $f 
 }
 
@@ -97,6 +103,7 @@ set alltests [test_set $alltests -exclude {
   all.test        async.test         quick.test  veryquick.test
   memleak.test    permutations.test  soak.test   fts3.test
   mallocAll.test  rtree.test         full.test   extraquick.test
+  session.test
 }]
 
 set allquicktests [test_set $alltests -exclude {
@@ -157,7 +164,8 @@ test_suite "veryquick" -prefix "" -description {
   This test suite is the same as the "quick" tests, except that some files
   that test malloc and IO errors are omitted.
 } -files [
-  test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err*
+  test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* \
+      *fts5corrupt* *fts5big* *fts5aj*
 ]
 
 test_suite "extraquick" -prefix "" -description {
@@ -612,6 +620,7 @@ test_suite "utf16" -description {
     trace.test trigger1.test trigger2.test trigger3.test
     trigger4.test types2.test types.test unique.test update.test
     vacuum.test view.test where.test
+    bestindex1.test
 }
 
 # Run some tests in exclusive locking mode.
@@ -719,7 +728,7 @@ test_suite "inmemory_journal" -description {
   ioerr.test ioerr2.test ioerr3.test ioerr4.test ioerr5.test
   vacuum3.test incrblob_err.test diskfull.test backup_ioerr.test
   e_fts3.test fts3cov.test fts3malloc.test fts3rnd.test
-  fts3snippet.test mmapfault.test
+  fts3snippet.test mmapfault.test sessionfault.test sessionfault2.test
 
   # Exclude test scripts that use tcl IO to access journal files or count
   # the number of fsync() calls.
@@ -737,6 +746,14 @@ test_suite "inmemory_journal" -description {
 
   # WAL mode is different.
   wal* tkt-2d1a5c67d.test backcompat.test e_wal* rowallock.test
+
+  # This test does not work as the "PRAGMA journal_mode = memory"
+  # statement switches the database out of wal mode at inopportune
+  # times.
+  snapshot_fault.test
+  
+  # This test assumes a journal file is created on disk.
+  delete_db.test
 }]
 
 ifcapable mem3 {
@@ -933,7 +950,10 @@ test_suite "journaltest" -description {
   unregister_jt_vfs
 } -files [test_set $::allquicktests -exclude {
   wal* incrvacuum.test ioerr.test corrupt4.test io.test crash8.test 
-  async4.test bigfile.test backcompat.test
+  async4.test bigfile.test backcompat.test e_wal* fstat.test mmap2.test
+  pager1.test syscall.test tkt3457.test *malloc* mmap* multiplex* nolock*
+  pager2.test *fault* rowal* snapshot* superlock* symlink.test
+  delete_db.test
 }]
 
 if {[info commands register_demovfs] != ""} {
@@ -970,6 +990,26 @@ test_suite "rtree" -description {
   All R-tree related tests. Provides coverage of source file rtree.c.
 } -files [glob -nocomplain $::testdir/../ext/rtree/*.test]
 
+test_suite "session" -description {
+  All session module related tests. 
+} -files [glob -nocomplain $::testdir/../ext/session/*.test]
+
+test_suite "session_eec" -description {
+  All session module related tests with sqlite3_extended_result_codes() set. 
+} -files [
+  glob -nocomplain $::testdir/../ext/session/*.test
+] -dbconfig {
+  sqlite3_extended_result_codes $::dbhandle 1
+}
+
+test_suite "session_strm" -description {
+  All session module related tests using the streaming APIs.
+} -files [
+  glob -nocomplain $::testdir/../ext/session/*.test
+] -dbconfig {
+  set ::sqlite3session_streams 1
+}
+
 test_suite "rbu" -description {
   RBU tests.
 } -files [
@@ -1021,15 +1061,13 @@ proc run_tests {name args} {
   set ::G(isquick)           1
   set ::G(perm:dbconfig)     $options(-dbconfig)
 
-  uplevel $options(-initialize)
-
   foreach file [lsort $options(-files)] {
+    uplevel $options(-initialize)
     if {[file tail $file] == $file} { set file [file join $::testdir $file] }
     slave_test_file $file
+    uplevel $options(-shutdown)
   }
 
-  uplevel $options(-shutdown)
-
   unset ::G(perm:name)
   unset ::G(perm:prefix)
   unset ::G(perm:presql)
@@ -1063,12 +1101,12 @@ proc help {} {
   exit -1
 }
 
-if {[info script] == $argv0} {
+if {[file tail $argv0] == "permutations.test"} {
   proc main {argv} {
     if {[llength $argv]==0} {
       help
     } else {
-      set suite [lindex $argv 0]
+      set suite [file tail [lindex $argv 0]]
       if {[info exists ::testspec($suite)]==0} help
       set extra ""
       if {[llength $argv]>1} { set extra [list -files [lrange $argv 1 end]] }
diff --git a/test/pragma.test b/test/pragma.test
index befa5cf5..44498d07 100644
--- a/test/pragma.test
+++ b/test/pragma.test
@@ -628,10 +628,10 @@ ifcapable tempdb&&attach {
 }
 do_test pragma-6.2 {
   execsql {
-    CREATE TABLE t2(a,b,c);
+    CREATE TABLE t2(a TYPE_X, b [TYPE_Y], c "TYPE_Z");
     pragma table_info(t2)
   }
-} {0 a {} 0 {} 0 1 b {} 0 {} 0 2 c {} 0 {} 0}
+} {0 a TYPE_X 0 {} 0 1 b TYPE_Y 0 {} 0 2 c TYPE_Z 0 {} 0}
 do_test pragma-6.2.1 {
   execsql {
     pragma table_info;
@@ -1083,7 +1083,20 @@ do_test pragma-8.2.15 {
 # "memory" or "disk" as appropriate.
 #
 proc check_temp_store {} {
-  db eval {CREATE TEMP TABLE IF NOT EXISTS a(b)}
+  db eval {
+    PRAGMA temp.cache_size = 1;
+    CREATE TEMP TABLE IF NOT EXISTS a(b);
+    DELETE FROM a;
+    INSERT INTO a VALUES(randomblob(1000));
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+    INSERT INTO a SELECT * FROM a;
+  }
   db eval {PRAGMA database_list} {
     if {$name=="temp"} {
       set bt [btree_from_db db 1]
@@ -1741,73 +1754,75 @@ forcedelete data_dir
 } ;# endif windows
 
 database_may_be_corrupt
+if {![nonzero_reserved_bytes]} {
 
-do_test 21.1 {
-  # Create a corrupt database in testerr.db. And a non-corrupt at test.db.
-  #
-  db close
-  forcedelete test.db
-  sqlite3 db test.db
-  execsql { 
-    PRAGMA page_size = 1024;
-    PRAGMA auto_vacuum = 0;
-    CREATE TABLE t1(a PRIMARY KEY, b);
-    INSERT INTO t1 VALUES(1, 1);
-  }
-  for {set i 0} {$i < 10} {incr i} {
-    execsql { INSERT INTO t1 SELECT a + (1 << $i), b + (1 << $i) FROM t1 }
-  }
-  db close
-  forcecopy test.db testerr.db
-  hexio_write testerr.db 15000 [string repeat 55 100]
-} {100}
-
-set mainerr {*** in database main ***
+  do_test 21.1 {
+    # Create a corrupt database in testerr.db. And a non-corrupt at test.db.
+    #
+    db close
+    forcedelete test.db
+    sqlite3 db test.db
+    execsql { 
+      PRAGMA page_size = 1024;
+      PRAGMA auto_vacuum = 0;
+      CREATE TABLE t1(a PRIMARY KEY, b);
+      INSERT INTO t1 VALUES(1, 1);
+    }
+    for {set i 0} {$i < 10} {incr i} {
+      execsql { INSERT INTO t1 SELECT a + (1 << $i), b + (1 << $i) FROM t1 }
+    }
+    db close
+    forcecopy test.db testerr.db
+    hexio_write testerr.db 15000 [string repeat 55 100]
+  } {100}
+  
+  set mainerr {*** in database main ***
 Multiple uses for byte 672 of page 15}
-set auxerr {*** in database aux ***
+  set auxerr {*** in database aux ***
 Multiple uses for byte 672 of page 15}
-
-set mainerr {/{\*\*\* in database main \*\*\*
+  
+  set mainerr {/{\*\*\* in database main \*\*\*
 Multiple uses for byte 672 of page 15}.*/}
-set auxerr {/{\*\*\* in database aux \*\*\*
+  set auxerr {/{\*\*\* in database aux \*\*\*
 Multiple uses for byte 672 of page 15}.*/}
-
-do_test 22.2 {
-  catch { db close }
-  sqlite3 db testerr.db
-  execsql { PRAGMA integrity_check }
-} $mainerr
-
-do_test 22.3.1 {
-  catch { db close }
-  sqlite3 db test.db
-  execsql { 
-    ATTACH 'testerr.db' AS 'aux';
-    PRAGMA integrity_check;
-  }
-} $auxerr
-do_test 22.3.2 {
-  execsql { PRAGMA main.integrity_check; }
-} {ok}
-do_test 22.3.3 {
-  execsql { PRAGMA aux.integrity_check; }
-} $auxerr
-
-do_test 22.4.1 {
-  catch { db close }
-  sqlite3 db testerr.db
-  execsql { 
-    ATTACH 'test.db' AS 'aux';
-    PRAGMA integrity_check;
-  }
-} $mainerr
-do_test 22.4.2 {
-  execsql { PRAGMA main.integrity_check; }
-} $mainerr
-do_test 22.4.3 {
-  execsql { PRAGMA aux.integrity_check; }
-} {ok}
-
+  
+  do_test 22.2 {
+    catch { db close }
+    sqlite3 db testerr.db
+    execsql { PRAGMA integrity_check }
+  } $mainerr
+  
+  do_test 22.3.1 {
+    catch { db close }
+    sqlite3 db test.db
+    execsql { 
+      ATTACH 'testerr.db' AS 'aux';
+      PRAGMA integrity_check;
+    }
+  } $auxerr
+  do_test 22.3.2 {
+    execsql { PRAGMA main.integrity_check; }
+  } {ok}
+  do_test 22.3.3 {
+    execsql { PRAGMA aux.integrity_check; }
+  } $auxerr
+  
+  do_test 22.4.1 {
+    catch { db close }
+    sqlite3 db testerr.db
+    execsql { 
+      ATTACH 'test.db' AS 'aux';
+      PRAGMA integrity_check;
+    }
+  } $mainerr
+  do_test 22.4.2 {
+    execsql { PRAGMA main.integrity_check; }
+  } $mainerr
+  do_test 22.4.3 {
+    execsql { PRAGMA aux.integrity_check; }
+  } {ok}
+}
+  
 db close
 forcedelete test.db test.db-wal test.db-journal
 sqlite3 db test.db
diff --git a/test/pragma3.test b/test/pragma3.test
index b7ea4d3f..8a4f4f9d 100644
--- a/test/pragma3.test
+++ b/test/pragma3.test
@@ -15,6 +15,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+do_not_use_codec
 
 do_execsql_test pragma3-100 {
   PRAGMA data_version;
@@ -220,7 +221,7 @@ ifcapable shared_cache {
 # This will not work with the in-memory journal permutation, as opening
 # [db2] switches the journal mode back to "memory"
 #
-ifcapable wal {
+if {[wal_is_capable]} {
 if {[permutation]!="inmemory_journal"} {
 
   sqlite3 db test.db
diff --git a/test/quota.test b/test/quota.test
index 7af55a83..5d0bda3a 100644
--- a/test/quota.test
+++ b/test/quota.test
@@ -527,4 +527,6 @@ do_faultsim_test quota-5.6 -prep {
 }
 
 catch { sqlite3_quota_shutdown }
+catch { db close }
+forcedelete test.db
 finish_test
diff --git a/test/regexp2.test b/test/regexp2.test
new file mode 100644
index 00000000..9928e8ce
--- /dev/null
+++ b/test/regexp2.test
@@ -0,0 +1,125 @@
+# 2016 February 19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file implements tests for the REGEXP operator in ext/misc/regexp.c.
+# It focuses on the use of the sqlite3_set_auxdata()/get_auxdata() APIs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix regexp2
+
+load_static_extension db regexp
+
+#-------------------------------------------------------------------------
+# Test that triggers do not become confused and use aux-data created by
+# a different trigger for a different REGEXP invocation.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE TABLE x1(x, y, z);
+  CREATE TABLE x2(x, y, z);
+
+  CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN
+    INSERT INTO x1 VALUES(
+        new.a REGEXP 'abc',
+        new.b REGEXP 'abc',
+        new.c REGEXP 'abc'
+    );
+  END;
+
+  CREATE TRIGGER tr2 AFTER INSERT ON t1 BEGIN
+    INSERT INTO x2 VALUES(
+        new.a REGEXP 'def',
+        new.b REGEXP 'def',
+        new.c REGEXP 'def'
+    );
+  END;
+
+  INSERT INTO t1 VALUES('abc', 'def', 'abc');
+  SELECT * FROM t1;
+} {abc def abc}
+
+do_execsql_test 1.1 { SELECT * FROM x1 } {1 0 1}
+do_execsql_test 1.2 { SELECT * FROM x2 } {0 1 0}
+
+#-------------------------------------------------------------------------
+# Test that if an exception is thrown several triggers deep, all aux-data
+# objects are cleaned up correctly.
+#
+proc sql_error {} {
+  error "SQL error!"
+}
+db func error sql_error
+do_execsql_test 2.0 {
+  CREATE TABLE t2(a, b);
+  CREATE TABLE t3(c, d);
+  CREATE TABLE t4(e, f);
+
+  CREATE TRIGGER t2_tr1 AFTER UPDATE ON t2 BEGIN
+    UPDATE t3 SET d = new.b WHERE c = old.a;
+  END;
+
+  CREATE TRIGGER t3_tr1 AFTER UPDATE ON t3 BEGIN
+    UPDATE t4 SET f = new.d WHERE e = old.c AND new.d REGEXP 'a.*';
+  END;
+
+  CREATE TRIGGER t4_tr1 AFTER UPDATE ON t4 BEGIN
+    SELECT CASE WHEN new.f REGEXP '.*y.*' THEN error() ELSE 1 END;
+  END;
+
+  INSERT INTO t2 VALUES(1, 'a_x_1');
+  INSERT INTO t2 VALUES(2, 'a_y_1');
+
+  INSERT INTO t3 VALUES(1, 'b1');
+  INSERT INTO t3 VALUES(2, 'b2');
+
+  INSERT INTO t4 VALUES(1, 'b1');
+  INSERT INTO t4 VALUES(2, 'b2');
+} {}
+
+do_catchsql_test 2.1 {
+  UPDATE t2 SET a=a+1 WHERE b REGEXP 'a.*' AND b REGEXP '.*1';
+} {1 {SQL error!}}
+
+# Test that the triggers used in the test above work as expected.
+#
+do_execsql_test 2.2 {
+  UPDATE t2 SET b = 'a_abc_1';
+} {}
+do_execsql_test 2.3 {
+  SELECT * FROM t2;
+  SELECT * FROM t3;
+  SELECT * FROM t4;
+} {1 a_abc_1 2 a_abc_1 1 a_abc_1 2 a_abc_1 1 a_abc_1 2 a_abc_1}
+
+#-------------------------------------------------------------------------
+# Test that trigger parameters (i.e. new.* and old.*) refs are not 
+# considered to be constant across separate invocations of the trigger.
+#
+do_execsql_test 3.0 {
+  CREATE TABLE t5(a);
+  CREATE TABLE t6(x);
+
+  CREATE TRIGGER t5tr AFTER DELETE ON t5 BEGIN
+    DELETE FROM t6 WHERE t6.x REGEXP old.a;
+  END;
+
+  INSERT INTO t5 VALUES ('^a.*'), ('^b.*'), ('^c.*');
+  INSERT INTO t6 VALUES ('eab'), ('abc'), ('bcd'), ('cde'), ('dea');
+
+  DELETE FROM t5;
+  SELECT * FROM t6;
+} {eab dea}
+
+
+finish_test
+
diff --git a/test/releasetest.tcl b/test/releasetest.tcl
old mode 100644
new mode 100755
index bb902eec..99823e3b
--- a/test/releasetest.tcl
+++ b/test/releasetest.tcl
@@ -9,21 +9,19 @@ This Tcl script is used to test the various configurations required
 before releasing a new version. Supported command line options (all
 optional) are:
 
-    --srcdir   TOP-OF-SQLITE-TREE      (see below)
-    --platform PLATFORM                (see below)
+    --buildonly                        (Just build testfixture - do not run)
     --config   CONFIGNAME              (Run only CONFIGNAME)
+    --dryrun                           (Print what would have happened)
+    -f|--force                         (Run even if uncommitted changes)
+    --info                             (Show diagnostic info)
+    --jobs     N                       (Use N processes - default 1)
+    --keep                             (Delete no files after each test run)
+    --msvc                             (Use MSVC as the compiler)
+    --platform PLATFORM                (see below)
+    --progress                         (Show progress messages)
     --quick                            (Run "veryquick.test" only)
     --veryquick                        (Run "make smoketest" only)
-    --msvc                             (Use MSVC as the compiler)
-    --buildonly                        (Just build testfixture - do not run)
-    --dryrun                           (Print what would have happened)
-    --info                             (Show diagnostic info)
     --with-tcl=DIR                     (Use TCL build at DIR)
-    --jobs     N                       (Use N processes - default 1)
-    --progress                         (Show progress messages)
-
-The default value for --srcdir is the parent of the directory holding
-this script.
 
 The script determines the default value for --platform using the
 $tcl_platform(os) and $tcl_platform(machine) variables.  Supported
@@ -51,10 +49,16 @@ array set ::Configs [strip_comments {
   "Default" {
     -O2
     --disable-amalgamation --disable-shared
+    --enable-session
   }
   "Sanitize" {
     CC=clang -fsanitize=undefined
     -DSQLITE_ENABLE_STAT4
+    --enable-session
+  }
+  "Stdcall" {
+    -DUSE_STDCALL=1
+    -O2
   }
   "Have-Not" {
     # The "Have-Not" configuration sets all possible -UHAVE_feature options
@@ -106,7 +110,7 @@ array set ::Configs [strip_comments {
     -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1
     -DSQLITE_ENABLE_STAT4
     -DSQLITE_ENABLE_STMT_SCANSTATUS
-    --enable-json1 --enable-fts5
+    --enable-json1 --enable-fts5 --enable-session
   }
   "Debug-One" {
     --disable-shared
@@ -118,7 +122,6 @@ array set ::Configs [strip_comments {
     -DSQLITE_ENABLE_FTS3=1
     -DSQLITE_ENABLE_RTREE=1
     -DSQLITE_ENABLE_MEMSYS5=1
-    -DSQLITE_ENABLE_MEMSYS3=1
     -DSQLITE_ENABLE_COLUMN_METADATA=1
     -DSQLITE_ENABLE_STAT4
     -DSQLITE_ENABLE_HIDDEN_COLUMNS
@@ -132,6 +135,7 @@ array set ::Configs [strip_comments {
     -DSQLITE_ENABLE_RBU
     -DSQLITE_MAX_ATTACHED=125
     -DLONGDOUBLE_TYPE=double
+    --enable-session
   }
   "Device-One" {
     -O2
@@ -168,30 +172,53 @@ array set ::Configs [strip_comments {
     -DSQLITE_OMIT_TRACE=1
     -DSQLITE_TEMP_STORE=3
     -DSQLITE_THREADSAFE=2
-    --enable-json1 --enable-fts5
+    --enable-json1 --enable-fts5 --enable-session
   }
   "Locking-Style" {
     -O2
     -DSQLITE_ENABLE_LOCKING_STYLE=1
   }
-  "OS-X" {
+  "Apple" {
     -O1   # Avoid a compiler bug in gcc 4.2.1 build 5658
-    -DSQLITE_OMIT_LOAD_EXTENSION=1
-    -DSQLITE_DEFAULT_MEMSTATUS=0
-    -DSQLITE_THREADSAFE=2
-    -DSQLITE_OS_UNIX=1
-    -DSQLITE_ENABLE_JSON1=1
-    -DSQLITE_ENABLE_LOCKING_STYLE=1
-    -DUSE_PREAD=1
-    -DSQLITE_ENABLE_RTREE=1
+    -DHAVE_GMTIME_R=1
+    -DHAVE_ISNAN=1
+    -DHAVE_LOCALTIME_R=1
+    -DHAVE_PREAD=1
+    -DHAVE_PWRITE=1
+    -DHAVE_USLEEP=1
+    -DHAVE_USLEEP=1
+    -DHAVE_UTIME=1
+    -DSQLITE_DEFAULT_CACHE_SIZE=1000
+    -DSQLITE_DEFAULT_CKPTFULLFSYNC=1
+    -DSQLITE_DEFAULT_MEMSTATUS=1
+    -DSQLITE_DEFAULT_PAGE_SIZE=1024
+    -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS=1
+    -DSQLITE_ENABLE_API_ARMOR=1
+    -DSQLITE_ENABLE_AUTO_PROFILE=1
+    -DSQLITE_ENABLE_FLOCKTIMEOUT=1
     -DSQLITE_ENABLE_FTS3=1
     -DSQLITE_ENABLE_FTS3_PARENTHESIS=1
-    -DSQLITE_DEFAULT_CACHE_SIZE=1000
+    -DSQLITE_ENABLE_FTS3_TOKENIZER=1
+    if:os=="Darwin" -DSQLITE_ENABLE_LOCKING_STYLE=1
+    -DSQLITE_ENABLE_PERSIST_WAL=1
+    -DSQLITE_ENABLE_PURGEABLE_PCACHE=1
+    -DSQLITE_ENABLE_RTREE=1
+    -DSQLITE_ENABLE_SNAPSHOT=1
+    # -DSQLITE_ENABLE_SQLLOG=1
+    -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
     -DSQLITE_MAX_LENGTH=2147483645
     -DSQLITE_MAX_VARIABLE_NUMBER=500000
-    -DSQLITE_DEBUG=1
+    # -DSQLITE_MEMDEBUG=1
+    -DSQLITE_NO_SYNC=1
+    -DSQLITE_OMIT_AUTORESET=1
+    -DSQLITE_OMIT_LOAD_EXTENSION=1
     -DSQLITE_PREFER_PROXY_LOCKING=1
-    -DSQLITE_ENABLE_API_ARMOR=1
+    -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
+    -DSQLITE_THREADSAFE=2
+    -DSQLITE_USE_URI=1
+    -DSQLITE_WRITE_WALFRAME_PREBUFFERED=1
+    -DUSE_GUARDED_FD=1
+    -DUSE_PREAD=1
     --enable-json1 --enable-fts5
   }
   "Extra-Robustness" {
@@ -203,6 +230,7 @@ array set ::Configs [strip_comments {
     -DSQLITE_MAX_ATTACHED=30
     -DSQLITE_ENABLE_COLUMN_METADATA
     -DSQLITE_ENABLE_FTS4
+    -DSQLITE_ENABLE_FTS5
     -DSQLITE_ENABLE_FTS4_PARENTHESIS
     -DSQLITE_DISABLE_FTS4_DEFERRED
     -DSQLITE_ENABLE_RTREE
@@ -247,6 +275,7 @@ array set ::Platforms [strip_comments {
     "Device-Two"              test
     "No-lookaside"            test
     "Devkit"                  test
+    "Apple"                   test
     "Sanitize"                {QUICKTEST_OMIT=func4.test,nan.test test}
     "Device-One"              fulltest
     "Default"                 "threadtest fulltest"
@@ -263,18 +292,20 @@ array set ::Platforms [strip_comments {
   Darwin-i386 {
     "Locking-Style"           "mptest test"
     "Have-Not"                test
-    "OS-X"                    "threadtest fulltest"
+    "Apple"                   "threadtest fulltest"
   }
   Darwin-x86_64 {
     "Locking-Style"           "mptest test"
     "Have-Not"                test
-    "OS-X"                    "threadtest fulltest"
+    "Apple"                   "threadtest fulltest"
   }
   "Windows NT-intel" {
+    "Stdcall"                 test
     "Have-Not"                test
     "Default"                 "mptest fulltestonly"
   }
   "Windows NT-amd64" {
+    "Stdcall"                 test
     "Have-Not"                test
     "Default"                 "mptest fulltestonly"
   }
@@ -437,14 +468,15 @@ proc count_tests_and_errors {logfile rcVar errmsgVar} {
 proc run_slave_test {} {
   # Read global vars configuration from stdin.
   set V [gets stdin]
-  foreach {::TRACE ::MSVC ::DRYRUN} $V {}
+  foreach {::TRACE ::MSVC ::DRYRUN ::KEEPFILES} $V {}
 
   # Read the test-suite configuration from stdin.
   set T [gets stdin]
   foreach {title dir configOpts testtarget makeOpts cflags opts} $T {}
 
   # Create and switch to the test directory.
-  set ::env(SQLITE_TMPDIR) [file normalize $dir]
+  set normaldir [file normalize $dir]
+  set ::env(SQLITE_TMPDIR) $normaldir
   trace_cmd file mkdir $dir
   trace_cmd cd $dir
   catch {file delete core}
@@ -468,6 +500,9 @@ proc run_slave_test {} {
     }
   }
 
+  # Clean up lots of extra files if --keep was not specified.
+  if {$::KEEPFILES==0} { cleanup $normaldir }
+
   # Exis successfully if the test passed, or with a non-zero error code
   # otherwise.
   exit $rc
@@ -567,7 +602,7 @@ proc run_all_test_suites {alltests} {
       set fd [open "|[info nameofexecutable] $script --slave" r+]
       fconfigure $fd -blocking 0
       fileevent $fd readable [list slave_fileevent $fd $T $tm1]
-      puts $fd [list $::TRACE $::MSVC $::DRYRUN]
+      puts $fd [list $::TRACE $::MSVC $::DRYRUN $::KEEPFILES]
       puts $fd [list {*}$T]
       flush $fd
     }
@@ -587,13 +622,23 @@ proc add_test_suite {listvar name testtarget config} {
   set opts ""
   set title ${name}($testtarget)
   set configOpts $::WITHTCL
+  set skip 0
 
   regsub -all {#[^\n]*\n} $config \n config
   foreach arg $config {
+    if {$skip} {
+      set skip 0
+      continue
+    }
     if {[regexp {^-[UD]} $arg]} {
       lappend opts $arg
     } elseif {[regexp {^[A-Z]+=} $arg]} {
       lappend testtarget $arg
+    } elseif {[regexp {^if:([a-z]+)(.*)} $arg all key tail]} {
+      # Arguments of the form 'if:os=="Linux"' will cause the subsequent
+      # argument to be skipped if the $tcl_platform(os) is not "Linux", for
+      # example...
+      set skip [expr !(\$::tcl_platform($key)$tail)]
     } elseif {[regexp {^--(enable|disable)-} $arg]} {
       if {$::MSVC} {
         if {$arg eq "--disable-amalgamation"} {
@@ -688,6 +733,9 @@ proc makeCommand { targets makeOpts cflags opts } {
     set nmakeDir [file nativename $::SRCDIR]
     set nmakeFile [file nativename [file join $nmakeDir Makefile.msc]]
     lappend result nmake /f $nmakeFile TOP=$nmakeDir
+    if {[regexp {USE_STDCALL=1} $cflags]} {
+      lappend result USE_STDCALL=1
+    }
   } else {
     lappend result make
   }
@@ -732,6 +780,8 @@ proc process_options {argv} {
   set ::JOBS           1
   set ::PROGRESS_MSGS  0
   set ::WITHTCL        {}
+  set ::FORCE          0
+  set ::KEEPFILES      0          ;# Keep extra files after test run
   set config {}
   set platform $::tcl_platform(os)-$::tcl_platform(machine)
 
@@ -744,6 +794,12 @@ proc process_options {argv} {
         exit
       }
 
+      # Undocumented legacy option: --srcdir DIRECTORY
+      #
+      # DIRECTORY is the root of the SQLite checkout.  This sets the
+      # SRCDIR global variable.  But that variable is already set
+      # automatically so there really is no reason to have this option.
+      #
       -srcdir {
         incr i
         set ::SRCDIR [file normalize [lindex $argv $i]]
@@ -787,6 +843,11 @@ proc process_options {argv} {
         set ::DRYRUN 1
       }
 
+      -force -
+      -f {
+        set ::FORCE 1
+      }
+
       -trace {
         set ::TRACE 1
       }
@@ -819,6 +880,10 @@ proc process_options {argv} {
         lappend ::EXTRACONFIG [lindex $argv $i]
       }
 
+      -keep {
+        set ::KEEPFILES 1
+      }
+
       -with-tcl=* {
         set ::WITHTCL -$x
       }
@@ -880,6 +945,47 @@ proc process_options {argv} {
   PUTS ""
 }
 
+# Check to see if there are uncommitted changes in the SQLite source
+# checkout.  Exit if there are.  Except:  Do nothing if the --force
+# flag is used.  Also, ignore this test if the fossil binary is
+# unavailable, or if the source tree is not a valid fossil checkout.
+#
+proc check_uncommitted {} {
+  if {$::FORCE} return
+  set pwd [pwd]
+  cd $::SRCDIR
+  if {[catch {exec fossil changes} res]==0 && [string trim $res]!=""} {
+    puts "ERROR: The check-out contains uncommitted changes:"
+    puts $res
+    puts "Use the -f or --force options to override"
+    exit 1
+  }
+  cd $pwd
+}
+
+# A test run has just finished in directory $dir. This command deletes all
+# non-essential files from the directory. Specifically, everything except
+#
+#   * The "testfixture" and "sqlite3" binaries,
+#   * The "test-out.log" and "test.log" log files.
+#
+proc cleanup {dir} {
+  set K(testfixture) 1
+  set K(testfixture.exe) 1
+  set K(sqlite3) 1
+  set K(sqlite3.exe) 1
+  set K(test-out.txt) 1
+  set K(test.log) 1
+
+  foreach f [glob -nocomplain [file join $dir *]] {
+    set tail [file tail $f]
+    if {[info exists K($tail)]==0} { 
+      file delete -force $f
+    }
+  }
+}
+
+
 # Main routine.
 #
 proc main {argv} {
@@ -887,6 +993,7 @@ proc main {argv} {
   # Process any command line options.
   set ::EXTRACONFIG {}
   process_options $argv
+  if {!$::DRYRUN} check_uncommitted
   PUTS [string repeat * 79]
 
   set ::NERR 0
diff --git a/test/rowvalue.test b/test/rowvalue.test
new file mode 100644
index 00000000..0df2e734
--- /dev/null
+++ b/test/rowvalue.test
@@ -0,0 +1,296 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the SELECT statement.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue
+
+do_execsql_test 0.0 {
+  CREATE TABLE one(o);
+  INSERT INTO one VALUES(1);
+}
+
+foreach {tn v1 v2 eq ne is isnot} {
+  1 "1, 2, 3"    "1, 2, 3"                   1  0     1 0
+  2 "1, 0, 3"    "1, 2, 3"                   0  1     0 1
+  3 "1, 2, NULL" "1, 2, 3"                   {} {}    0 1
+  4 "1, 2, NULL" "1, 2, NULL"                {} {}    1 0
+  5 "NULL, NULL, NULL" "NULL, NULL, NULL"    {} {}    1 0
+
+  6 "1, NULL, 1" "1, 1, 1"                   {} {}    0 1
+  7 "1, NULL, 1" "1, 1, 2"                   0  1     0 1
+} {
+  do_execsql_test 1.$tn.eq "SELECT ($v1) == ($v2)" [list $eq]
+  do_execsql_test 1.$tn.ne "SELECT ($v1) != ($v2)" [list $ne]
+
+  do_execsql_test 1.$tn.is    "SELECT ($v1) IS ($v2)"     [list $is]
+  do_execsql_test 1.$tn.isnot "SELECT ($v1) IS NOT ($v2)" [list $isnot]
+
+  do_execsql_test 1.$tn.2.eq "SELECT (SELECT $v1) == (SELECT $v2)" [list $eq]
+  do_execsql_test 1.$tn.2.ne "SELECT (SELECT $v1) != (SELECT $v2)" [list $ne]
+}
+
+foreach {tn v1 v2 lt gt le ge} {
+  1 "(1, 1, 3)"    "(1, 2, 3)"                   1 0      1 0
+  2 "(1, 2, 3)"    "(1, 2, 3)"                   0 0      1 1
+  3 "(1, 3, 3)"    "(1, 2, 3)"                   0 1      0 1
+
+  4 "(1, NULL, 3)"    "(1, 2, 3)"                {} {}      {} {}
+  5 "(1, 3, 3)"    "(1, NULL, 3)"                {} {}      {} {}
+  6 "(1, NULL, 3)"    "(1, NULL, 3)"             {} {}      {} {}
+} {
+  foreach {tn2 expr res} [list \
+    2.$tn.lt "$v1 < $v2" $lt   \
+    2.$tn.gt "$v1 > $v2" $gt   \
+    2.$tn.le "$v1 <= $v2" $le   \
+    2.$tn.ge "$v1 >= $v2" $ge   \
+  ] {
+    do_execsql_test $tn2 "SELECT $expr" [list $res]
+
+    set map(0) [list]
+    set map() [list]
+    set map(1) [list 1]
+    do_execsql_test $tn2.where1 "SELECT * FROM one WHERE $expr" $map($res)
+
+    set map(0) [list 1]
+    set map() [list]
+    set map(1) [list]
+    do_execsql_test $tn2.where2 "SELECT * FROM one WHERE NOT $expr" $map($res)
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(1, 1);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(2, 3);
+  INSERT INTO t1 VALUES(2, 4);
+  INSERT INTO t1 VALUES(3, 5);
+  INSERT INTO t1 VALUES(3, 6);
+}
+
+foreach {tn r order} {
+  1 "(1, 1)"           "ORDER BY y"
+  2 "(1, 1)"           "ORDER BY x, y"
+  3 "(1, 2)"           "ORDER BY x, y DESC"
+  4 "(3, 6)"           "ORDER BY x DESC, y DESC"
+  5 "((3, 5))"         "ORDER BY x DESC, y"
+  6 "(SELECT 3, 5)"    "ORDER BY x DESC, y"
+} {
+  do_execsql_test 3.$tn.1 "SELECT $r == (SELECT x,y FROM t1 $order)" 1
+  do_execsql_test 3.$tn.2 "SELECT $r == (SELECT * FROM t1 $order)" 1
+
+  do_execsql_test 3.$tn.3 "
+    SELECT (SELECT * FROM t1 $order) == (SELECT * FROM t1 $order)
+  " 1
+  do_execsql_test 3.$tn.4 "
+    SELECT (SELECT 0, 0) == (SELECT * FROM t1 $order)
+  " 0
+}
+
+foreach {tn expr res} {
+  1 {(2, 2) BETWEEN (2, 2) AND (3, 3)} 1
+  2 {(2, 2) BETWEEN (2, NULL) AND (3, 3)} {}
+  3 {(2, 2) BETWEEN (3, NULL) AND (3, 3)} 0
+} {
+  do_execsql_test 4.$tn "SELECT $expr" [list $res]
+}
+
+foreach {tn expr res} {
+  1 {(2, 4) IN (SELECT * FROM t1)} 1
+  2 {(3, 4) IN (SELECT * FROM t1)} 0
+
+  3 {(NULL, 4) IN (SELECT * FROM t1)} {}
+  4 {(NULL, 0) IN (SELECT * FROM t1)} 0
+
+  5 {(NULL, 4) NOT IN (SELECT * FROM t1)} {}
+  6 {(NULL, 0) NOT IN (SELECT * FROM t1)} 1
+} {
+  do_execsql_test 5.$tn "SELECT $expr" [list $res]
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE hh(a, b, c);
+  INSERT INTO hh VALUES('abc', 1, 'i');
+  INSERT INTO hh VALUES('ABC', 1, 'ii');
+  INSERT INTO hh VALUES('def', 2, 'iii');
+  INSERT INTO hh VALUES('DEF', 2, 'iv');
+  INSERT INTO hh VALUES('GHI', 3, 'v');
+  INSERT INTO hh VALUES('ghi', 3, 'vi');
+
+  CREATE INDEX hh_ab ON hh(a, b); 
+}
+
+do_execsql_test 6.1 {
+  SELECT c FROM hh WHERE (a, b) = (SELECT 'abc', 1);
+} {i}
+do_execsql_test 6.2 {
+  SELECT c FROM hh WHERE (a, b) = (SELECT 'abc' COLLATE nocase, 1);
+} {i}
+do_execsql_test 6.3 {
+  SELECT c FROM hh WHERE a = (SELECT 'abc' COLLATE nocase) AND b = (SELECT 1);
+} {i}
+do_execsql_test 6.4 {
+  SELECT c FROM hh WHERE +a = (SELECT 'abc' COLLATE nocase) AND b = (SELECT 1);
+} {i}
+do_execsql_test 6.5 {
+  SELECT c FROM hh WHERE a = (SELECT 'abc') COLLATE nocase AND b = (SELECT 1);
+} {i ii}
+do_catchsql_test 6.6 {
+  SELECT c FROM hh WHERE (a, b) = (SELECT 'abc', 1) COLLATE nocase;
+} {1 {row value misused}}
+do_catchsql_test 6.7 {
+  SELECT c FROM hh WHERE (a, b) = 1;
+} {1 {row value misused}}
+do_execsql_test 6.8 {
+  SELECT c FROM hh WHERE (a COLLATE nocase, b) = (SELECT 'def', 2);
+} {iii iv}
+do_execsql_test 6.9 {
+  SELECT c FROM hh WHERE (a COLLATE nocase, b) IS NOT (SELECT 'def', 2);
+} {i ii v vi}
+do_execsql_test 6.10 {
+  SELECT c FROM hh WHERE (b, a) = (SELECT 2, 'def');
+} {iii}
+
+do_execsql_test 7.0 {
+  CREATE TABLE xy(i INTEGER PRIMARY KEY, j, k);
+  INSERT INTO xy VALUES(1, 1, 1);
+  INSERT INTO xy VALUES(2, 2, 2);
+  INSERT INTO xy VALUES(3, 3, 3);
+  INSERT INTO xy VALUES(4, 4, 4);
+}
+
+
+foreach {tn sql res eqp} {
+  1 "SELECT * FROM xy WHERE (i, j) IS (2, 2)" {2 2 2} 
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid=?)}"
+
+  2 "SELECT * FROM xy WHERE (k, j) < (2, 3)" {1 1 1 2 2 2}
+    "0 0 0 {SCAN TABLE xy}"
+
+  3 "SELECT * FROM xy WHERE (i, j) < (2, 3)" {1 1 1 2 2 2}
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid (2, 1)" {2 2 2 3 3 3 4 4 4}
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid>?)}"
+
+  5 "SELECT * FROM xy WHERE (i, j) > ('2', 1)" {2 2 2 3 3 3 4 4 4}
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid>?)}"
+
+} {
+  do_eqp_test 7.$tn.1 $sql $eqp
+  do_execsql_test 7.$tn.2 $sql $res
+}
+
+do_execsql_test 8.0 {
+  CREATE TABLE j1(a);
+}
+do_execsql_test 8.1 {
+  SELECT * FROM j1 WHERE (select min(a) FROM j1) IN (?, ?, ?)
+}
+
+do_execsql_test 9.0 {
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c);
+  INSERT INTO t2 VALUES(1, 1, 1);
+  INSERT INTO t2 VALUES(2, 2, 2);
+  INSERT INTO t2 VALUES(3, 3, 3);
+  INSERT INTO t2 VALUES(4, 4, 4);
+  INSERT INTO t2 VALUES(5, 5, 5);
+}
+
+foreach {tn q res} {
+  1 "(a, b) > (2, 1)" {2 3 4 5}
+  2 "(a, b) > (2, 2)" {3 4 5}
+  3 "(a, b) < (4, 5)" {1 2 3 4}
+  4 "(a, b) < (4, 3)" {1 2 3}
+} {
+  do_execsql_test 9.$tn "SELECT c FROM t2 WHERE $q" $res
+} 
+
+do_execsql_test 10.0 {
+  CREATE TABLE dual(dummy); INSERT INTO dual(dummy) VALUES('X');
+  CREATE TABLE t3(a TEXT,b TEXT,c TEXT,d TEXT,e TEXT,f TEXT);
+  CREATE INDEX t3x ON t3(b,c,d,e,f);
+
+  SELECT a FROM t3
+    WHERE (c,d) IN (SELECT 'c','d' FROM dual)
+    AND (a,b,e) IN (SELECT 'a','b','d' FROM dual);
+}
+
+do_catchsql_test 11.1 {
+  CREATE TABLE t11(a);
+  SELECT * FROM t11 WHERE (a,a)<=1;
+} {1 {row value misused}}
+do_catchsql_test 11.2 {
+  SELECT * FROM t11 WHERE (a,a)<1;
+} {1 {row value misused}}
+do_catchsql_test 11.3 {
+  SELECT * FROM t11 WHERE (a,a)>=1;
+} {1 {row value misused}}
+do_catchsql_test 11.4 {
+  SELECT * FROM t11 WHERE (a,a)>1;
+} {1 {row value misused}}
+do_catchsql_test 11.5 {
+  SELECT * FROM t11 WHERE (a,a)==1;
+} {1 {row value misused}}
+do_catchsql_test 11.6 {
+  SELECT * FROM t11 WHERE (a,a)<>1;
+} {1 {row value misused}}
+do_catchsql_test 11.7 {
+  SELECT * FROM t11 WHERE (a,a) IS 1;
+} {1 {row value misused}}
+do_catchsql_test 11.8 {
+  SELECT * FROM t11 WHERE (a,a) IS NOT 1;
+} {1 {row value misused}}
+
+# 2016-10-27: https://www.sqlite.org/src/tktview/fef4bb4bd9185ec8f
+# Incorrect result from a LEFT JOIN with a row-value constraint
+#
+do_execsql_test 12.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a,b); INSERT INTO t1 VALUES(1,2);
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(x,y); INSERT INTO t2 VALUES(3,4);
+  SELECT *,'x' FROM t1 LEFT JOIN t2 ON (a,b)=(x,y);
+} {1 2 {} {} x}
+
+
+foreach {tn sql} {
+  0 "SELECT (1,2) AS x WHERE x=3"
+  1 "SELECT (1,2) BETWEEN 1 AND 2"
+  2 "SELECT 1 BETWEEN (1,2) AND 2"
+  3 "SELECT 2 BETWEEN 1 AND (1,2)"
+  4 "SELECT (1,2) FROM (SELECT 1) ORDER BY 1"
+  5 "SELECT (1,2) FROM (SELECT 1) GROUP BY 1"
+} {
+  do_catchsql_test 13.$tn $sql {1 {row value misused}}
+}
+
+do_execsql_test 14.0 {
+  CREATE TABLE t12(x);
+  INSERT INTO t12 VALUES(2), (4);
+}
+do_execsql_test 14.1 "SELECT 1 WHERE (2,2) BETWEEN (1,1) AND (3,3)" 1
+do_execsql_test 14.2 "SELECT CASE (2,2) WHEN (1, 1) THEN 2 ELSE 1 END" 1
+do_execsql_test 14.3 "SELECT CASE (SELECT 2,2) WHEN (1, 1) THEN 2 ELSE 1 END" 1
+do_execsql_test 14.4 "SELECT 1 WHERE (SELECT 2,2) BETWEEN (1,1) AND (3,3)" 1
+do_execsql_test 14.5 "SELECT 1 FROM t12 WHERE (x,1) BETWEEN (1,1) AND (3,3)" 1
+do_execsql_test 14.6 {
+  SELECT 1 FROM t12 WHERE (1,x) BETWEEN (1,1) AND (3,3)
+} {1 1}
+
+finish_test
+
+
diff --git a/test/rowvalue2.test b/test/rowvalue2.test
new file mode 100644
index 00000000..1502e31a
--- /dev/null
+++ b/test/rowvalue2.test
@@ -0,0 +1,279 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the SELECT statement.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue2
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  INSERT INTO t1 VALUES(0, 0, 0);
+  INSERT INTO t1 VALUES(0, 1, 1);
+  INSERT INTO t1 VALUES(1, 0, 2);
+  INSERT INTO t1 VALUES(1, 1, 3);
+
+  CREATE INDEX i1 ON t1(a, b);
+}
+
+do_execsql_test 1.1.1 { SELECT c FROM t1 WHERE (a, b) >= (1, 0) } {2 3} 
+do_execsql_test 1.1.2 { SELECT c FROM t1 WHERE (a, b) > (1, 0)  } {3}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0.1 {
+  CREATE TABLE t2(a INTEGER, b INTEGER, c INTEGER, d INTEGER);
+  CREATE INDEX i2 ON t2(a, b, c);
+}
+do_test 2.0.2 {
+  foreach a {0 1 2 3} {
+  foreach b {0 1 2 3} {
+  foreach c {0 1 2 3} {
+    execsql { INSERT INTO t2 VALUES($a, $b, $c, $c + $b*4 + $a*16); }
+  }}}
+} {}
+
+do_execsql_test 2.1 {
+  SELECT d FROM t2 WHERE (a, b) > (2, 2);
+} [db eval { SELECT d FROM t2 WHERE a>2 OR (a=2 AND b>2) }]
+
+do_execsql_test 2.2 {
+  SELECT d FROM t2 WHERE (a, b) >= (2, 2);
+} [db eval { SELECT d FROM t2 WHERE a>2 OR (a=2 AND b>=2) }]
+
+do_execsql_test 2.3 {
+  SELECT d FROM t2 WHERE a=1 AND (b, c) >= (1, 2);
+} [db eval { SELECT d FROM t2 WHERE +a=1 AND (b>1 OR (b==1 AND c>=2)) }]
+
+do_execsql_test 2.4 {
+  SELECT d FROM t2 WHERE a=1 AND (b, c) > (1, 2);
+} [db eval { SELECT d FROM t2 WHERE +a=1 AND (b>1 OR (b==1 AND c>2)) }]
+
+#-------------------------------------------------------------------------
+
+set words {
+airfare airfield airfields airflow airfoil
+airfoils airframe airframes airily airing
+airings airless airlift airlifts airline
+airliner airlines airlock airlocks airmail
+airmails airman airmen airplane airplanes
+
+arraignment arraignments arraigns arrange arranged
+arrangement arrangements arranger arrangers arranges
+arranging arrant array arrayed arrays
+arrears arrest arrested arrester arresters
+arresting arrestingly arrestor arrestors arrests
+
+edifices edit edited editing edition
+editions editor editorial editorially editorials
+editors edits educable educate educated
+educates educating education educational educationally
+educations educator educators eel eelgrass
+}
+
+do_test 3.0 {
+  execsql { CREATE TABLE t3(a, b, c, w); }
+  foreach w $words {
+    set a [string range $w 0 2]
+    set b [string range $w 3 5]
+    set c [string range $w 6 end]
+    execsql { INSERT INTO t3 VALUES($a, $b, $c, $w) }
+  }
+} {}
+
+
+foreach {tn idx} {
+  IDX1 {}
+  IDX2 { CREATE INDEX i3 ON t3(a, b, c); }
+  IDX3 { CREATE INDEX i3 ON t3(a, b); }
+  IDX4 { CREATE INDEX i3 ON t3(a); }
+} {
+  execsql { DROP INDEX IF EXISTS i3 }
+  execsql $idx
+
+  foreach w $words {
+    set a [string range $w 0 2]
+    set b [string range $w 3 5]
+    set c [string range $w 6 end]
+
+    foreach op [list > >= < <= == IS] {
+      do_execsql_test 3.1.$tn.$w.$op [subst -novar {
+        SELECT rowid FROM t3 WHERE (a, b, c) [set op] ($a, $b, $c) 
+        ORDER BY +rowid
+      }] [db eval [subst -novar {
+        SELECT rowid FROM t3 WHERE w [set op] $w ORDER BY +rowid
+      }]]
+
+      do_execsql_test 3.1.$tn.$w.$op.subselect [subst -novar {
+        SELECT rowid FROM t3 WHERE (a, b, c) [set op] (
+          SELECT a, b, c FROM t3 WHERE w = $w
+        )
+        ORDER BY +rowid
+      }] [db eval [subst -novar {
+        SELECT rowid FROM t3 WHERE w [set op] $w ORDER BY +rowid
+      }]]
+    }
+
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 4.0 {
+  CREATE TABLE t4(a, b, c);
+  INSERT INTO t4 VALUES(NULL, NULL, NULL);
+  INSERT INTO t4 VALUES(NULL, NULL, 0);
+  INSERT INTO t4 VALUES(NULL, NULL, 1);
+  INSERT INTO t4 VALUES(NULL,    0, NULL);
+  INSERT INTO t4 VALUES(NULL,    0, 0);
+  INSERT INTO t4 VALUES(NULL,    0, 1);
+  INSERT INTO t4 VALUES(NULL,    1, NULL);
+  INSERT INTO t4 VALUES(NULL,    1, 0);
+  INSERT INTO t4 VALUES(NULL,    1, 1);
+
+  INSERT INTO t4 VALUES(   0, NULL, NULL);
+  INSERT INTO t4 VALUES(   0, NULL, 0);
+  INSERT INTO t4 VALUES(   0, NULL, 1);
+  INSERT INTO t4 VALUES(   0,    0, NULL);
+  INSERT INTO t4 VALUES(   0,    0, 0);
+  INSERT INTO t4 VALUES(   0,    0, 1);
+  INSERT INTO t4 VALUES(   0,    1, NULL);
+  INSERT INTO t4 VALUES(   0,    1, 0);
+  INSERT INTO t4 VALUES(   0,    1, 1);
+
+  INSERT INTO t4 VALUES(   1, NULL, NULL);
+  INSERT INTO t4 VALUES(   1, NULL, 0);
+  INSERT INTO t4 VALUES(   1, NULL, 1);
+  INSERT INTO t4 VALUES(   1,    0, NULL);
+  INSERT INTO t4 VALUES(   1,    0, 0);
+  INSERT INTO t4 VALUES(   1,    0, 1);
+  INSERT INTO t4 VALUES(   1,    1, NULL);
+  INSERT INTO t4 VALUES(   1,    1, 0);
+  INSERT INTO t4 VALUES(   1,    1, 1);
+}
+
+proc make_expr1 {cList vList op} {
+  return "([join $cList ,]) $op ([join $vList ,])"
+}
+
+proc make_expr3 {cList vList op} {
+  set n [llength $cList]
+
+  set aList [list]
+  foreach c [lrange $cList 0 end-1] v [lrange $vList 0 end-1] {
+    lappend aList "$c == $v"
+  }
+  lappend aList "[lindex $cList end] $op [lindex $vList end]"
+
+  return "([join $aList { AND }])"
+}
+
+proc make_expr2 {cList vList op} {
+  set ret ""
+
+  switch -- $op {
+    == - IS {
+      set aList [list]
+      foreach c $cList v $vList { lappend aList "($c $op $v)" }
+      set ret [join $aList " AND "]
+    }
+
+    < - > {
+      set oList [list]
+      for {set i 0} {$i < [llength $cList]} {incr i} {
+        lappend oList [make_expr3 [lrange $cList 0 $i] [lrange $vList 0 $i] $op]
+      }
+      set ret [join $oList " OR "]
+    }
+
+    <= - >= {
+      set o2 [string range $op 0 0]
+      set oList [list]
+      for {set i 0} {$i < [llength $cList]-1} {incr i} {
+        lappend oList [make_expr3 [lrange $cList 0 $i] [lrange $vList 0 $i] $o2]
+      }
+      lappend oList [make_expr3 $cList $vList $op]
+      set ret [join $oList " OR "]
+    }
+
+
+    default {
+      error "Unknown op: $op"
+    }
+  }
+
+  set ret
+}
+
+foreach {tn idx} {
+  IDX1 {}
+  IDX2 { CREATE INDEX i4 ON t4(a, b, c); }
+  IDX3 { CREATE INDEX i4 ON t4(a, b); }
+  IDX4 { CREATE INDEX i4 ON t4(a); }
+} {
+  execsql { DROP INDEX IF EXISTS i4 }
+  execsql $idx
+
+  foreach {tn2 vector} {
+    1 {0 0 0}
+    2 {1 1 1}
+    3 {0 0 NULL}
+    4 {0 NULL 0}
+    5 {NULL 0 0}
+    6 {1 1 NULL}
+    7 {1 NULL 1}
+    8 {NULL 1 1}
+  } {
+    foreach op { IS == < <= > >= } {
+      set e1 [make_expr1 {a b c} $vector $op]
+      set e2 [make_expr2 {a b c} $vector $op]
+
+      do_execsql_test 4.$tn.$tn2.$op \
+          "SELECT rowid FROM t4 WHERE $e2 ORDER BY +rowid" [
+          db eval "SELECT rowid FROM t4 WHERE $e1 ORDER BY +rowid"
+      ]
+    }
+  }
+}
+
+do_execsql_test 5.0 {
+  CREATE TABLE r1(a TEXT, iB TEXT);
+  CREATE TABLE r2(x TEXT, zY INTEGER);
+  CREATE INDEX r1ab ON r1(a, iB);
+
+  INSERT INTO r1 VALUES(35, 35);
+  INSERT INTO r2 VALUES(35, 36);
+  INSERT INTO r2 VALUES(35, 4);
+  INSERT INTO r2 VALUES(35, 35);
+} {}
+
+foreach {tn lhs rhs} {
+  1 {x +zY} {a iB}
+  2 {x  zY} {a iB}
+  3 {x  zY} {a +iB}
+  4 {+x  zY} {a iB}
+  5 {x  zY} {+a iB}
+} {
+  foreach op { IS == < <= > >= } {
+    set e1 [make_expr1 $lhs $rhs $op]
+    set e2 [make_expr2 $lhs $rhs $op]
+    do_execsql_test 5.$tn.$op \
+      "SELECT * FROM r1, r2 WHERE $e2 ORDER BY iB" [db eval \
+      "SELECT * FROM r1, r2 WHERE $e1 ORDER BY iB"
+    ]
+  }
+}
+
+
+finish_test
diff --git a/test/rowvalue3.test b/test/rowvalue3.test
new file mode 100644
index 00000000..988eea90
--- /dev/null
+++ b/test/rowvalue3.test
@@ -0,0 +1,220 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing "(...) IN (SELECT ...)" expressions
+# where the SELECT statement returns more than one column.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue3
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX i1 ON t1(a, b);
+  INSERT INTO t1 VALUES(1, 2, 3);
+  INSERT INTO t1 VALUES(4, 5, 6);
+  INSERT INTO t1 VALUES(7, 8, 9);
+}
+
+foreach {tn sql res} {
+  1  "SELECT 1 WHERE (4, 5) IN (SELECT a, b FROM t1)"  1
+  2  "SELECT 1 WHERE (5, 5) IN (SELECT a, b FROM t1)"  {}
+  3  "SELECT 1 WHERE (5, 4) IN (SELECT a, b FROM t1)"  {}
+  4  "SELECT 1 WHERE (5, 4) IN (SELECT b, a FROM t1)"  1
+  5  "SELECT 1 WHERE (SELECT a, b FROM t1 WHERE c=6) IN (SELECT a, b FROM t1)" 1
+  6  "SELECT (5, 4) IN (SELECT a, b FROM t1)" 0
+  7  "SELECT 1 WHERE (5, 4) IN (SELECT +b, +a FROM t1)"  1
+  8  "SELECT (5, 4) IN (SELECT +b, +a FROM t1)"  1
+  9  "SELECT (1, 2) IN (SELECT rowid, b FROM t1)"  1
+  10 "SELECT 1 WHERE (1, 2) IN (SELECT rowid, b FROM t1)"  1
+  11 "SELECT 1 WHERE (1, NULL) IN (SELECT rowid, b FROM t1)"  {}
+  12 "SELECT 1 FROM t1 WHERE (a, b) = (SELECT +a, +b FROM t1)" {1}
+} {
+  do_execsql_test 1.$tn $sql $res
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  CREATE TABLE z1(x, y, z);
+  CREATE TABLE kk(a, b);
+
+  INSERT INTO z1 VALUES('a', 'b', 'c');
+  INSERT INTO z1 VALUES('d', 'e', 'f');
+  INSERT INTO z1 VALUES('g', 'h', 'i');
+
+  -- INSERT INTO kk VALUES('y', 'y');
+  INSERT INTO kk VALUES('d', 'e');
+  -- INSERT INTO kk VALUES('x', 'x');
+
+}
+
+foreach {tn idx} {
+  1 { }
+  2 { CREATE INDEX z1idx ON z1(x, y) }
+  3 { CREATE UNIQUE INDEX z1idx ON z1(x, y) }
+  4 { CREATE INDEX z1idx ON kk(a, b) }
+} {
+  execsql "DROP INDEX IF EXISTS z1idx"
+  execsql $idx
+
+  do_execsql_test 2.$tn.1 {
+    SELECT * FROM z1 WHERE x IN (SELECT a FROM kk)
+  } {d e f}
+
+  do_execsql_test 2.$tn.2 {
+    SELECT * FROM z1 WHERE (x,y) IN (SELECT a, b FROM kk)
+  } {d e f}
+
+  do_execsql_test 2.$tn.3 {
+    SELECT * FROM z1 WHERE (x, +y) IN (SELECT a, b FROM kk)
+  } {d e f}
+  
+  do_execsql_test 2.$tn.4 {
+    SELECT * FROM z1 WHERE (x, +y) IN (SELECT a, b||'x' FROM kk)
+  } {}
+
+  do_execsql_test 2.$tn.5 {
+    SELECT * FROM z1 WHERE (+x, y) IN (SELECT a, b FROM kk)
+  } {d e f}
+}
+
+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 3.0 {
+  CREATE TABLE c1(a, b, c, d);
+  INSERT INTO c1(rowid, a, b) VALUES(1,   NULL, 1);
+  INSERT INTO c1(rowid, a, b) VALUES(2,   2, NULL);
+  INSERT INTO c1(rowid, a, b) VALUES(3,   2, 2);
+  INSERT INTO c1(rowid, a, b) VALUES(4,   3, 3);
+
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(101, 'a', 'b', 1, 1);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(102, 'a', 'b', 1, 2);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(103, 'a', 'b', 1, 3);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(104, 'a', 'b', 2, 1);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(105, 'a', 'b', 2, 2);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(106, 'a', 'b', 2, 3);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(107, 'a', 'b', 3, 1);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(108, 'a', 'b', 3, 2);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(109, 'a', 'b', 3, 3);
+}
+
+
+foreach {tn idx} {
+  1 { }
+  2 { CREATE INDEX c1ab ON c1(a, b); }
+  3 { CREATE INDEX c1ba ON c1(b, a); }
+
+  4 { CREATE INDEX c1cd ON c1(c, d); }
+  5 { CREATE INDEX c1dc ON c1(d, c); }
+} {
+  drop_all_indexes
+
+  foreach {tn2 sql res} {
+    1 "SELECT (1, 2) IN (SELECT a, b FROM c1)" {0}
+    2 "SELECT (1, 1) IN (SELECT a, b FROM c1)" {{}}
+    3 "SELECT (2, 1) IN (SELECT a, b FROM c1)" {{}}
+    4 "SELECT (2, 2) IN (SELECT a, b FROM c1)" {1}
+    5 "SELECT c, d FROM c1 WHERE (c, d) IN (SELECT d, c FROM c1)"
+      { 1 1 1 2 1 3   2 1 2 2 2 3   3 1 3 2 3 3 }
+
+    6 "SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) ORDER BY c DESC"
+      { 3 1 3 2 3 3   2 1 2 2 2 3   1 1 1 2 1 3 }
+
+    7 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c DESC, d ASC
+      } { 3 1 3 2 3 3   2 1 2 2 2 3   1 1 1 2 1 3 }
+
+    8 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c ASC, d DESC
+      } { 1 3 1 2 1 1   2 3 2 2 2 1   3 3 3 2 3 1 }
+
+    9 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c ASC, d ASC
+      } { 1 1 1 2 1 3   2 1 2 2 2 3   3 1 3 2 3 3 }
+    10 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c DESC, d DESC
+      } { 3 3 3 2 3 1   2 3 2 2 2 1   1 3 1 2 1 1 }
+
+  } {
+    do_execsql_test 3.$tn.$tn2 $sql $res
+  }
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 4.0 {
+  CREATE TABLE hh(a, b, c);
+
+  INSERT INTO hh VALUES('a', 'a', 1);
+  INSERT INTO hh VALUES('a', 'b', 2);
+  INSERT INTO hh VALUES('b', 'a', 3);
+  INSERT INTO hh VALUES('b', 'b', 4);
+
+  CREATE TABLE k1(x, y);
+  INSERT INTO k1 VALUES('a', 'a');
+  INSERT INTO k1 VALUES('b', 'b');
+  INSERT INTO k1 VALUES('a', 'b');
+  INSERT INTO k1 VALUES('b', 'a');
+}
+
+foreach {tn idx} {
+  1 { }
+  2 { CREATE INDEX h1 ON hh(a, b); }
+  3 { CREATE UNIQUE INDEX k1idx ON k1(x, y) }
+  4 { CREATE UNIQUE INDEX k1idx ON k1(x, y DESC) }
+  5 { 
+    CREATE INDEX h1 ON hh(a, b);
+    CREATE UNIQUE INDEX k1idx ON k1(x, y); 
+  }
+  6 { 
+    CREATE INDEX h1 ON hh(a, b);
+    CREATE UNIQUE INDEX k1idx ON k1(x, y DESC); 
+  }
+} {
+  drop_all_indexes
+  execsql $idx
+  foreach {tn2 orderby res} {
+    1 "a ASC, b ASC"  {1 2 3 4}
+    2 "a ASC, b DESC" {2 1 4 3}
+    3 "a DESC, b ASC" {3 4 1 2}
+    4 "a DESC, b DESC" {4 3 2 1}
+  } {
+    do_execsql_test 4.$tn.$tn2 "
+      SELECT c FROM hh WHERE (a, b) in (SELECT x, y FROM k1) ORDER BY $orderby
+    " $res
+  }
+}
+
+#-------------------------------------------------------------------------
+
+# 2016-11-17.  Query flattening in a vector SELECT on the RHS of an IN
+# operator.  Ticket https://www.sqlite.org/src/info/da7841375186386c
+#
+do_execsql_test 5.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE T1(a TEXT);
+  INSERT INTO T1(a) VALUES ('aaa');
+  CREATE TABLE T2(a TEXT PRIMARY KEY,n INT);
+  INSERT INTO T2(a, n) VALUES('aaa',0);
+  SELECT * FROM T2
+  WHERE (a,n) IN (SELECT T1.a, V.n FROM T1, (SELECT * FROM (SELECT 0 n)) V);
+} {aaa 0}
+
+
+finish_test
diff --git a/test/rowvalue4.test b/test/rowvalue4.test
new file mode 100644
index 00000000..2e6a28c2
--- /dev/null
+++ b/test/rowvalue4.test
@@ -0,0 +1,313 @@
+# 2016 July 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is syntax errors involving row-value constructors
+# and sub-selects that return multiple arguments.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue4
+
+#-------------------------------------------------------------------------
+# Test some error conditions:
+# 
+#   * row values used where they are not supported,
+#   * row values or sub-selects that contain/return the wrong number
+#     of elements.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1bac ON t1(b, a, c);
+}
+
+foreach {tn e} {
+  1 "(1, 2, 3)"
+  2 "1 + (1, 2)"
+  3 "(1,2,3) == (1, 2)"
+} {
+  do_catchsql_test 1.$tn "SELECT $e" {1 {row value misused}}
+}
+
+foreach {tn s error} {
+  1 "SELECT * FROM t1 WHERE a = (1, 2)"       {row value misused}
+  2 "SELECT * FROM t1 WHERE b = (1, 2)"       {row value misused}
+  3 "SELECT * FROM t1 WHERE NOT (b = (1, 2))" {row value misused}
+  4 "SELECT * FROM t1 LIMIT (1, 2)"           {row value misused}
+  5 "SELECT (a, b) IN (SELECT * FROM t1) FROM t1" 
+                             {sub-select returns 3 columns - expected 2}
+
+  6 "SELECT * FROM t1 WHERE (a, b) IN (SELECT * FROM t1)" 
+                             {sub-select returns 3 columns - expected 2}
+  7 "SELECT * FROM t1 WHERE (c, c) <= 1" {row value misused}
+  8 "SELECT * FROM t1 WHERE (b, b) <= 1" {row value misused}
+} {
+  do_catchsql_test 2.$tn "$s" [list 1 $error]
+}
+
+#-------------------------------------------------------------------------
+do_execsql_test 2.0 {
+  CREATE TABLE t2(a, b, c, d);
+  INSERT INTO t2 VALUES(1, 1, 1,   1);
+  INSERT INTO t2 VALUES(1, 1, 2,   2);
+  INSERT INTO t2 VALUES(1, 1, 3,   3);
+  INSERT INTO t2 VALUES(1, 2, 1,   4);
+  INSERT INTO t2 VALUES(1, 2, 2,   5);
+  INSERT INTO t2 VALUES(1, 2, 3,   6);
+  INSERT INTO t2 VALUES(1, 3, 1,   7);
+  INSERT INTO t2 VALUES(1, 3, 2,   8);
+  INSERT INTO t2 VALUES(1, 3, 3,   9);
+
+  INSERT INTO t2 VALUES(2, 1, 1,   10);
+  INSERT INTO t2 VALUES(2, 1, 2,   11);
+  INSERT INTO t2 VALUES(2, 1, 3,   12);
+  INSERT INTO t2 VALUES(2, 2, 1,   13);
+  INSERT INTO t2 VALUES(2, 2, 2,   14);
+  INSERT INTO t2 VALUES(2, 2, 3,   15);
+  INSERT INTO t2 VALUES(2, 3, 1,   16);
+  INSERT INTO t2 VALUES(2, 3, 2,   17);
+  INSERT INTO t2 VALUES(2, 3, 3,   18);
+
+  INSERT INTO t2 VALUES(3, 1, 1,   19);
+  INSERT INTO t2 VALUES(3, 1, 2,   20);
+  INSERT INTO t2 VALUES(3, 1, 3,   21);
+  INSERT INTO t2 VALUES(3, 2, 1,   22);
+  INSERT INTO t2 VALUES(3, 2, 2,   23);
+  INSERT INTO t2 VALUES(3, 2, 3,   24);
+  INSERT INTO t2 VALUES(3, 3, 1,   25);
+  INSERT INTO t2 VALUES(3, 3, 2,   26);
+  INSERT INTO t2 VALUES(3, 3, 3,   27);
+}
+
+foreach {nm idx} {
+  idx1 {}
+  idx2 { CREATE INDEX t2abc ON t2(a, b, c); }
+  idx3 { CREATE INDEX t2abc ON t2(a, b DESC, c); }
+  idx4 { CREATE INDEX t2abc ON t2(a DESC, b DESC, c DESC); }
+  idx5 { CREATE INDEX t2abc ON t2(a ASC, b ASC, c ASC); }
+  idx6 { CREATE INDEX t2abc ON t2(a DESC, b, c); }
+  idx7 { CREATE INDEX t2abc ON t2(a DESC, b DESC) }
+  idx8 { CREATE INDEX t2abc ON t2(c, b, a); }
+  idx9 { CREATE INDEX t2d ON t2(d); }
+  idx10 { CREATE INDEX t2abc ON t2(a DESC, b, c DESC); }
+} {
+  drop_all_indexes
+  execsql $idx
+
+  foreach {tn where res} {
+    1 "(a, b, c) < (2, 2, 2)"  {1 2 3 4 5 6 7 8 9 10 11 12 13}
+    2 "(a, b, c) <= (2, 2, 2)" {1 2 3 4 5 6 7 8 9 10 11 12 13 14}
+    3 "(a, b, c) > (2, 2, 2)"  {15 16 17 18 19 20 21 22 23 24 25 26 27}
+    4 "(a, b, c) >= (2, 2, 2)" {14 15 16 17 18 19 20 21 22 23 24 25 26 27}
+    5 "(a, b, c) >= (2, 2, NULL)" {16 17 18 19 20 21 22 23 24 25 26 27}
+    6 "(a, b, c) <= (2, 2, NULL)" {1 2 3 4 5 6 7 8 9 10 11 12}
+    7 "(a, b, c) >= (2, NULL, NULL)" {19 20 21 22 23 24 25 26 27}
+    8 "(a, b, c) <= (2, NULL, NULL)" {1 2 3 4 5 6 7 8 9}
+
+    9 "(a, b, c) < (SELECT a, b, c FROM t2 WHERE d=14)"  
+      {1 2 3 4 5 6 7 8 9 10 11 12 13}
+
+    10 "(a, b, c) = (SELECT a, b, c FROM t2 WHERE d=14)" 14
+
+    11 "a = 2 AND (b, c) > (2, 2)" {15 16 17 18}
+    12 "a = 2 AND (b, c) < (3, 3) AND (b, c) > (1, 1)" {11 12 13 14 15 16 17}
+  } {
+    set result [db eval "SELECT d FROM t2 WHERE $where"]
+    do_test 2.1.$nm.$tn { lsort -integer $result } $res
+  }
+
+  foreach {tn e res} {
+    1 "(2, 1) IN (SELECT a, b FROM t2)" 1
+    2 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d)" 1
+    3 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 9)" 0
+    4 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 10)" 1
+
+    5 "(3, 3) = (SELECT a, b FROM t2 ORDER BY d DESC LIMIT 1)" 1
+    6 "(3, 3) = (SELECT a, b FROM t2 ORDER BY d ASC LIMIT 1)" 0
+    7 "(1, NULL) = (SELECT a, b FROM t2 ORDER BY d ASC LIMIT 1)" {{}}
+
+    8 "(3, 1) = (SELECT b, c FROM t2 ORDER BY d DESC LIMIT 1 OFFSET 2)" 1
+    9 "(3, 1) = (SELECT b, c FROM t2 ORDER BY d ASC LIMIT 1 OFFSET 2)" 0
+    10 "(1, NULL) = (SELECT b, c FROM t2 ORDER BY d ASC LIMIT 1 OFFSET 2)" {{}}
+
+    11 "(3, 3) = (SELECT max(a), max(b) FROM t2)" 1
+    12 "(3, 1) = (SELECT max(a), min(b) FROM t2)" 1
+    13 "(NULL, NULL) = (SELECT max(a), min(b) FROM t2)" {{}}
+
+    14 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 5 OFFSET 11)" 1
+    15 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 5 OFFSET 12)" 0
+  } {
+    do_execsql_test 2.2.$nm.$tn "SELECT $e" $res
+  }
+}
+
+ifcapable stat4 {
+  do_execsql_test 3.0 {
+    CREATE TABLE c1(a, b, c, d);
+    INSERT INTO c1(a, b) VALUES(1, 'a');
+    INSERT INTO c1(a, b) VALUES(1, 'b');
+    INSERT INTO c1(a, b) VALUES(1, 'c');
+    INSERT INTO c1(a, b) VALUES(1, 'd');
+    INSERT INTO c1(a, b) VALUES(1, 'e');
+    INSERT INTO c1(a, b) VALUES(1, 'f');
+    INSERT INTO c1(a, b) VALUES(1, 'g');
+    INSERT INTO c1(a, b) VALUES(1, 'h');
+    INSERT INTO c1(a, b) VALUES(1, 'i');
+    INSERT INTO c1(a, b) VALUES(1, 'j');
+    INSERT INTO c1(a, b) VALUES(1, 'k');
+    INSERT INTO c1(a, b) VALUES(1, 'l');
+    INSERT INTO c1(a, b) VALUES(1, 'm');
+    INSERT INTO c1(a, b) VALUES(1, 'n');
+    INSERT INTO c1(a, b) VALUES(1, 'o');
+    INSERT INTO c1(a, b) VALUES(1, 'p');
+    INSERT INTO c1(a, b) VALUES(2, 'a');
+    INSERT INTO c1(a, b) VALUES(2, 'b');
+    INSERT INTO c1(a, b) VALUES(2, 'c');
+    INSERT INTO c1(a, b) VALUES(2, 'd');
+    INSERT INTO c1(a, b) VALUES(2, 'e');
+    INSERT INTO c1(a, b) VALUES(2, 'f');
+    INSERT INTO c1(a, b) VALUES(2, 'g');
+    INSERT INTO c1(a, b) VALUES(2, 'h');
+
+    INSERT INTO c1(c, d) SELECT a, b FROM c1;
+
+    CREATE INDEX c1ab ON c1(a, b);
+    CREATE INDEX c1cd ON c1(c, d);
+    ANALYZE;
+  }
+
+  do_eqp_test 3.1.1 { SELECT * FROM c1 WHERE a=1 AND c=2 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd (c=?)}
+  }
+  do_eqp_test 3.1.2 { SELECT * FROM c1 WHERE a=1 AND b>'d' AND c=2 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd (c=?)}
+  }
+  do_eqp_test 3.1.3 { SELECT * FROM c1 WHERE a=1 AND b>'l' AND c=2 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=? AND b>?)}
+  }
+
+  do_eqp_test 3.2.1 { SELECT * FROM c1 WHERE a=1 AND c>1 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd (c>?)}
+  }
+  do_eqp_test 3.2.2 { SELECT * FROM c1 WHERE a=1 AND c>0 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+  do_eqp_test 3.2.3 { SELECT * FROM c1 WHERE a=1 AND c>=1 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+  do_eqp_test 3.2.4 { SELECT * FROM c1 WHERE a=1 AND (c, d)>(1, 'c') } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+  do_eqp_test 3.2.5 { SELECT * FROM c1 WHERE a=1 AND (c, d)>(1, 'o') } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd ((c,d)>(?,?))}
+  }
+  do_eqp_test 3.2.6 { SELECT * FROM c1 WHERE a=1 AND (c, +b)>(1, 'c') } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+}
+
+#------------------------------------------------------------------------
+
+do_execsql_test 5.0 {
+  CREATE TABLE d1(x, y);
+  CREATE TABLE d2(a, b, c);
+  CREATE INDEX d2ab ON d2(a, b);
+  CREATE INDEX d2c ON d2(c);
+
+  WITH i(i) AS (
+    VALUES(1) UNION ALL SELECT i+1 FROM i WHERE i<1000
+  )
+  INSERT INTO d2 SELECT i/3, i%3, i/3 FROM i;
+  ANALYZE;
+}
+
+do_eqp_test 5.1 {
+  SELECT * FROM d2 WHERE 
+    (a, b) IN (SELECT x, y FROM d1) AND
+    (c) IN (SELECT y FROM d1)
+} {
+  0 0 0 {SEARCH TABLE d2 USING INDEX d2ab (a=? AND b=?)}
+  0 0 0 {EXECUTE LIST SUBQUERY 1} 
+  1 0 0 {SCAN TABLE d1}
+  0 0 0 {EXECUTE LIST SUBQUERY 2} 
+  2 0 0 {SCAN TABLE d1}
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE e1(a, b, c, d, e);
+  CREATE INDEX e1ab ON e1(a, b);
+  CREATE INDEX e1cde ON e1(c, d, e);
+}
+
+do_eqp_test 6.1 {
+  SELECT * FROM e1 WHERE (a, b) > (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1ab ((a,b)>(?,?))}
+}
+do_eqp_test 6.2 {
+  SELECT * FROM e1 WHERE (a, b) < (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1ab ((a,b)<(?,?))}
+}
+do_eqp_test 6.3 {
+  SELECT * FROM e1 WHERE c = ? AND (d, e) > (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1cde (c=? AND (d,e)>(?,?))}
+}
+do_eqp_test 6.4 {
+  SELECT * FROM e1 WHERE c = ? AND (d, e) < (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1cde (c=? AND (d,e)<(?,?))}
+}
+
+do_eqp_test 6.5 {
+  SELECT * FROM e1 WHERE (d, e) BETWEEN (?, ?) AND (?, ?) AND c = ?
+} {
+  0 0 0 
+  {SEARCH TABLE e1 USING INDEX e1cde (c=? AND (d,e)>(?,?) AND (d,e)<(?,?))}
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 7.1 {
+  CREATE TABLE f1(a, b, c);
+  CREATE INDEX f1ab ON f1(a, b);
+}
+
+do_catchsql_test 7.2 {
+  SELECT (a COLLATE nocase, b) IN (SELECT a, b FROM f1) FROM f1;
+} {0 {}}
+
+do_catchsql_test 7.3 {
+  SELECT (a COLLATE nose, b) IN (SELECT a, b FROM f1) FROM f1;
+} {1 {no such collation sequence: nose}}
+
+do_catchsql_test 7.4 {
+  SELECT * FROM f1 WHERE (?, ? COLLATE nose) > (a, b);
+} {1 {no such collation sequence: nose}}
+
+#-------------------------------------------------------------------------
+drop_all_tables
+do_execsql_test 8.1 {
+  CREATE TABLE c1(x, y);
+  CREATE TABLE c2(a, b, c);
+  CREATE INDEX c2ab ON c2(a, b);
+  CREATE INDEX c2c ON c2(c);
+
+  CREATE TABLE c3(d);
+}
+do_catchsql_test 8.2 {
+  SELECT * FROM c2 CROSS JOIN c3 WHERE 
+    ( (a, b) == (SELECT x, y FROM c1) AND c3.d = c ) OR
+    ( c == (SELECT x, y FROM c1) AND c3.d = c )
+} {1 {row value misused}}
+
+finish_test
diff --git a/test/rowvalue5.test b/test/rowvalue5.test
new file mode 100644
index 00000000..58b39f27
--- /dev/null
+++ b/test/rowvalue5.test
@@ -0,0 +1,115 @@
+# 2016 July 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is syntax errors involving row-values and
+# virtual tables.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue5
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c, d, expr)"
+    }
+
+    xBestIndex {
+      set COL(0) a
+      set COL(1) b
+      set COL(2) c
+      set COL(3) d
+      set COL(4) expr
+
+      set OP(eq) =
+      set OP(ne) !=
+      set OP(gt) >
+      set OP(le) <=
+      set OP(lt) <
+      set OP(ge) >=
+      set OP(match) MATCH
+      set OP(like) LIKE
+      set OP(glob) GLOB
+      set OP(regexp) REGEXP
+
+      set clist [lindex $args 0]
+      set ret [list]
+      set elist [list]
+      set i 0
+      foreach c $clist {
+        array set C $c
+        if {$C(usable)} {
+          lappend ret omit $i
+          lappend elist "$COL($C(column)) $OP($C(op)) %$i%"
+        }
+        incr i
+      }
+
+      lappend ret idxstr [join $elist " AND "]
+      #puts "xBestIndex: $ret"
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr arglist} $args {}
+      set i 0
+      set ee $idxstr
+      foreach a $arglist {
+        if {[string is double $a]==0} {
+          set a "'[string map {' ''} $a]'"
+        }
+        set ee [string map [list "%$i%" $a] $ee]
+        incr i
+      }
+      set ee [string map [list "'" "''"] $ee]
+
+      set ret [list sql "SELECT 1, 'a', 'b', 'c', 'd', '$ee'"]
+      #puts "xFilter: $ret"
+      return $ret
+    }
+  }
+
+  return {}
+}
+
+register_tcl_module db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+} {}
+
+
+foreach {tn where res} {
+  1 "1"                              {{}}
+  2 "a=1"                            {{a = 1}}
+  3 "a=1 AND 4 = b"                  {{a = 1 AND b = 4}}
+  4 "c>'hello'"                      {{c > 'hello'}}
+  5 "c<='hel''lo'"                   {{c <= 'hel''lo'}}
+  6 "(a, b) = (SELECT 9, 10)"        {{a = 9 AND b = 10}}
+  7 "(+a, b) = (SELECT 'a', 'b')"    {{b = 'b'}}
+  8 "(a, +b) = (SELECT 'a', 'b')"    {{a = 'a'}}
+  11 "(+a, b) IN (SELECT 'a', 'b')"  {{b = 'b'}}
+  12 "(a, +b) IN (SELECT 'a', 'b')"  {{a = 'a'}}
+
+  13 "(a, b) < ('d', 'e')"           {{a <= 'd'}}
+  14 "(a, b) < ('a', 'c')"           {{a <= 'a'}}
+  15 "(a, b) <= ('a', 'b')"          {{a <= 'a'}}
+  16 "(a, b) < ('a', 'b')"           {}
+} {
+  do_execsql_test 1.$tn "SELECT expr FROM x1 WHERE $where" $res
+}
+
+finish_test
diff --git a/test/rowvalue6.test b/test/rowvalue6.test
new file mode 100644
index 00000000..d90e61c8
--- /dev/null
+++ b/test/rowvalue6.test
@@ -0,0 +1,36 @@
+# 2016-08-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# The focus of this file is handling of NULL values in row-value IN
+# expressions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue6
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(a,b,c);
+  CREATE INDEX t1x1 ON t1(a,b);
+  INSERT INTO t1 VALUES(1,NULL,200);
+
+  CREATE TABLE t2(x,y,z);
+  INSERT INTO t2 VALUES(1,NULL,55);
+
+  SELECT c FROM t1 WHERE (a,b) IN (SELECT x,y FROM t2 WHERE z==55);
+} {}
+do_execsql_test 1.2 {
+  INSERT INTO t1 VALUES(2,3,400);
+  INSERT INTO t2 VALUES(2,3,55);  
+
+  SELECT c FROM t1 WHERE (a,b) IN (SELECT x,y FROM t2 WHERE z==55);
+} {400}
+
+finish_test
diff --git a/test/rowvalue7.test b/test/rowvalue7.test
new file mode 100644
index 00000000..f6764f30
--- /dev/null
+++ b/test/rowvalue7.test
@@ -0,0 +1,58 @@
+# 2016-08-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# The focus of this file is vector assignments in the SET clause of
+# an UPDATE statement.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue7
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(a,b,c,d);
+  CREATE INDEX t1x ON t1(a,b);
+  INSERT INTO t1(a,b,c,d) VALUES(1,2,0,0),(3,4,0,0),(5,6,0,0);
+  CREATE TABLE t2(w,x,y,z);
+  CREATE INDEX t2x ON t2(w,x);
+  INSERT INTO t2(w,x,y,z) VALUES(1,2,11,22),(8,9,88,99),(3,5,33,55),(5,6,55,66);
+
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 0 0 | 3 4 0 0 | 5 6 0 0 |}
+
+do_execsql_test 1.2 {
+  UPDATE t1 SET (c,d) = (SELECT y,z FROM t2 WHERE (w,x)=(a,b));
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 4 {} {} | 5 6 55 66 |}
+
+do_execsql_test 1.3 {
+  UPDATE t1 SET (c,d) = (SELECT y,z FROM t2 WHERE w=a);
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 4 33 55 | 5 6 55 66 |}
+
+do_execsql_test 1.4 {
+  UPDATE t1 SET (c) = 99 WHERE a=3;
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 4 99 55 | 5 6 55 66 |}
+
+do_execsql_test 1.5 {
+  UPDATE t1 SET b = 8, (c,d) = (SELECT 123,456) WHERE a=3;
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 8 123 456 | 5 6 55 66 |}
+
+do_catchsql_test 2.1 {
+  UPDATE t1 SET (c,d) = (SELECT x,y,z FROM t2 WHERE w=a);
+} {1 {2 columns assigned 3 values}}
+
+do_catchsql_test 2.2 {
+  UPDATE t1 SET (b,c,d) = (SELECT x,y FROM t2 WHERE w=a);
+} {1 {3 columns assigned 2 values}}
+
+finish_test
diff --git a/test/rowvalue8.test b/test/rowvalue8.test
new file mode 100644
index 00000000..432dad12
--- /dev/null
+++ b/test/rowvalue8.test
@@ -0,0 +1,59 @@
+# 2016-08-22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Use of row values in CASE statements.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue8
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d);
+  INSERT INTO t1(a,b,c,d) VALUES
+     (1,1,2,3),
+     (2,2,3,4),
+     (3,1,2,4),
+     (4,2,3,5),
+     (5,3,4,6),
+     (6,4,5,9);
+  SELECT a, CASE (b,c) WHEN (1,2) THEN 'aleph'
+                       WHEN (2,3) THEN 'bet'
+                       WHEN (3,4) THEN 'gimel'
+                       ELSE '-' END,
+         '|'
+    FROM t1
+   ORDER BY a;
+} {1 aleph | 2 bet | 3 aleph | 4 bet | 5 gimel | 6 - |}
+do_execsql_test 1.2 {
+  SELECT a, CASE (b,c,d) WHEN (1,2,3) THEN 'aleph'
+                         WHEN (2,3,4) THEN 'bet'
+                         WHEN (3,4,6) THEN 'gimel'
+                         ELSE '-' END,
+         '|'
+    FROM t1
+   ORDER BY a;
+} {1 aleph | 2 bet | 3 - | 4 - | 5 gimel | 6 - |}
+
+do_execsql_test 2.1 {
+  CREATE TABLE t2(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t2(x,y) VALUES(1,6),(2,5),(3,4),(4,3),(5,2),(6,1);
+  SELECT x, CASE (SELECT b,c FROM t1 WHERE a=y)
+            WHEN (1,2) THEN 'aleph'
+            WHEN (2,3) THEN 'bet'
+            WHEN (3,4) THEN 'gimel'
+            ELSE '-' END,
+         '|'
+    FROM t2
+   ORDER BY +x;
+} {1 - | 2 gimel | 3 bet | 4 aleph | 5 bet | 6 aleph |}
+
+
+finish_test
diff --git a/test/rowvalue9.test b/test/rowvalue9.test
new file mode 100644
index 00000000..ce324f0b
--- /dev/null
+++ b/test/rowvalue9.test
@@ -0,0 +1,302 @@
+# 2016 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing SQL statements that use row value
+# constructors.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue9
+
+# Tests:
+#
+#  1.*: Test that affinities are handled correctly by various row-value
+#       operations without indexes.
+#
+#  2.*: Test an affinity bug that came up during testing.
+#
+#  3.*: Test a row-value version of the bug tested by 2.*.
+#
+#  4.*: Test that affinities are handled correctly by various row-value
+#       operations with assorted indexes.
+#
+
+do_execsql_test 1.0.1 {
+  CREATE TABLE a1(c, b INTEGER, a TEXT, PRIMARY KEY(a, b));
+ 
+  INSERT INTO a1 (rowid, c, b, a) VALUES(3,  '0x03', 1, 1);
+  INSERT INTO a1 (rowid, c, b, a) VALUES(14, '0x0E', 2, 2);
+  INSERT INTO a1 (rowid, c, b, a) VALUES(15, '0x0F', 3, 3);
+  INSERT INTO a1 (rowid, c, b, a) VALUES(92, '0x5C', 4, 4);
+
+  CREATE TABLE a2(x BLOB, y BLOB);
+  INSERT INTO a2(x, y) VALUES(1, 1);
+  INSERT INTO a2(x, y) VALUES(2, '2');
+  INSERT INTO a2(x, y) VALUES('3', 3);
+  INSERT INTO a2(x, y) VALUES('4', '4');
+}
+
+do_execsql_test 1.0.2 { 
+  SELECT x, typeof(x), y, typeof(y) FROM a2 ORDER BY rowid
+} {
+  1 integer 1 integer 
+  2 integer 2 text 
+  3 text    3 integer 
+  4 text    4 text
+}
+
+do_execsql_test 1.1.1 {
+  SELECT (SELECT rowid FROM a1 WHERE a=x AND b=y) FROM a2
+} {{} {} 15 92}
+do_execsql_test 1.1.2 {
+  SELECT (SELECT rowid FROM a1 WHERE (a, b) = (x, y)) FROM a2
+} {{} {} 15 92}
+
+do_execsql_test 1.2.3 {
+  SELECT a1.rowid FROM a1, a2 WHERE a=x AND b=y;
+} {15 92}
+do_execsql_test 1.2.4 {
+  SELECT a1.rowid FROM a1, a2 WHERE (a, b) = (x, y)
+} {15 92}
+
+
+do_execsql_test 1.3.1 {
+  SELECT a1.rowid FROM a1, a2 WHERE coalesce(NULL,x)=a AND coalesce(NULL,y)=b
+} {3 14 15 92}
+do_execsql_test 1.3.2 {
+  SELECT a1.rowid FROM a1, a2 
+  WHERE (coalesce(NULL,x), coalesce(NULL,y)) = (a, b)
+} {3 14 15 92}
+
+do_execsql_test 1.4.1 {
+  SELECT a1.rowid FROM a1, a2 WHERE +x=a AND +y=b
+} {3 14 15 92}
+do_execsql_test 1.4.2 {
+  SELECT a1.rowid FROM a1, a2 WHERE (+x, +y) = (a, b)
+} {3 14 15 92}
+
+do_execsql_test 1.5.1 {
+  SELECT (SELECT rowid FROM a1 WHERE a=+x AND b=+y) FROM a2
+} {3 14 15 92}
+do_execsql_test 1.5.2 {
+  SELECT (SELECT rowid FROM a1 WHERE (a, b) = (+x, +y)) FROM a2
+} {3 14 15 92}
+do_execsql_test 1.5.3 {
+  SELECT (SELECT rowid FROM a1 WHERE (+x, +y) = (a, b)) FROM a2
+} {3 14 15 92}
+
+do_execsql_test 1.6.1 {
+  SELECT a1.rowid FROM a1 WHERE (a, b) IN (SELECT x, y FROM a2)
+} {15 92}
+do_execsql_test 1.6.2 {
+  SELECT a1.rowid FROM a1, a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE a=x AND b=y
+  )
+} {3 14 15 92 3 14 15 92}
+
+# Test that [199df416] is fixed.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE b1(a TEXT);
+  CREATE TABLE b2(x BLOB);
+  INSERT INTO b1 VALUES(1);
+  INSERT INTO b2 VALUES(1);
+}
+do_execsql_test 2.2 { SELECT * FROM b1, b2 WHERE a=x; } {}
+do_execsql_test 2.3 { SELECT * FROM b1 WHERE a IN (SELECT x FROM b2) } {}
+do_execsql_test 2.4 { CREATE UNIQUE INDEX b1a ON b1(a); }
+do_execsql_test 2.5 { SELECT * FROM b1 WHERE a IN (SELECT x FROM b2) } {}
+
+# Test that a multi-column version of the query that revealed problem 
+# [199df416] also works.
+#
+do_execsql_test 3.1 {
+  CREATE TABLE c1(a INTEGER, b TEXT);
+  INSERT INTO c1 VALUES(1, 1);
+  CREATE TABLE c2(x BLOB, y BLOB);
+  INSERT INTO c2 VALUES(1, 1);
+}
+do_execsql_test 3.2 {
+  SELECT * FROM c1 WHERE (a, b) IN (SELECT x, y FROM c2)
+} {}
+do_execsql_test 3.3 {
+  CREATE UNIQUE INDEX c1ab ON c1(a, b);
+  SELECT * FROM c1 WHERE (a, b) IN (SELECT x, y FROM c2)
+} {}
+do_execsql_test 3.4 {
+  SELECT * FROM c1 WHERE (a, +b) IN (SELECT x, y FROM c2)
+} {}
+
+do_execsql_test 3.5 {
+  SELECT c1.rowid FROM c1 WHERE b = (SELECT y FROM c2);
+} {}
+do_execsql_test 3.6 {
+  SELECT c1.rowid FROM c1 WHERE (a, b) = (SELECT x, y FROM c2);
+} {}
+
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 4.0 {
+  CREATE TABLE d1(a TEXT, b INTEGER, c NUMERIC);
+  CREATE TABLE d2(x BLOB, y BLOB);
+
+  INSERT INTO d1 VALUES(1, 1, 1);
+  INSERT INTO d1 VALUES(2, 2, 2);
+  INSERT INTO d1 VALUES(3, 3, 3);
+  INSERT INTO d1 VALUES(4, 4, 4);
+
+  INSERT INTO d2 VALUES (1, 1);
+  INSERT INTO d2 VALUES (2, '2');
+  INSERT INTO d2 VALUES ('3', 3);
+  INSERT INTO d2 VALUES ('4', '4');
+}
+
+foreach {tn idx} {
+  1 {}
+  2 { CREATE INDEX idx ON d1(a) }
+  3 { CREATE INDEX idx ON d1(a, c) }
+  4 { CREATE INDEX idx ON d1(c) }
+  5 { CREATE INDEX idx ON d1(c, a) }
+
+  6 { 
+    CREATE INDEX idx ON d1(c, a) ;
+    CREATE INDEX idx1 ON d2(x, y);
+  }
+
+  7 { 
+    CREATE INDEX idx ON d1(c, a) ;
+    CREATE UNIQUE INDEX idx2 ON d2(x, y) ;
+  }
+
+  8 { 
+    CREATE INDEX idx ON d1(c) ;
+    CREATE UNIQUE INDEX idx2 ON d2(x);
+  }
+
+} {
+  execsql { DROP INDEX IF EXISTS idx } 
+  execsql { DROP INDEX IF EXISTS idx2 } 
+  execsql { DROP INDEX IF EXISTS idx3 } 
+  execsql $idx
+
+  do_execsql_test 4.$tn.1 {
+    SELECT rowid FROM d1 WHERE (a, c) IN (SELECT x, y FROM d2);
+  } {3 4}
+
+  do_execsql_test 4.$tn.2 {
+    SELECT rowid FROM d1 WHERE (c, a) IN (SELECT x, y FROM d2);
+  } {2 4}
+
+  do_execsql_test 4.$tn.3 {
+    SELECT rowid FROM d1 WHERE (+c, a) IN (SELECT x, y FROM d2);
+  } {2}
+
+  do_execsql_test 4.$tn.4 {
+    SELECT rowid FROM d1 WHERE (c, a) = (
+      SELECT x, y FROM d2 WHERE d2.rowid=d1.rowid
+    );
+  } {2 4}
+
+  do_execsql_test 4.$tn.5 {
+    SELECT d1.rowid FROM d1, d2 WHERE a = y;
+  } {2 4}
+
+  do_execsql_test 4.$tn.6 {
+    SELECT d1.rowid FROM d1 WHERE a = (
+      SELECT y FROM d2 where d2.rowid=d1.rowid
+    );
+  } {2 4}
+}
+
+do_execsql_test 5.0 {
+  CREATE TABLE e1(a TEXT, c NUMERIC);
+  CREATE TABLE e2(x BLOB, y BLOB);
+
+  INSERT INTO e1 VALUES(2, 2);
+
+  INSERT INTO e2 VALUES ('2', 2);
+  INSERT INTO e2 VALUES ('2', '2');
+  INSERT INTO e2 VALUES ('2', '2.0');
+
+  CREATE INDEX e1c ON e1(c);
+}
+
+do_execsql_test 5.1 {
+  SELECT rowid FROM e1 WHERE (a, c) IN (SELECT x, y FROM e2);
+} {1}
+do_execsql_test 5.2 {
+  SELECT rowid FROM e2 WHERE rowid IN (SELECT +c FROM e1);
+} {2}
+do_execsql_test 5.3 {
+  SELECT rowid FROM e2 WHERE rowid IN (SELECT 0+c FROM e1);
+} {2}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 6.0 {
+  CREATE TABLE f1(a, b);
+  CREATE TABLE f2(c, d);
+  CREATE TABLE f3(e, f);
+}
+
+do_execsql_test 6.1 {
+  SELECT * FROM f3 WHERE (e, f) IN (
+    SELECT a, b FROM f1 UNION ALL SELECT c, d FROM f2
+  );
+}
+do_execsql_test 6.2 {
+  CREATE INDEX f3e ON f3(e);
+  SELECT * FROM f3 WHERE (e, f) IN (
+    SELECT a, b FROM f1 UNION ALL SELECT c, d FROM f2
+  );
+}
+
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 7.0 {
+  CREATE TABLE g1(a, b); 
+  INSERT INTO g1 VALUES
+      (1, 1), (1, 2), (1, 3), (1, 'i'), (1, 'j'),
+      (1, 6), (1, 7), (1, 8), (1, 9), (1, 10),
+      (1, 4), (1, 5);
+
+  CREATE TABLE g2(x, y);
+  CREATE INDEX g2x ON g2(x);
+
+  INSERT INTO g2 VALUES(1, 4);
+  INSERT INTO g2 VALUES(1, 5);
+}
+
+do_execsql_test 7.1 {
+  SELECT * FROM g2 WHERE (x, y) IN (
+    SELECT a, b FROM g1 ORDER BY +a, +b LIMIT 10
+  );
+} { 1 4 1 5 }
+
+do_execsql_test 7.2 {
+  SELECT * FROM g2 WHERE (x, y) IN (
+    SELECT a, b FROM g1 ORDER BY a, b LIMIT 10
+  );
+} { 1 4 1 5 }
+
+do_execsql_test 7.3 {
+  SELECT * FROM g2 WHERE (x, y) IN (
+    SELECT a, b FROM g1 ORDER BY 1, 2 LIMIT 10
+  );
+} { 1 4 1 5 }
+
+
+finish_test
+
diff --git a/test/rowvaluefault.test b/test/rowvaluefault.test
new file mode 100644
index 00000000..0a4da6ba
--- /dev/null
+++ b/test/rowvaluefault.test
@@ -0,0 +1,72 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix rowvaluefault
+
+do_execsql_test 1.0 {
+  CREATE TABLE xyz(one, two, thr, fou);
+  INSERT INTO xyz VALUES('A', 'A', 'A',  1);
+  INSERT INTO xyz VALUES('B', 'B', 'B',  2);
+  INSERT INTO xyz VALUES('C', 'C', 'C',  3);
+  INSERT INTO xyz VALUES('D', 'D', 'D',  4);
+
+  CREATE UNIQUE INDEX xyz_one_two ON xyz(one, two);
+}
+
+do_faultsim_test 1 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (one, two, thr) = ('B', 'B', 'B') }
+} -test {
+  faultsim_test_result {0 2} 
+}
+
+do_faultsim_test 2 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (two, thr) IS ('C', 'C') }
+} -test {
+  faultsim_test_result {0 3} 
+}
+
+do_faultsim_test 3 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (one, two, thr) > ('B', 'B', 'B') }
+} -test {
+  faultsim_test_result {0 {3 4}} 
+}
+
+do_faultsim_test 4 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (one, two) IN (SELECT one, two FROM xyz) }
+} -test {
+  faultsim_test_result {0 {1 2 3 4}} 
+}
+
+do_faultsim_test 5 -faults oom* -body {
+  execsql { 
+    SELECT fou FROM xyz 
+    WHERE (one, two, thr) IN (SELECT one, two, thr FROM xyz) 
+  }
+} -test {
+  faultsim_test_result {0 {1 2 3 4}} 
+}
+
+do_faultsim_test 6 -faults oom* -body {
+  execsql { 
+    SELECT fou FROM xyz 
+    WHERE (one, two, thr) BETWEEN ('B', 'B', 'B') AND ('C', 'C', 'C') }
+} -test {
+  faultsim_test_result {0 {2 3}} 
+}
+
+finish_test
+
diff --git a/test/savepoint7.test b/test/savepoint7.test
index d8a02f1f..59c3cd6c 100644
--- a/test/savepoint7.test
+++ b/test/savepoint7.test
@@ -95,4 +95,38 @@ do_test savepoint7-2.2 {
   list $rc $msg [db eval {SELECT * FROM t2}]
 } {1 {abort due to ROLLBACK} {}}
 
+# Ticket: https://www.sqlite.org/src/tktview/7f7f8026eda387d544b
+# Segfault in the in-memory journal logic triggered by a tricky
+# combination of SAVEPOINT operations.
+#
+unset -nocomplain i
+for {set i 248} {$i<=253} {incr i} {
+  do_test savepoint7-3.$i {
+    db close
+    forcedelete test.db
+    sqlite3 db test.db
+    db eval {
+      PRAGMA page_size=1024;
+      PRAGMA temp_store=MEMORY;
+      BEGIN;
+      CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT);
+      WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<$::i)
+      INSERT INTO t1(x,y) SELECT x*10, printf('%04d%.800c',x,'*') FROM c;
+      SAVEPOINT one;
+        SELECT count(*) FROM t1;
+        WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<$::i)
+        INSERT INTO t1(x,y) SELECT x*10+1, printf('%04d%.800c',x,'*') FROM c;
+      ROLLBACK TO one;
+        SELECT count(*) FROM t1;
+        SAVEPOINT twoB;
+          WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<10)
+          INSERT INTO t1(x,y) SELECT x*10+2, printf('%04d%.800c',x,'*') FROM c;
+        ROLLBACK TO twoB;
+      RELEASE one;
+      COMMIT;
+    }
+  } [list $i $i]
+}
+
+
 finish_test
diff --git a/test/select4.test b/test/select4.test
index be8d0e0a..51a1b1c4 100644
--- a/test/select4.test
+++ b/test/select4.test
@@ -12,7 +12,6 @@
 # focus of this file is testing UNION, INTERSECT and EXCEPT operators
 # in SELECT statements.
 #
-# $Id: select4.test,v 1.30 2009/04/16 00:24:24 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -936,4 +935,76 @@ do_execsql_test select4-15.1 {
    ORDER BY 1;
 } {1 33 456 2 33 789}
 
+# Enhancement (2016-03-15):  Use a co-routine for subqueries if the
+# subquery is guaranteed to be the outer-most query
+#
+do_execsql_test select4-16.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,
+  PRIMARY KEY(a,b DESC)) WITHOUT ROWID;
+
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b,c,d)
+    SELECT x%10, x/10, x, printf('xyz%dabc',x) FROM c;
+
+  SELECT t3.c FROM 
+    (SELECT a,max(b) AS m FROM t1 WHERE a>=5 GROUP BY a) AS t2
+    JOIN t1 AS t3
+  WHERE t2.a=t3.a AND t2.m=t3.b
+  ORDER BY t3.a;
+} {95 96 97 98 99}
+do_execsql_test select4-16.2 {
+  SELECT t3.c FROM 
+    (SELECT a,max(b) AS m FROM t1 WHERE a>=5 GROUP BY a) AS t2
+    CROSS JOIN t1 AS t3
+  WHERE t2.a=t3.a AND t2.m=t3.b
+  ORDER BY t3.a;
+} {95 96 97 98 99}
+do_execsql_test select4-16.3 {
+  SELECT t3.c FROM 
+    (SELECT a,max(b) AS m FROM t1 WHERE a>=5 GROUP BY a) AS t2
+    LEFT JOIN t1 AS t3
+  WHERE t2.a=t3.a AND t2.m=t3.b
+  ORDER BY t3.a;
+} {95 96 97 98 99}
+
+# Ticket https://www.sqlite.org/src/tktview/f7f8c97e975978d45  on 2016-04-25
+#
+# The where push-down optimization from 2015-06-02 is suppose to disable
+# on aggregate subqueries.  But if the subquery is a compound where the
+# last SELECT is non-aggregate but some other SELECT is an aggregate, the
+# test is incomplete and the optimization is not properly disabled.
+# 
+# The following test cases verify that the fix works.
+#
+do_execsql_test select4-17.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a int, b int);
+  INSERT INTO t1 VALUES(1,2),(1,18),(2,19);
+  SELECT x, y FROM (
+    SELECT 98 AS x, 99 AS y
+    UNION
+    SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a
+  ) AS w WHERE y>=20
+  ORDER BY +x;
+} {1 20 98 99}
+do_execsql_test select4-17.2 {
+  SELECT x, y FROM (
+    SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a
+    UNION
+    SELECT 98 AS x, 99 AS y
+  ) AS w WHERE y>=20
+  ORDER BY +x;
+} {1 20 98 99}
+do_catchsql_test select4-17.3 {
+  SELECT x, y FROM (
+    SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a LIMIT 3
+    UNION
+    SELECT 98 AS x, 99 AS y
+  ) AS w WHERE y>=20
+  ORDER BY +x;
+} {1 {LIMIT clause should come after UNION not before}}
+
+
+
 finish_test
diff --git a/test/select7.test b/test/select7.test
index 0df84e13..d705ebfa 100644
--- a/test/select7.test
+++ b/test/select7.test
@@ -114,26 +114,22 @@ ifcapable {subquery && compound} {
       CREATE TABLE t2(a,b);
       SELECT 5 IN (SELECT a,b FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
   do_test select7-5.2 {
     catchsql {
       SELECT 5 IN (SELECT * FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
   do_test select7-5.3 {
     catchsql {
       SELECT 5 IN (SELECT a,b FROM t2 UNION SELECT b,a FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
   do_test select7-5.4 {
     catchsql {
       SELECT 5 IN (SELECT * FROM t2 UNION SELECT * FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
 }
 
 # Verify that an error occurs if you have too many terms on a
diff --git a/test/selectA.test b/test/selectA.test
index 03383389..78d04be1 100644
--- a/test/selectA.test
+++ b/test/selectA.test
@@ -1434,5 +1434,14 @@ do_catchsql_test 5.4 {
   SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE
 } {1 {1st ORDER BY term does not match any column in the result set}}
 
+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER);
+  CREATE TABLE t2(b TEXT);
+  INSERT INTO t2(b) VALUES('12345');
+  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
+} {12345}
+
 
 finish_test
diff --git a/test/session.test b/test/session.test
new file mode 100644
index 00000000..4fd1ec01
--- /dev/null
+++ b/test/session.test
@@ -0,0 +1,22 @@
+# 2008 June 23
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file runs all rtree related tests.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/permutations.test
+
+ifcapable session {
+  # First run tests with sqlite3_extended_error_codes() set, then
+  # again with it clear.
+  run_test_suite session_eec
+  run_test_suite session
+  run_test_suite session_strm
+}
+
+finish_test
diff --git a/test/shell1.test b/test/shell1.test
index 2fda62ee..9a966fd8 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -21,15 +21,7 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
@@ -53,7 +45,7 @@ do_test shell1-1.1.1b {
 } {1 1}
 # error on extra options
 do_test shell1-1.1.2 {
-  catchcmd "test.db \"select 3\" \"select 4\"" ""
+  catchcmd "test.db \"select+3\" \"select+4\"" ""
 } {0 {3
 4}}
 # error on extra options
@@ -84,20 +76,20 @@ do_test shell1-1.3.3 {
 
 # -echo                print commands before execution
 do_test shell1-1.4.1 {
-  catchcmd "-echo test.db" "" 
+  catchcmd "-echo test.db" ""
 } {0 {}}
 
 # -[no]header          turn headers on or off
 do_test shell1-1.5.1 {
-  catchcmd "-header test.db" "" 
+  catchcmd "-header test.db" ""
 } {0 {}}
 do_test shell1-1.5.2 {
-  catchcmd "-noheader test.db" "" 
+  catchcmd "-noheader test.db" ""
 } {0 {}}
 
 # -bail                stop after hitting an error
 do_test shell1-1.6.1 {
-  catchcmd "-bail test.db" "" 
+  catchcmd "-bail test.db" ""
 } {0 {}}
 
 # -interactive         force interactive I/O
@@ -111,40 +103,40 @@ do_test shell1-1.7.1 {
 
 # -batch               force batch I/O
 do_test shell1-1.8.1 {
-  catchcmd "-batch test.db" "" 
+  catchcmd "-batch test.db" ""
 } {0 {}}
 
 # -column              set output mode to 'column'
 do_test shell1-1.9.1 {
-  catchcmd "-column test.db" "" 
+  catchcmd "-column test.db" ""
 } {0 {}}
 
 # -csv                 set output mode to 'csv'
 do_test shell1-1.10.1 {
-  catchcmd "-csv test.db" "" 
+  catchcmd "-csv test.db" ""
 } {0 {}}
 
 # -html                set output mode to HTML
 do_test shell1-1.11.1 {
-  catchcmd "-html test.db" "" 
+  catchcmd "-html test.db" ""
 } {0 {}}
 
 # -line                set output mode to 'line'
 do_test shell1-1.12.1 {
-  catchcmd "-line test.db" "" 
+  catchcmd "-line test.db" ""
 } {0 {}}
 
 # -list                set output mode to 'list'
 do_test shell1-1.13.1 {
-  catchcmd "-list test.db" "" 
+  catchcmd "-list test.db" ""
 } {0 {}}
 
 # -separator 'x'       set output field separator (|)
 do_test shell1-1.14.1 {
-  catchcmd "-separator 'x' test.db" "" 
+  catchcmd "-separator 'x' test.db" ""
 } {0 {}}
 do_test shell1-1.14.2 {
-  catchcmd "-separator x test.db" "" 
+  catchcmd "-separator x test.db" ""
 } {0 {}}
 do_test shell1-1.14.3 {
   set res [catchcmd "-separator" ""]
@@ -155,7 +147,7 @@ do_test shell1-1.14.3 {
 
 # -stats               print memory stats before each finalize
 do_test shell1-1.14b.1 {
-  catchcmd "-stats test.db" "" 
+  catchcmd "-stats test.db" ""
 } {0 {}}
 
 # -nullvalue 'text'    set text string for NULL values
@@ -183,7 +175,7 @@ do_test shell1-1.16.1 {
 
 # check first token handling
 do_test shell1-2.1.1 {
-  catchcmd "test.db" ".foo" 
+  catchcmd "test.db" ".foo"
 } {1 {Error: unknown command or invalid arguments:  "foo". Enter ".help" for help}}
 do_test shell1-2.1.2 {
   catchcmd "test.db" ".\"foo OFF\""
@@ -284,11 +276,11 @@ do_test shell1-3.2.4 {
 # .databases             List names and files of attached databases
 do_test shell1-3.3.1 {
   catchcmd "-csv test.db" ".databases"
-} "/0 +.*main +[string map {/ .} [string range [get_pwd] 0 10]].*/"
+} "/0.+main.+[string map {/ ".{1,2}"} [string range [get_pwd] 0 10]].*/"
 do_test shell1-3.3.2 {
   # extra arguments ignored
   catchcmd "test.db" ".databases BAD"
-} "/0 +.*main +[string map {/ .} [string range [get_pwd] 0 10]].*/"
+} "/0.+main.+[string map {/ ".{1,2}"} [string range [get_pwd] 0 10]].*/"
 
 # .dump ?TABLE? ...      Dump the database in an SQL text format
 #                          If TABLE specified, only dump tables matching
@@ -576,7 +568,7 @@ do_test shell1-3.21.2 {
 do_test shell1-3.21.3 {
   # too many arguments
   catchcmd "test.db" ".schema FOO BAD"
-} {1 {Usage: .schema ?LIKE-PATTERN?}}
+} {1 {Usage: .schema ?--indent? ?LIKE-PATTERN?}}
 
 do_test shell1-3.21.4 {
   catchcmd "test.db" {
@@ -625,9 +617,9 @@ do_test shell1-3.23.2 {
 } {1 {Usage: .show}}
 
 # .stats ON|OFF          Turn stats on or off
-do_test shell1-3.23b.1 {
-  catchcmd "test.db" ".stats"
-} {1 {Usage: .stats on|off}}
+#do_test shell1-3.23b.1 {
+#  catchcmd "test.db" ".stats"
+#} {1 {Usage: .stats on|off}}
 do_test shell1-3.23b.2 {
   catchcmd "test.db" ".stats ON"
 } {0 {}}
@@ -637,7 +629,7 @@ do_test shell1-3.23b.3 {
 do_test shell1-3.23b.4 {
   # too many arguments
   catchcmd "test.db" ".stats OFF BAD"
-} {1 {Usage: .stats on|off}}
+} {1 {Usage: .stats ?on|off?}}
 
 # .tables ?TABLE?        List names of tables
 #                          If TABLE specified, only list tables matching
@@ -878,6 +870,8 @@ do_test shell1-5.0 {
     if {$i==0x0D || ($tcl_platform(platform)=="windows" && $i==0x1A)} {
       continue
     }
+    if {$i>=0xE0 && $tcl_platform(os)=="OpenBSD"}  continue
+    if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os)=="Linux"}  continue
     set hex [format %02X $i]
     set char [subst \\x$hex]; set oldChar $char
     set escapes [list]
@@ -913,9 +907,55 @@ do_test shell1-5.0 {
       error "failed with error: $res"
     }
     if {$res ne "$oldChar\n"} {
-      error "failed with byte $hex mismatch"
+      if {[llength $res] > 0} {
+        set got [format %02X [scan $res %c]]
+      } else {
+        set got 
+      }
+      error "failed with byte $hex mismatch, got $got"
     }
   }
 } {}
 
+# These test cases do not work on MinGW
+if 0 {
+
+# The string used here is the word "test" in Chinese.
+# In UTF-8, it is encoded as: \xE6\xB5\x8B\xE8\xAF\x95
+set test \u6D4B\u8BD5
+
+do_test shell1-6.0 {
+  set fileName $test; append fileName .db
+  catch {forcedelete $fileName}
+  set x [catchcmdex $fileName "CREATE TABLE t1(x);\n.schema\n"]
+  set code [lindex $x 0]
+  set res [string trim [lindex $x 1]]
+  if {$code ne "0"} {
+    error "failed with error: $res"
+  }
+  if {$res ne "CREATE TABLE t1(x);"} {
+    error "failed with mismatch: $res"
+  }
+  if {![file exists $fileName]} {
+    error "file \"$fileName\" (Unicode) does not exist"
+  }
+  forcedelete $fileName
+} {}
+
+do_test shell1-6.1 {
+  catch {forcedelete test3.db}
+  set x [catchcmdex test3.db \
+      "CREATE TABLE [encoding convertto utf-8 $test](x);\n.schema\n"]
+  set code [lindex $x 0]
+  set res [string trim [lindex $x 1]]
+  if {$code ne "0"} {
+    error "failed with error: $res"
+  }
+  if {$res ne "CREATE TABLE ${test}(x);"} {
+    error "failed with mismatch: $res"
+  }
+  forcedelete test3.db
+} {}
+}
+
 finish_test
diff --git a/test/shell2.test b/test/shell2.test
index 616610bd..2de6bf75 100644
--- a/test/shell2.test
+++ b/test/shell2.test
@@ -20,15 +20,7 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
@@ -51,7 +43,7 @@ do_test shell2-1.1.1 {
 # Shell silently ignores extra parameters.
 # Ticket [f5cb008a65].
 do_test shell2-1.2.1 {
-  set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg]
+  set rc [catch { eval exec $CLI \":memory:\" \"select+3\" \"select+4\" } msg]
   list $rc $msg
 } {0 {3
 4}}
diff --git a/test/shell3.test b/test/shell3.test
index ce1fd4ec..6e38021d 100644
--- a/test/shell3.test
+++ b/test/shell3.test
@@ -21,19 +21,23 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
 
+# There are inconsistencies in command-line argument quoting on Windows.
+# In particular, individual applications are responsible for command-line
+# parsing in Windows, not the shell.  Depending on whether the sqlite3.exe
+# program is compiled with MinGW or MSVC, the command-line parsing is
+# different.  This causes problems for the tests below.  To avoid
+# issues, these tests are disabled for windows.
+#
+if {$::tcl_platform(platform)=="windows"} {
+  finish_test
+  return
+}
+
 #----------------------------------------------------------------------------
 #   shell3-1.*: Basic tests for running SQL statments from command line.
 #
diff --git a/test/shell4.test b/test/shell4.test
index fcb0b2b7..88e5e69a 100644
--- a/test/shell4.test
+++ b/test/shell4.test
@@ -18,18 +18,11 @@
 #
 #   shell4-1.*: Basic tests specific to the "stats" command.
 #   shell4-2.*: Basic tests for ".trace"
+#   shell4-3.*: The ".read" command takes the shell out of interactive mode
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
@@ -61,9 +54,9 @@ do_test shell4-1.2.2 {
 } {0}
 
 # .stats ON|OFF          Turn stats on or off
-do_test shell4-1.3.1 {
-  catchcmd "test.db" ".stats"
-} {1 {Usage: .stats on|off}}
+#do_test shell4-1.3.1 {
+#  catchcmd "test.db" ".stats"
+#} {1 {Usage: .stats on|off}}
 do_test shell4-1.3.2 {
   catchcmd "test.db" ".stats ON"
 } {0 {}}
@@ -73,7 +66,7 @@ do_test shell4-1.3.3 {
 do_test shell4-1.3.4 {
   # too many arguments
   catchcmd "test.db" ".stats OFF BAD"
-} {1 {Usage: .stats on|off}}
+} {1 {Usage: .stats ?on|off?}}
 
 # NB. whitespace is important
 do_test shell4-1.4.1 {
@@ -132,5 +125,17 @@ do_test shell4-2.5 {
 } {0 {SELECT * FROM t1;}}
 }
 
+do_test shell4-3.1 {
+  set fd [open t1.txt wb]
+  puts $fd "SELECT 'squirrel';"
+  close $fd
+  exec $::CLI :memory: --interactive ".read t1.txt"
+} {squirrel}
+do_test shell4-3.2 {
+  set fd [open t1.txt wb]
+  puts $fd "SELECT 'pound: \302\243';"
+  close $fd
+  exec $::CLI :memory: --interactive ".read t1.txt"
+} {pound: £}
 
 finish_test
diff --git a/test/shell5.test b/test/shell5.test
index b921accc..bb2511a5 100644
--- a/test/shell5.test
+++ b/test/shell5.test
@@ -21,15 +21,7 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 
diff --git a/test/skipscan1.test b/test/skipscan1.test
index ac267116..6cdeed54 100644
--- a/test/skipscan1.test
+++ b/test/skipscan1.test
@@ -43,23 +43,23 @@ do_execsql_test skipscan1-1.2 {
 } {abc 345 7 8 | def 345 9 10 |}
 do_execsql_test skipscan1-1.2eqp {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a;
 } {/* USING INDEX t1abc (ANY(a) AND b=?)*/}
 do_execsql_test skipscan1-1.2sort {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a;
 } {~/*ORDER BY*/}
 
 do_execsql_test skipscan1-1.3 {
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a DESC;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a DESC;
 } {def 345 9 10 | abc 345 7 8 |}
 do_execsql_test skipscan1-1.3eqp {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a DESC;
 } {/* USING INDEX t1abc (ANY(a) AND b=?)*/}
 do_execsql_test skipscan1-1.3sort {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a DESC;
 } {~/*ORDER BY*/}
 
 do_execsql_test skipscan1-1.4 {
@@ -150,15 +150,15 @@ do_execsql_test skipscan1-2.1 {
 } {}
 
 do_execsql_test skipscan1-2.2 {
-  SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t2 WHERE d<>99 AND b=345 ORDER BY a;
 } {abc 345 7 8 | def 345 9 10 |}
 do_execsql_test skipscan1-2.2eqp {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t2 WHERE d<>99 AND b=345 ORDER BY a;
 } {/* USING INDEX sqlite_autoindex_t2_1 (ANY(a) AND b=?)*/}
 do_execsql_test skipscan1-2.2sort {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t2 WHERE d<>99 AND b=345 ORDER BY a;
 } {~/*ORDER BY*/}
 
 
@@ -322,4 +322,19 @@ do_execsql_test skipscan1-8.2 {
   ORDER BY +x;
 } {1 AB 2 CD}
 
+# Segfault reported on the mailing list by Keith Medcalf on 2016-09-18.
+# A skip-scan with a "column IN (SELECT ...)" on the second term of the
+# index.
+#
+do_execsql_test skipscan1-9.2 {
+  CREATE TABLE t9a(a,b,c);
+  CREATE INDEX t9a_ab ON t9a(a,b);
+  CREATE TABLE t9b(x,y);
+  ANALYZE sqlite_master;
+  INSERT INTO sqlite_stat1 VALUES('t9a','t9a_ab','1000000 250000 1');
+  ANALYZE sqlite_master;
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t9a WHERE b IN (SELECT x FROM t9b WHERE y!=5);
+} {/USING INDEX t9a_ab .ANY.a. AND b=./}
+
 finish_test
diff --git a/test/snapshot.test b/test/snapshot.test
index 63a0e301..b2d61941 100644
--- a/test/snapshot.test
+++ b/test/snapshot.test
@@ -17,6 +17,15 @@ source $testdir/tester.tcl
 ifcapable !snapshot {finish_test; return}
 set testprefix snapshot
 
+# This test does not work with the inmemory_journal permutation. The reason
+# is that each connection opened as part of this permutation executes
+# "PRAGMA journal_mode=memory", which fails if the database is in wal mode
+# and there are one or more existing connections.
+if {[permutation]=="inmemory_journal"} {
+  finish_test
+  return
+}
+
 #-------------------------------------------------------------------------
 # Check some error conditions in snapshot_get(). It is an error if:
 #
@@ -334,6 +343,96 @@ do_test 6.3 {
   sqlite3_snapshot_open db2 main $::snapshot
   db2 eval { SELECT * FROM x1 }
 } {z zz zzz}
+do_test 6.4 {
+  db2 close
+  sqlite3 db2 test.db 
+  db2 eval "PRAGMA application_id"
+  db2 eval "BEGIN"
+  sqlite3_snapshot_open db2 main $::snapshot
+  db2 eval { SELECT * FROM x1 }
+} {z zz zzz}
+
+do_test 6.5 {
+  db2 close
+  sqlite3 db2 test.db 
+  db2 eval "BEGIN"
+  list [catch {sqlite3_snapshot_open db2 main $::snapshot} msg] $msg
+} {1 SQLITE_ERROR}
+
 sqlite3_snapshot_free $snapshot
 
+#-------------------------------------------------------------------------
+# The following tests investigate the sqlite3_snapshot_cmp() API.
+#
+
+# Compare snapshots $p1 and $p2, checking that the result is $r.
+#
+proc do_snapshot_cmp_test {tn p1 p2 r} {
+  uplevel [list do_test $tn.1 [list sqlite3_snapshot_cmp $p1 $p2] $r]
+  uplevel [list do_test $tn.2 [list sqlite3_snapshot_cmp $p2 $p1] [expr $r*-1]]
+  uplevel [list do_test $tn.3 [list sqlite3_snapshot_cmp $p1 $p1] 0]
+  uplevel [list do_test $tn.4 [list sqlite3_snapshot_cmp $p2 $p2] 0]
+}
+
+catch { db2 close }
+reset_db
+
+do_execsql_test 7.1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x);
+} wal
+
+do_test 7.1.2 {
+  execsql { BEGIN ; PRAGMA application_id }
+  set p1 [sqlite3_snapshot_get db main]
+  execsql {
+    INSERT INTO t1 VALUES(10);
+    COMMIT;
+  }
+  execsql { BEGIN ; PRAGMA application_id }
+  set p2 [sqlite3_snapshot_get db main]
+  execsql COMMIT
+} {}
+
+do_snapshot_cmp_test 7.1.3 $p1 $p2 -1
+sqlite3_snapshot_free $p1
+sqlite3_snapshot_free $p2
+
+do_execsql_test 7.2.1 {
+  INSERT INTO t1 VALUES(11);
+  INSERT INTO t1 VALUES(12);
+  INSERT INTO t1 VALUES(13);
+  BEGIN; 
+    PRAGMA application_id;
+} {0}
+do_test 7.2.2 {
+  set p1 [sqlite3_snapshot_get db main]
+  execsql {
+    COMMIT;
+    INSERT INTO t1 VALUES(14);
+    PRAGMA wal_checkpoint;
+    BEGIN;
+      PRAGMA application_id;
+  }
+  set p2 [sqlite3_snapshot_get db main]
+  execsql COMMIT
+} {}
+
+do_snapshot_cmp_test 7.2.3 $p1 $p2 -1
+sqlite3_snapshot_free $p2
+
+do_test 7.3.1 {
+  execsql {
+    INSERT INTO t1 VALUES(14);
+    BEGIN;
+      PRAGMA application_id;
+  }
+  set p2 [sqlite3_snapshot_get db main]
+  execsql COMMIT
+} {}
+
+do_snapshot_cmp_test 7.3.2 $p1 $p2 -1
+sqlite3_snapshot_free $p1
+sqlite3_snapshot_free $p2
+
 finish_test
diff --git a/test/snapshot_fault.test b/test/snapshot_fault.test
index 3ac13dae..f90690e3 100644
--- a/test/snapshot_fault.test
+++ b/test/snapshot_fault.test
@@ -146,7 +146,7 @@ do_faultsim_test 3.0 -prep {
     error $msg
   }
 } -test {
-  faultsim_test_result {0 {}} {1 SQLITE_IOERR} \
+  faultsim_test_result {0 {}} {1 SQLITE_IOERR} {1 SQLITE_NOMEM} \
                               {1 SQLITE_IOERR_NOMEM} {1 SQLITE_IOERR_READ}
   if {$testrc==0} {
     set res [db eval { 
diff --git a/test/sort5.test b/test/sort5.test
index 5b1292bb..6ada81c9 100644
--- a/test/sort5.test
+++ b/test/sort5.test
@@ -41,4 +41,78 @@ do_execsql_test 1.2 {
 
 db close
 tvfs delete
+
+#-------------------------------------------------------------------------
+# Test that the PMA size is determined correctly. The PMA size should be
+# roughly the same amount of memory allocated to the main pager cache, or
+# 250 pages if this is larger.
+#
+testvfs tvfs
+tvfs script tv_callback
+tvfs filter {xOpen xWrite}
+
+proc tv_callback {method args} {
+  global iTemp
+  global F
+  switch $method {
+    xOpen {
+      if {[lindex $args 0]==""} { return "temp[incr iTemp]" }
+      return "SQLITE_OK"
+    }
+
+    xWrite {
+      foreach {filename id off amt} $args {}
+      if {[info exists F($id)]==0 || $F($id)<($off + $amt)} {
+        set F($id) [expr $off+$amt]
+      }
+    }
+  }
+}
+
+catch { db close }
+forcedelete test.db
+sqlite3 db test.db -vfs tvfs
+execsql { CREATE TABLE t1(x) }
+
+# Each iteration of the following loop attempts to sort 10001 records
+# each a bit over 100 bytes in size. In total a little more than 1MiB 
+# of data.
+#
+foreach {tn pgsz cachesz bTemp} {
+  1 4096   1000  0
+  2 1024   1000  1
+
+  3 4096  -1000  1
+  4 1024  -1000  1
+
+  5 4096  -9000  0
+  6 1024  -9000  0
+} {
+  do_execsql_test 2.$tn.0 "
+    PRAGMA page_size = $pgsz;
+    VACUUM;
+    PRAGMA cache_size = $cachesz;
+  "
+
+  if {[db one {PRAGMA page_size}]!=$pgsz} {
+    # SEE is not able to change page sizes and that messes up the
+    # results that follow.
+    continue
+  }
+
+  do_test 2.$tn.1 {
+    set ::iTemp 0
+    catch { array unset F }
+    execsql {
+      WITH x(i, j) AS (
+        SELECT 1, randomblob(100)
+        UNION ALL
+        SELECT i+1, randomblob(100) FROM x WHERE i<10000
+      )
+      SELECT * FROM x ORDER BY j;
+    }
+    expr {[array names F]!=""}
+  } $bTemp
+}
+
 finish_test
diff --git a/test/speed3.test b/test/speed3.test
index 1beaeb74..4aa90943 100644
--- a/test/speed3.test
+++ b/test/speed3.test
@@ -105,7 +105,7 @@ proc io_log {db} {
   puts "Normal    : Read $stats2(read), wrote $stats2(write)"
 }
 
-proc reset_db {} {
+proc speed3_reset_db {} {
   db close
   sqlite3 db test.db
   db eval { 
@@ -117,7 +117,7 @@ proc reset_db {} {
 }
 
 forcedelete test2.db test2.db-journal
-reset_db
+speed3_reset_db
 
 # Set up a database in auto-vacuum mode and create a database schema.
 #
@@ -154,7 +154,7 @@ do_test speed3-0.4 {
 # Delete all content in a table, one row at a time.
 #
 #io_log db
-reset_db
+speed3_reset_db
 speed_trial speed3-1.incrvacuum $::NROW row {DELETE FROM main.t1 WHERE 1}
 speed_trial speed3-1.normal     $::NROW row {DELETE FROM aux.t1 WHERE 1}
 io_log db
@@ -164,7 +164,7 @@ io_log db
 #
 #db eval {PRAGMA incremental_vacuum(500000)}
 populate_t1 db
-reset_db
+speed3_reset_db
 speed_trial speed3-2.incrvacuum $::NROW row {SELECT c FROM main.t1}
 speed_trial speed3-2.normal     $::NROW row {SELECT c FROM aux.t1}
 io_log db
diff --git a/test/speedtest1.c b/test/speedtest1.c
index b41c7320..7ae6739f 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -30,6 +30,7 @@ static const char zHelp[] =
   "  --shrink-memory     Invoke sqlite3_db_release_memory() frequently.\n"
   "  --size N            Relative test size.  Default=100\n"
   "  --stats             Show statistics at the end\n"
+  "  --temp N            N from 0 to 9.  0: no temp table. 9: all temp tables\n"
   "  --testset T         Run test-set T\n"
   "  --trace             Turn on SQL tracing\n"
   "  --threads N         Use up to N threads for sorting\n"
@@ -69,6 +70,7 @@ static struct Global {
   int bExplain;              /* Print SQL with EXPLAIN prefix */
   int bVerify;               /* Try to verify that results are correct */
   int bMemShrink;            /* Call sqlite3_db_release_memory() often */
+  int eTemp;                 /* 0: no TEMP.  9: always TEMP. */
   int szTest;                /* Scale factor for test iterations */
   const char *zWR;           /* Might be WITHOUT ROWID */
   const char *zNN;           /* Might be NOT NULL */
@@ -78,6 +80,12 @@ static struct Global {
   char zResult[3000];        /* Text of the current result */
 } g;
 
+/* Return " TEMP" or "", as appropriate for creating a table.
+*/
+static const char *isTemp(int N){
+  return g.eTemp>=N ? " TEMP" : "";
+}
+
 
 /* Print an error message and exit */
 static void fatal_error(const char *zMsg, ...){
@@ -413,12 +421,14 @@ void speedtest1_run(void){
   speedtest1_shrink_memory();
 }
 
+#ifndef SQLITE_OMIT_DEPRECATED
 /* The sqlite3_trace() callback function */
 static void traceCallback(void *NotUsed, const char *zSql){
   int n = (int)strlen(zSql);
   while( n>0 && (zSql[n-1]==';' || ISSPACE(zSql[n-1])) ) n--;
   fprintf(stderr,"%.*s;\n", n, zSql);
 }
+#endif /* SQLITE_OMIT_DEPRECATED */
 
 /* Substitute random() function that gives the same random
 ** sequence on each run, for repeatability. */
@@ -459,8 +469,8 @@ void testset_main(void){
   maxb = roundup_allones(sz);
   speedtest1_begin_test(100, "%d INSERTs into table with no index", n);
   speedtest1_exec("BEGIN");
-  speedtest1_exec("CREATE TABLE t1(a INTEGER %s, b INTEGER %s, c TEXT %s);",
-                  g.zNN, g.zNN, g.zNN);
+  speedtest1_exec("CREATE%s TABLE t1(a INTEGER %s, b INTEGER %s, c TEXT %s);",
+                  isTemp(9), g.zNN, g.zNN, g.zNN);
   speedtest1_prepare("INSERT INTO t1 VALUES(?1,?2,?3); --  %d times", n);
   for(i=1; i<=n; i++){
     x1 = swizzle(i,maxb);
@@ -477,8 +487,9 @@ void testset_main(void){
   n = sz;
   speedtest1_begin_test(110, "%d ordered INSERTS with one index/PK", n);
   speedtest1_exec("BEGIN");
-  speedtest1_exec("CREATE TABLE t2(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
-                   g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
+  speedtest1_exec(
+     "CREATE%s TABLE t2(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
+     isTemp(5), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
   speedtest1_prepare("INSERT INTO t2 VALUES(?1,?2,?3); -- %d times", n);
   for(i=1; i<=n; i++){
     x1 = swizzle(i,maxb);
@@ -495,8 +506,9 @@ void testset_main(void){
   n = sz;
   speedtest1_begin_test(120, "%d unordered INSERTS with one index/PK", n);
   speedtest1_exec("BEGIN");
-  speedtest1_exec("CREATE TABLE t3(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
-                   g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
+  speedtest1_exec(
+      "CREATE%s TABLE t3(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
+      isTemp(3), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
   speedtest1_prepare("INSERT INTO t3 VALUES(?1,?2,?3); -- %d times", n);
   for(i=1; i<=n; i++){
     x1 = swizzle(i,maxb);
@@ -654,12 +666,12 @@ void testset_main(void){
   speedtest1_begin_test(180, "%d INSERTS with three indexes", n);
   speedtest1_exec("BEGIN");
   speedtest1_exec(
-    "CREATE TABLE t4(\n"
+    "CREATE%s TABLE t4(\n"
     "  a INTEGER %s %s,\n"
     "  b INTEGER %s,\n"
     "  c TEXT %s\n"
     ") %s",
-    g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
+    isTemp(1), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
   speedtest1_exec("CREATE INDEX t4b ON t4(b)");
   speedtest1_exec("CREATE INDEX t4c ON t4(c)");
   speedtest1_exec("INSERT INTO t4 SELECT * FROM t1");
@@ -1047,7 +1059,7 @@ void testset_rtree(int p1, int p2){
   speedtest1_end_test();
 
   speedtest1_begin_test(101, "Copy from rtree to a regular table");
-  speedtest1_exec("CREATE TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)");
+  speedtest1_exec(" TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)");
   speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1");
   speedtest1_end_test();
 
@@ -1167,6 +1179,49 @@ void testset_debug1(void){
   }
 }
 
+#ifdef __linux__
+#include 
+#include 
+
+/*
+** Attempt to display I/O stats on Linux using /proc/PID/io
+*/
+static void displayLinuxIoStats(FILE *out){
+  FILE *in;
+  char z[200];
+  sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());
+  in = fopen(z, "rb");
+  if( in==0 ) return;
+  while( fgets(z, sizeof(z), in)!=0 ){
+    static const struct {
+      const char *zPattern;
+      const char *zDesc;
+    } aTrans[] = {
+      { "rchar: ",                  "Bytes received by read():" },
+      { "wchar: ",                  "Bytes sent to write():"    },
+      { "syscr: ",                  "Read() system calls:"      },
+      { "syscw: ",                  "Write() system calls:"     },
+      { "read_bytes: ",             "Bytes rcvd from storage:"  },
+      { "write_bytes: ",            "Bytes sent to storage:"    },
+      { "cancelled_write_bytes: ",  "Cancelled write bytes:"    },
+    };
+    int i;
+    for(i=0; i=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
+        i++;
+        if( argv[i][0]<'0' || argv[i][0]>'9' || argv[i][1]!=0 ){
+          fatal_error("argument to --temp should be integer between 0 and 9");
+        }
+        g.eTemp = argv[i][0] - '0';
       }else if( strcmp(z,"testset")==0 ){
         if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
         zTSet = argv[++i];
@@ -1309,11 +1375,7 @@ int main(int argc, char **argv){
                   argv[i], argv[0]);
     }
   }
-#if 0
-  if( zDbName==0 ){
-    fatal_error(zHelp, argv[0]);
-  }
-#endif
+  if( zDbName!=0 ) unlink(zDbName);
 #if SQLITE_VERSION_NUMBER>=3006001
   if( nHeap>0 ){
     pHeap = malloc( nHeap );
@@ -1356,7 +1418,9 @@ int main(int argc, char **argv){
 
   /* Set database connection options */
   sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0);
+#ifndef SQLITE_OMIT_DEPRECATED
   if( doTrace ) sqlite3_trace(g.db, traceCallback, 0);
+#endif
   speedtest1_exec("PRAGMA threads=%d", nThread);
   if( zKey ){
     speedtest1_exec("PRAGMA key('%s')", zKey);
@@ -1457,6 +1521,12 @@ int main(int argc, char **argv){
   }
 #endif
 
+#ifdef __linux__
+  if( showStats ){
+    displayLinuxIoStats(stdout);
+  }
+#endif
+
   /* Release memory */
   free( pLook );
   free( pPCache );
diff --git a/test/spellfix3.test b/test/spellfix3.test
index ce002edd..e24ea31e 100644
--- a/test/spellfix3.test
+++ b/test/spellfix3.test
@@ -35,9 +35,18 @@ do_execsql_test 140 {
 } {160}
 do_execsql_test 200 {
   SELECT spellfix1_scriptcode('+3.14159');
-} {999}
+} {215}
 do_execsql_test 210 {
   SELECT spellfix1_scriptcode('And God said: "Да будет свет"');
 } {998}
+do_execsql_test 220 {
+  SELECT spellfix1_scriptcode('+3.14159 light');
+} {215}
+do_execsql_test 230 {
+  SELECT spellfix1_scriptcode('+3.14159 свет');
+} {220}
+do_execsql_test 240 {
+  SELECT spellfix1_scriptcode('וַיֹּ֥אמֶר +3.14159');
+} {125}
 
 finish_test
diff --git a/test/sqldiff1.test b/test/sqldiff1.test
index 3201fb36..ea4e1f99 100644
--- a/test/sqldiff1.test
+++ b/test/sqldiff1.test
@@ -14,16 +14,8 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-if {$tcl_platform(platform)=="windows"} {
-  set PROG "sqldiff.exe"
-} else {
-  set PROG "./sqldiff"
-}
-if {![file exe $PROG]} {
-  puts "sqldiff cannot run because $PROG is not available"
-  finish_test
-  return
-}
+set PROG [test_find_sqldiff]
+
 db close
 forcedelete test.db test2.db
 sqlite3 db test.db
diff --git a/test/stat.test b/test/stat.test
index 288153db..66ca5e2f 100644
--- a/test/stat.test
+++ b/test/stat.test
@@ -21,6 +21,10 @@ ifcapable !vtab||!compound {
   return
 }
 
+# This module uses hard-coded results that depend on exact measurements of
+# pages sizes at the byte level, and hence will not work if the reserved_bytes
+# value is nonzero.
+if {[nonzero_reserved_bytes]} {finish_test; return;}
 
 set ::asc 1
 proc a_string {n} { string range [string repeat [incr ::asc]. $n] 1 $n }
@@ -28,12 +32,28 @@ db func a_string a_string
 
 register_dbstat_vtab db
 do_execsql_test stat-0.0 {
+  PRAGMA table_info(dbstat);
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+
+# Attempts to drop an eponymous virtual table are a no-op.
+do_execsql_test stat-0.1 {
+  DROP TABLE dbstat;
+  PRAGMA table_info=dbstat;
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+db func a_string a_string
+register_dbstat_vtab db
+do_execsql_test stat-0.2 {
   PRAGMA auto_vacuum = OFF;
   CREATE VIRTUAL TABLE temp.stat USING dbstat;
   SELECT * FROM stat;
 } {}
 
-ifcapable wal {
+
+if {[wal_is_capable]} {
   do_execsql_test stat-0.1 {
     PRAGMA journal_mode = WAL;
     PRAGMA journal_mode = delete;
diff --git a/test/stmt.test b/test/stmt.test
index 49a41414..df501f76 100644
--- a/test/stmt.test
+++ b/test/stmt.test
@@ -46,7 +46,8 @@ do_test stmt-1.4 {
     INSERT INTO t1 SELECT a+1, b+1 FROM t1;
   }
   set sqlite_open_file_count
-} {3}
+  # 2016-03-04: statement-journal open deferred
+} {2}
 do_test stmt-1.5 {
   execsql COMMIT
   set sqlite_open_file_count
@@ -61,7 +62,8 @@ do_test stmt-1.6.1 {
 do_test stmt-1.6.2 {
   execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 }
   set sqlite_open_file_count
-} {3}
+  # 2016-03-04: statement-journal open deferred
+} {2}
 do_test stmt-1.7 {
   execsql COMMIT
   set sqlite_open_file_count
@@ -84,7 +86,7 @@ filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9   } 2
 filecount stmt-2.4 { 
     INSERT INTO t1 SELECT 9, 9;
     INSERT INTO t1 SELECT 10, 10;
-} 3
+} 2
 
 do_test stmt-2.5 {
   execsql { CREATE INDEX i1 ON t1(b) }
@@ -92,6 +94,6 @@ do_test stmt-2.5 {
 filecount stmt-2.6 { 
   REPLACE INTO t1 VALUES(5, 5);
   REPLACE INTO t1 VALUES(5, 5); 
-} 3
+} 2
 
 finish_test
diff --git a/test/subselect.test b/test/subselect.test
index 247f68ee..4d0efde4 100644
--- a/test/subselect.test
+++ b/test/subselect.test
@@ -40,7 +40,7 @@ do_test subselect-1.1 {
 do_test subselect-1.2 {
   set v [catch {execsql {SELECT * FROM t1 WHERE a = (SELECT * FROM t1)}} msg]
   lappend v $msg
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {row value misused}}
 
 # A subselect without an aggregate.
 #
diff --git a/test/superlock.test b/test/superlock.test
index 8199d521..704b0677 100644
--- a/test/superlock.test
+++ b/test/superlock.test
@@ -15,6 +15,7 @@ source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 
 set testprefix superlock
+do_not_use_codec
 
 # Test organization:
 #
@@ -238,13 +239,23 @@ db_swap test.db2 test.db
 do_catchsql_test 6.9 { SELECT * FROM t1 } {0 {1 2 3 4}}
 do_catchsql_test 6.10 { SELECT * FROM t2 } {1 {no such table: t2}}
 
-do_execsql_test  6.11 { 
-  PRAGMA journal_mode = delete;
-  PRAGMA page_size = 512;
-  VACUUM;
-  PRAGMA journal_mode = wal;
-  INSERT INTO t1 VALUES(5, 6);
-} {delete wal}
+if {[nonzero_reserved_bytes]} {
+  # Vacuum with a size change is not allowed with the codec
+  do_execsql_test  6.11codec { 
+    PRAGMA journal_mode = delete;
+    VACUUM;
+    PRAGMA journal_mode = wal;
+    INSERT INTO t1 VALUES(5, 6);
+  } {delete wal}
+} else {
+  do_execsql_test  6.11 { 
+    PRAGMA journal_mode = delete;
+    PRAGMA page_size = 512;
+    VACUUM;
+    PRAGMA journal_mode = wal;
+    INSERT INTO t1 VALUES(5, 6);
+  } {delete wal}
+}
 
 db_swap test.db2 test.db
 do_catchsql_test 6.12 { SELECT * FROM t1 } {1 {no such table: t1}}
diff --git a/test/sync.test b/test/sync.test
index 2ee636a2..210039ac 100644
--- a/test/sync.test
+++ b/test/sync.test
@@ -81,6 +81,7 @@ do_test sync-1.3 {
   set sqlite_sync_count
 } 11
 ifcapable pager_pragmas {
+if {[permutation]!="journaltest"} {
   do_test sync-1.4 {
     set sqlite_sync_count 0
     execsql {
@@ -94,6 +95,7 @@ ifcapable pager_pragmas {
     set sqlite_sync_count
   } 0
 }
+}
 
 
 finish_test
diff --git a/test/tabfunc01.test b/test/tabfunc01.test
index d85cb20f..19d3cc66 100644
--- a/test/tabfunc01.test
+++ b/test/tabfunc01.test
@@ -22,6 +22,7 @@ ifcapable !vtab {
   return
 }
 load_static_extension db series
+load_static_extension db carray
 
 do_execsql_test tabfunc01-1.1 {
   SELECT *, '|' FROM generate_series WHERE start=1 AND stop=9 AND step=2;
@@ -120,4 +121,76 @@ do_catchsql_test tabfunc01-4.3 {
   SELECT * FROM aux1.generate_series(1,4)
 } {1 {no such table: aux1.generate_series}}
 
+# The next series of tests is verifying that virtual table are able
+# to optimize the IN operator, even on terms that are not marked "omit".
+# When the generate_series virtual table is compiled for the testfixture,
+# the special -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 option is used, which
+# causes the xBestIndex method of generate_series to leave the
+# sqlite3_index_constraint_usage.omit flag set to 0, which should cause
+# the SQLite core to verify the start=, stop=, and step= constraints on
+# each step of output.  At one point, the IN operator could not be used
+# by virtual tables unless omit was set.
+#
+do_execsql_test tabfunc01-500 {
+  SELECT * FROM generate_series WHERE start IN (1,7) AND stop=20 AND step=10
+  ORDER BY +1;
+} {1 7 11 17}
+
+# Table-valued functions on the RHS of an IN operator
+#
+do_execsql_test tabfunc01-600 {
+  CREATE TABLE t600(a INTEGER PRIMARY KEY, b TEXT);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+    INSERT INTO t600(a,b) SELECT x, printf('(%03d)',x) FROM c;
+  SELECT b FROM t600 WHERE a IN generate_series(2,52,10);
+} {(002) (012) (022) (032) (042) (052)}
+
+
+do_test tabfunc01-700 {
+  set PTR [intarray_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5) WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-701 {
+  db eval {
+    SELECT b FROM t600 WHERE a IN carray($PTR,5,'int32');
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-702 {
+  db eval {
+    SELECT b FROM t600 WHERE a IN carray($PTR,4,'int32');
+  }
+} {(005) (007) (013) (017)}
+do_catchsql_test tabfunc01-710 {
+  SELECT b FROM t600 WHERE a IN carray($PTR,5,'int33');
+} {1 {unknown datatype: 'int33'}}
+
+do_test tabfunc01-720 {
+  set PTR [int64array_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5,'int64') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+do_test tabfunc01-730 {
+  set PTR [doublearray_addr 5.0 7.0 13.0 17.0 23.0]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5,'double') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+do_test tabfunc01-740 {
+  set PTR [textarray_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR,5,'char*') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+
+intarray_addr
+int64array_addr
+doublearray_addr
+textarray_addr
+
 finish_test
diff --git a/test/tclsqlite.test b/test/tclsqlite.test
index 8d7fea0d..bdb0fc56 100644
--- a/test/tclsqlite.test
+++ b/test/tclsqlite.test
@@ -22,10 +22,9 @@ source $testdir/tester.tcl
 
 # Check the error messages generated by tclsqlite
 #
+set r "sqlite_orig HANDLE FILENAME ?-vfs VFSNAME? ?-readonly BOOLEAN? ?-create BOOLEAN? ?-nomutex BOOLEAN? ?-fullmutex BOOLEAN? ?-uri BOOLEAN?"
 if {[sqlite3 -has-codec]} {
-  set r "sqlite_orig HANDLE FILENAME ?-key CODEC-KEY?"
-} else {
-  set r "sqlite_orig HANDLE FILENAME ?-vfs VFSNAME? ?-readonly BOOLEAN? ?-create BOOLEAN? ?-nomutex BOOLEAN? ?-fullmutex BOOLEAN? ?-uri BOOLEAN?"
+  append r " ?-key CODECKEY?"
 }
 do_test tcl-1.1 {
   set v [catch {sqlite3 bogus} msg]
@@ -35,7 +34,7 @@ do_test tcl-1.1 {
 do_test tcl-1.2 {
   set v [catch {db bogus} msg]
   lappend v $msg
-} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, transaction, unlock_notify, update_hook, version, or wal_hook}}
+} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}}
 do_test tcl-1.2.1 {
   set v [catch {db cache bogus} msg]
   lappend v $msg
@@ -636,6 +635,39 @@ do_test tcl-14.2 {
   db one {SELECT x FROM t6 WHERE xCall()!='value'}
 } {}
 
+# Verify that the "exists" and "onecolumn" methods work when
+# a "profile" is registered.
+#
+catch {db close}
+sqlite3 db :memory:
+proc noop-profile {args} {
+  return
+}
+do_test tcl-15.0 {
+  db eval {CREATE TABLE t1(a); INSERT INTO t1 VALUES(1),(2),(3);}
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.1 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.2 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+db profile noop-profile
+do_test tcl-15.3 {
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.4 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.5 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+
+
+
+
+
 
 
 finish_test
diff --git a/test/tempdb.test b/test/tempdb.test
index ecd23d4e..61416ec8 100644
--- a/test/tempdb.test
+++ b/test/tempdb.test
@@ -76,7 +76,7 @@ do_test tempdb-2.2 {
   }
   catchsql { INSERT INTO t1 SELECT * FROM t2 }
   set sqlite_open_file_count
-} [expr 1 + (0==$jrnl_in_memory) + (0==$subj_in_memory)]
+} [expr 1 + (0==$jrnl_in_memory)]
 do_test tempdb-2.3 {
   execsql {
     PRAGMA temp_store = 'memory';
diff --git a/test/tempdb2.test b/test/tempdb2.test
new file mode 100644
index 00000000..bc2822e7
--- /dev/null
+++ b/test/tempdb2.test
@@ -0,0 +1,77 @@
+# 2016 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix tempdb2
+
+db close
+sqlite3 db ""
+
+proc int2str {i} { string range [string repeat "$i." 450] 0 899 }
+db func int2str int2str
+
+#-------------------------------------------------------------------------
+#
+#  1.1: Write a big transaction to the db. One so large that it forces
+#       the file to be created and the cache flushed to disk on COMMIT.
+#
+#  1.2: Write a small transaction - one small enough that it remains in
+#       memory on COMMIT. All the pages of table [t1] are now dirty.
+#
+#  1.3: Delete the contents of [t1]. This moves all of its leaves to the
+#       free-list and causes the btree layer to call PagerDontWrite() on
+#       each of them.
+#
+#       Then do a big update on table [t2]. So big that the former leaves
+#       of [t1] are forced out of the cache. Then roll back the transaction.
+#       If the PagerDontWrite() calls are honoured and the data is not written
+#       to disk, the update made in test 1.2 will be lost at this point. Or, if
+#       they are ignored (as they should be for temp databases), the update
+#       will be safely written out to disk before the cache entries are
+#       discarded.
+#
+do_execsql_test 1.1 {
+  PRAGMA page_size=1024;
+  PRAGMA cache_size=50;
+
+  BEGIN;
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(1, int2str(1));
+    INSERT INTO t1 VALUES(2, int2str(1));
+    INSERT INTO t1 VALUES(3, int2str(1));
+
+    CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+    WITH c(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100 ) 
+    INSERT INTO t2 SELECT x, int2str(x) FROM c;
+  COMMIT;
+
+  PRAGMA lock_status;
+} {main unlocked temp closed}
+
+do_execsql_test 1.2 {
+  UPDATE t1 SET b=int2str(2);
+  SELECT b=int2str(2) FROM t1
+} {1 1 1}
+
+do_execsql_test 1.3 {
+  BEGIN;
+    DELETE FROM t1;
+    UPDATE t2 SET b=int2str(a+1);
+  ROLLBACK;
+}
+
+do_execsql_test 1.4 {
+  SELECT b=int2str(2) FROM t1
+} {1 1 1}
+
+finish_test
+
diff --git a/test/tempfault.test b/test/tempfault.test
new file mode 100644
index 00000000..4eae116a
--- /dev/null
+++ b/test/tempfault.test
@@ -0,0 +1,134 @@
+# 2016 April 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for fault-injection when SQLite is used with
+# a temp file database.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix tempfault
+
+# sqlite3_memdebug_vfs_oom_test 0
+
+do_faultsim_test 1 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    CREATE TABLE t1(a, b);
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+  }
+} -body {
+  execsql { INSERT INTO t1 VALUES(5, 6) }
+} -test {
+  faultsim_test_result {0 {}}
+  set rc [catch { execsql { SELECT * FROM t1 } } msg]
+  if {$rc==0 && $msg != "1 2 3 4 5 6" && $msg != "1 2 3 4"} {
+    error "data mismatch 1: $msg"
+  }
+  if {$testrc==0 && $msg != "1 2 3 4 5 6"} {
+    error "data mismatch 2: $msg"
+  }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 2 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(b, a);
+    WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100)
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  }
+} -body {
+  execsql { UPDATE t1 SET a = randomblob(99) }
+} -test {
+  faultsim_test_result {0 {}}
+  faultsim_integrity_check db
+}
+
+catch { db close }
+do_faultsim_test 2.1 -faults * -prep {
+  if {[info commands db]==""} {
+    sqlite3 db ""
+    execsql {
+      PRAGMA page_size = 1024;
+      PRAGMA cache_size = 10;
+      CREATE TABLE t1(a, b);
+      CREATE INDEX i1 ON t1(b, a);
+      WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100)
+          INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+    }
+  } 
+} -body {
+  execsql { UPDATE t1 SET a = randomblob(99) }
+} -test {
+  faultsim_test_result {0 {}}
+  faultsim_integrity_check db
+}
+
+do_faultsim_test 3 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(b, a);
+    WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50)
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  }
+} -body {
+  execsql { 
+    BEGIN;
+      UPDATE t1 SET a = randomblob(99);
+      SAVEPOINT abc;
+        UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0;
+      ROLLBACK TO abc;
+        UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0;
+      ROLLBACK TO abc;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+  faultsim_integrity_check db
+}
+
+do_faultsim_test 4 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(b, a);
+    WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50)
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  }
+} -body {
+  execsql { 
+    BEGIN;
+      UPDATE t1 SET a = randomblob(99);
+      SAVEPOINT abc;
+        UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0;
+      ROLLBACK TO abc;
+        UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0;
+      ROLLBACK TO abc;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+sqlite3_memdebug_vfs_oom_test 1
+finish_test
diff --git a/test/temptable2.test b/test/temptable2.test
new file mode 100644
index 00000000..ffa69b61
--- /dev/null
+++ b/test/temptable2.test
@@ -0,0 +1,357 @@
+# 2016 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix temptable2
+
+do_execsql_test 1.1 {
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+}
+
+do_execsql_test 1.2 {
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100000 )
+  INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM X;
+} {}
+
+do_execsql_test 1.3 {
+  PRAGMA temp.integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.1 {
+  CREATE TEMP TABLE t2(a, b);
+  INSERT INTO t2 VALUES(1, 2);
+} {}
+
+do_execsql_test 2.2 {
+  BEGIN;
+    INSERT INTO t2 VALUES(3, 4);
+    SELECT * FROM t2;
+} {1 2 3 4}
+
+do_execsql_test 2.3 {
+  ROLLBACK;
+  SELECT * FROM t2;
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.1.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 )
+  INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+
+  SELECT count(*) FROM t1;
+} {1000}
+do_execsql_test 3.1.2 {
+  BEGIN;
+    UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0;
+  ROLLBACK;
+}
+do_execsql_test 3.1.3 {
+  SELECT count(*) FROM t1;
+} {1000}
+do_execsql_test 3.1.4 { PRAGMA temp.integrity_check } {ok}
+
+do_execsql_test 3.2.1 {
+  BEGIN;
+    UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0;
+    SAVEPOINT abc;
+      UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==1;
+    ROLLBACK TO abc;
+    UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==2;
+  COMMIT;
+}
+do_execsql_test 3.2.2 { PRAGMA temp.integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.1.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<10 )
+  INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+
+  SELECT count(*) FROM t1;
+  PRAGMA temp.page_count;
+} {10 9}
+
+do_execsql_test 4.1.2 {
+  BEGIN;
+    UPDATE t1 SET b=randomblob(100);
+  ROLLBACK;
+}
+
+do_execsql_test 4.1.3 {
+  CREATE TEMP TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x;
+
+  SELECT count(*) FROM t2;
+  SELECT count(*) FROM t1;
+} {500 10}
+
+do_test 4.1.4 {
+  set n [db one { PRAGMA temp.page_count }]
+  expr ($n >280 && $n < 300) 
+} 1
+
+do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.1.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x;
+
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+}
+
+# Test that the temp database is now much bigger than the configured
+# cache size (10 pages).
+do_test 5.1.2 {
+  set n [db one { PRAGMA temp.page_count }]
+  expr ($n > 270 && $n < 290)
+} {1}
+
+do_execsql_test 5.1.3 {
+  BEGIN;
+    UPDATE t1 SET a=2;
+    UPDATE t2 SET a=randomblob(100);
+    SELECT count(*) FROM t1;
+  ROLLBACK;
+} {1}
+
+do_execsql_test 5.1.4 {
+  UPDATE t2 SET a=randomblob(100);
+
+  SELECT * FROM t1;
+} {1 2}
+
+do_execsql_test 5.1.5 { PRAGMA temp.integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+# Test this:
+#
+#   1. Page is DIRTY at the start of a transaction.
+#   2. Page is written out as part of the transaction.
+#   3. Page is then read back in.
+#   4. Transaction is rolled back. Is the page now clean or dirty?
+#
+# This actually does work. Step 4 marks the page as clean. But it also
+# writes to the database file itself. So marking it clean is correct - 
+# the page does match the contents of the db file.
+#
+reset_db
+
+do_execsql_test 6.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t1(x);
+  INSERT INTO t1 VALUES('one');
+
+  CREATE TEMP TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x;
+}
+
+do_execsql_test 6.2 {
+  UPDATE t1 SET x='two';             -- step 1
+  BEGIN;
+    UPDATE t2 SET a=randomblob(100); -- step 2
+    SELECT * FROM t1;                -- step 3
+  ROLLBACK;                          -- step 4
+
+  SELECT count(*) FROM t2;
+  SELECT * FROM t1;
+} {two 500 two}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3 db ""
+do_execsql_test 7.1 {
+  PRAGMA auto_vacuum=INCREMENTAL;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(zeroblob(900));
+  INSERT INTO t1 VALUES(zeroblob(900));
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1 SELECT x FROM t1;
+  BEGIN;
+  DELETE FROM t1 WHERE rowid%2;
+  PRAGMA incremental_vacuum(4);
+  ROLLBACK;
+  PRAGMA integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+# Try changing the page size using a backup operation when pages are
+# stored in main-memory only.
+#
+reset_db
+do_execsql_test 8.1 {
+  PRAGMA auto_vacuum = OFF;
+  CREATE TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x ORDER BY 1, 2;
+  PRAGMA page_count;
+} {13}
+
+do_test 8.2 {
+  sqlite3 tmp ""
+  execsql {
+    PRAGMA auto_vacuum = OFF;
+    PRAGMA page_size = 8192;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(a, b);
+    WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100 )
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x ORDER BY 1, 2;
+    PRAGMA page_count;
+  } tmp
+} {10}
+
+do_test 8.3 {
+  sqlite3_backup B tmp main db main
+  B step 5
+  B finish
+} {SQLITE_READONLY}
+
+do_test 8.4 {
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+    PRAGMA page_size;
+  } tmp
+} {100 ok 8192}
+
+do_test 8.5 { 
+  tmp eval { UPDATE t1 SET a=randomblob(100) }
+} {}
+
+do_test 8.6 {
+  sqlite3_backup B tmp main db main
+  B step 1000
+  B finish
+} {SQLITE_READONLY}
+
+tmp close
+
+#-------------------------------------------------------------------------
+# Try inserts and deletes with a large db in auto-vacuum mode. Check
+#
+foreach {tn mode} {
+  1 delete
+  2 wal
+} {
+  reset_db
+  sqlite3 db ""
+  do_execsql_test 9.$tn.1.1 {
+    PRAGMA cache_size = 15;
+    PRAGMA auto_vacuum = 1;
+  }
+  execsql "PRAGMA journal_mode = $mode"
+
+  do_execsql_test 9.$tn.1.2 {
+    CREATE TABLE tx(a, b);
+    CREATE INDEX i1 ON tx(a);
+    CREATE INDEX i2 ON tx(b);
+    WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 )
+      INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x;
+  }
+
+  for {set i 2} {$i<20} {incr i} {
+    do_execsql_test 9.$tn.$i.1 { DELETE FROM tx WHERE (random()%3)==0 }
+
+    do_execsql_test 9.$tn.$i.2 { PRAGMA integrity_check } ok
+
+      do_execsql_test 9.$tn.$i.3 { 
+        WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 )
+          INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x;
+      }
+
+    do_execsql_test 9.$tn.$i.4 { PRAGMA integrity_check } ok
+
+    do_execsql_test 9.$tn.$i.5 { 
+      BEGIN;
+      DELETE FROM tx WHERE (random()%3)==0;
+      WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+        INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x;
+      COMMIT;
+    }
+
+    do_execsql_test 9.$tn.$i.6 { PRAGMA integrity_check } ok
+  }
+}
+
+#-------------------------------------------------------------------------
+# When using mmap mode with a temp file, SQLite must search the cache 
+# before using a mapped page even when there is no write transaction
+# open. For a temp file, the on-disk version may not be up to date.
+#
+sqlite3 db ""
+do_execsql_test 10.0 {
+  PRAGMA cache_size = 50;
+  PRAGMA page_size = 1024;
+  CREATE TABLE t1(a, b, PRIMARY KEY(a)) WITHOUT ROWID;
+  CREATE INDEX i1 ON t1(a);
+  CREATE TABLE t2(x, y);
+  INSERT INTO t2 VALUES(1, 2);
+}
+
+do_execsql_test 10.1 {
+  BEGIN;
+    WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+      INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  COMMIT;
+  INSERT INTO t2 VALUES(3, 4);
+}
+
+ifcapable mmap {
+  if {[permutation]!="journaltest"} {
+    # The journaltest permutation does not support mmap, so this part of
+    # the test is omitted.
+    do_execsql_test 10.2 { PRAGMA mmap_size = 512000 } 512000
+  }
+}
+
+do_execsql_test 10.3 { SELECT * FROM t2 } {1 2 3 4}
+do_execsql_test 10.4 { PRAGMA integrity_check } ok
+
+finish_test
diff --git a/test/temptable3.test b/test/temptable3.test
new file mode 100644
index 00000000..16f51d78
--- /dev/null
+++ b/test/temptable3.test
@@ -0,0 +1,41 @@
+# 2016-05-10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix temptable3
+
+db close
+sqlite3 db {}
+do_execsql_test 1.1 {
+  PRAGMA cache_size = 1;
+  PRAGMA page_size = 1024;
+  PRAGMA auto_vacuum = 2;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES( randomblob(800) );
+  INSERT INTO t1 VALUES( randomblob(800) );
+  CREATE TABLE t2(x);
+  PRAGMA integrity_check;
+} {ok}
+
+db close
+sqlite3 db {}
+do_execsql_test 1.2 {
+  PRAGMA cache_size = 1;
+  PRAGMA auto_vacuum = 2;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x UNIQUE);
+  INSERT INTO t2 VALUES(1), (2), (3);
+  DROP TABLE t1;
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test
diff --git a/test/tester.tcl b/test/tester.tcl
index dc532c97..814788ba 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -25,6 +25,7 @@
 #      copy_file              FROM TO
 #      delete_file            FILENAME
 #      drop_all_tables        ?DB?
+#      drop_all_indexes       ?DB?
 #      forcecopy              FROM TO
 #      forcedelete            FILENAME
 #
@@ -373,6 +374,28 @@ proc do_not_use_codec {} {
   set ::do_not_use_codec 1
   reset_db
 }
+unset -nocomplain do_not_use_codec
+
+# Return true if the "reserved_bytes" integer on database files is non-zero.
+#
+proc nonzero_reserved_bytes {} {
+  return [sqlite3 -has-codec]
+}
+
+# Print a HELP message and exit
+#
+proc print_help_and_quit {} {
+  puts {Options:
+  --pause                  Wait for user input before continuing
+  --soft-heap-limit=N      Set the soft-heap-limit to N
+  --maxerror=N             Quit after N errors
+  --verbose=(0|1)          Control the amount of output.  Default '1'
+  --output=FILE            set --verbose=2 and output to FILE.  Implies -q
+  -q                       Shorthand for --verbose=0
+  --help                   This message
+}
+  exit 1
+}
 
 # The following block only runs the first time this file is sourced. It
 # does not run in slave interpreters (since the ::cmdlinearg array is
@@ -396,6 +419,8 @@ if {[info exists cmdlinearg]==0} {
   #   --match=$pattern
   #   --verbose=$val
   #   --output=$filename
+  #   -q                                      Reduce output
+  #   --testdir=$dir                          Run tests in subdirectory $dir
   #   --help
   #
   set cmdlinearg(soft-heap-limit)    0
@@ -410,6 +435,7 @@ if {[info exists cmdlinearg]==0} {
   set cmdlinearg(match)             ""
   set cmdlinearg(verbose)           ""
   set cmdlinearg(output)            ""
+  set cmdlinearg(testdir)           "testdir"
 
   set leftover [list]
   foreach a $argv {
@@ -439,6 +465,7 @@ if {[info exists cmdlinearg]==0} {
       }
       {^-+binarylog=.+$} {
         foreach {dummy cmdlinearg(binarylog)} [split $a =] break
+        set cmdlinearg(binarylog) [file normalize $cmdlinearg(binarylog)]
       }
       {^-+soak=.+$} {
         foreach {dummy cmdlinearg(soak)} [split $a =] break
@@ -471,6 +498,7 @@ if {[info exists cmdlinearg]==0} {
 
       {^-+output=.+$} {
         foreach {dummy cmdlinearg(output)} [split $a =] break
+        set cmdlinearg(output) [file normalize $cmdlinearg(output)]
         if {$cmdlinearg(verbose)==""} {
           set cmdlinearg(verbose) 2
         }
@@ -483,12 +511,34 @@ if {[info exists cmdlinearg]==0} {
           error "option --verbose= must be set to a boolean or to \"file\""
         }
       }
+      {^-+testdir=.*$} {
+        foreach {dummy cmdlinearg(testdir)} [split $a =] break
+      }
+      {.*help.*} {
+         print_help_and_quit
+      }
+      {^-q$} {
+        set cmdlinearg(output) test-out.txt
+        set cmdlinearg(verbose) 2
+      }
 
       default {
-        lappend leftover $a
+        if {[file tail $a]==$a} {
+          lappend leftover $a
+        } else {
+          lappend leftover [file normalize $a]
+        }
       }
     }
   }
+  set testdir [file normalize $testdir]
+  set cmdlinearg(TESTFIXTURE_HOME) [pwd]
+  set cmdlinearg(INFO_SCRIPT) [file normalize [info script]]
+  set argv0 [file normalize $argv0]
+  if {$cmdlinearg(testdir)!=""} {
+    file mkdir $cmdlinearg(testdir)
+    cd $cmdlinearg(testdir)
+  }
   set argv $leftover
 
   # Install the malloc layer used to inject OOM errors. And the 'automatic'
@@ -672,6 +722,17 @@ proc puts {args} { uplevel puts_override $args }
 
 # Invoke the do_test procedure to run a single test
 #
+# The $expected parameter is the expected result.  The result is the return
+# value from the last TCL command in $cmd.
+#
+# Normally, $expected must match exactly.  But if $expected is of the form
+# "/regexp/" then regular expression matching is used.  If $expected is
+# "~/regexp/" then the regular expression must NOT match.  If $expected is
+# of the form "#/value-list/" then each term in value-list must be numeric
+# and must approximately match the corresponding numeric term in $result.
+# Values must match within 10%.  Or if the $expected term is A..B then the
+# $result term must be in between A and B.
+#
 proc do_test {name cmd expected} {
   global argv cmdlinearg
 
@@ -705,7 +766,7 @@ proc do_test {name cmd expected} {
       output2 "\nError: $result"
       fail_test $name
     } else {
-      if {[regexp {^~?/.*/$} $expected]} {
+      if {[regexp {^[~#]?/.*/$} $expected]} {
         # "expected" is of the form "/PATTERN/" then the result if correct if
         # regular expression PATTERN matches the result.  "~/PATTERN/" means
         # the regular expression must not match.
@@ -719,6 +780,21 @@ proc do_test {name cmd expected} {
             set ok [regexp $re $result]
           }
           set ok [expr {!$ok}]
+        } elseif {[string index $expected 0]=="#"} {
+          # Numeric range value comparison.  Each term of the $result is matched
+          # against one term of $expect.  Both $result and $expected terms must be
+          # numeric.  The values must match within 10%.  Or if $expected is of the
+          # form A..B then the $result term must be between A and B.
+          set e2 [string range $expected 2 end-1]
+          foreach i $result j $e2 {
+            if {[regexp {^(-?\d+)\.\.(-?\d)$} $j all A B]} {
+              set ok [expr {$i+0>=$A && $i+0<=$B}]
+            } else {
+              set ok [expr {$i+0>=0.9*$j && $i+0<=1.1*$j}]
+            }
+            if {!$ok} break
+          }
+          if {$ok && [llength $result]!=[llength $e2]} {set ok 0}
         } else {
           set re [string range $expected 1 end-1]
           if {[string index $re 0]=="*"} {
@@ -837,6 +913,12 @@ proc fix_testname {varname} {
   }
 }
 
+proc normalize_list {L} {
+  set L2 [list]
+  foreach l $L {lappend L2 $l}
+  set L2
+}
+
 proc do_execsql_test {testname sql {result {}}} {
   fix_testname testname
   uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]]
@@ -1208,9 +1290,9 @@ proc explain_i {sql {db db}} {
     set D ""
   }
   foreach opcode {
-      Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind
+      Seek SeekGE SeekGT SeekLE SeekLT NotFound Last Rewind
       NoConflict Next Prev VNext VPrev VFilter
-      SorterSort SorterNext
+      SorterSort SorterNext NextIfOpen
   } {
     set color($opcode) $B
   }
@@ -1231,9 +1313,15 @@ proc explain_i {sql {db db}} {
       set bSeenGoto 1
     }
 
+    if {$opcode=="Once"} {
+      for {set i $addr} {$i<$p2} {incr i} {
+        set star($i) $addr
+      }
+    }
+
     if {$opcode=="Next"  || $opcode=="Prev" 
      || $opcode=="VNext" || $opcode=="VPrev"
-     || $opcode=="SorterNext"
+     || $opcode=="SorterNext" || $opcode=="NextIfOpen"
     } {
       for {set i $p2} {$i<$addr} {incr i} {
         incr x($i) 2
@@ -1257,6 +1345,12 @@ proc explain_i {sql {db db}} {
     }
     set I [string repeat " " $x($addr)]
 
+    if {[info exists star($addr)]} {
+      set ii [expr $x($star($addr))]
+      append I "  "
+      set I [string replace $I $ii $ii *]
+    }
+
     set col ""
     catch { set col $color($opcode) }
 
@@ -1858,6 +1952,16 @@ proc drop_all_tables {{db db}} {
   }
 }
 
+# Drop all auxiliary indexes from the main database opened by handle [db].
+#
+proc drop_all_indexes {{db db}} {
+  set L [$db eval {
+    SELECT name FROM sqlite_master WHERE type='index' AND sql LIKE 'create%'
+  }]
+  foreach idx $L { $db eval "DROP INDEX $idx" }
+}
+
+
 #-------------------------------------------------------------------------
 # If a test script is executed with global variable $::G(perm:name) set to
 # "wal", then the tests are run in WAL mode. Otherwise, they should be run
@@ -1894,6 +1998,12 @@ proc wal_check_journal_mode {testname {db db}} {
   }
 }
 
+proc wal_is_capable {} {
+  ifcapable !wal { return 0 }
+  if {[permutation]=="journaltest"} { return 0 }
+  return 1
+}
+
 proc permutation {} {
   set perm ""
   catch {set perm $::G(perm:name)}
@@ -2092,6 +2202,41 @@ proc test_restore_config_pagecache {} {
   sqlite3 db test.db
 }
 
+proc test_find_binary {nm} {
+  if {$::tcl_platform(platform)=="windows"} {
+    set ret "$nm.exe"
+  } else {
+    set ret $nm
+  }
+  set ret [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]]
+  if {![file executable $ret]} {
+    finish_test
+    return ""
+  }
+  return $ret
+}
+
+# Find the name of the 'shell' executable (e.g. "sqlite3.exe") to use for
+# the tests in shell[1-5].test. If no such executable can be found, invoke
+# [finish_test ; return] in the callers context.
+#
+proc test_find_cli {} {
+  set prog [test_find_binary sqlite3]
+  if {$prog==""} { return -code return }
+  return $prog
+}
+
+# Find the name of the 'sqldiff' executable (e.g. "sqlite3.exe") to use for
+# the tests in sqldiff tests. If no such executable can be found, invoke
+# [finish_test ; return] in the callers context.
+#
+proc test_find_sqldiff {} {
+  set prog [test_find_binary sqldiff]
+  if {$prog==""} { return -code return }
+  return $prog
+}
+
+
 # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set
 # to non-zero, then set the global variable $AUTOVACUUM to 1.
 set AUTOVACUUM $sqlite_options(default_autovacuum)
diff --git a/test/time-wordcount.sh b/test/time-wordcount.sh
new file mode 100644
index 00000000..df9edc6f
--- /dev/null
+++ b/test/time-wordcount.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# This script runs the wordcount program in different ways and generates
+# an output useful for performance comparisons.
+#
+
+# Select the source text to be analyzed.
+#
+if test "x$1" = "x";
+then echo "Usage: $0 FILENAME [ARGS...]"; exit 1;
+fi
+
+# Do test runs
+#
+rm -f wcdb1.db
+./wordcount --tag A: --timer --summary wcdb1.db $* --insert
+rm -f wcdb2.db
+./wordcount --tag B: --timer --summary wcdb2.db $* --insert --without-rowid
+rm -f wcdb1.db
+./wordcount --tag C: --timer --summary wcdb1.db $* --replace
+rm -f wcdb2.db
+./wordcount --tag D: --timer --summary wcdb2.db $* --replace --without-rowid
+rm -f wcdb1.db
+./wordcount --tag E: --timer --summary wcdb1.db $* --select
+rm -f wcdb2.db
+./wordcount --tag F: --timer --summary wcdb2.db $* --select --without-rowid
+./wordcount --tag G: --timer --summary wcdb1.db $* --query
+./wordcount --tag H: --timer --summary wcdb1.db $* --query --without-rowid
+./wordcount --tag I: --timer --summary wcdb1.db $* --delete
+./wordcount --tag J: --timer --summary wcdb2.db $* --delete --without-rowid
+
+# Clean up temporary files created.
+#
+rm -f wcdb1.db wcdb2.db
diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test
index 3fef187e..0d12a6ec 100644
--- a/test/tkt-2d1a5c67d.test
+++ b/test/tkt-2d1a5c67d.test
@@ -19,7 +19,8 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix tkt-2d1a5c67d
 
-ifcapable {!wal || !vtab} {finish_test; return}
+ifcapable {!vtab} {finish_test; return}
+if {[wal_is_capable]==0} {finish_test; return}
 
 for {set ii 1} {$ii<=10} {incr ii} {
   do_test tkt-2d1a5c67d.1.$ii {
diff --git a/test/tkt-313723c356.test b/test/tkt-313723c356.test
index 8c08c349..5325fd22 100644
--- a/test/tkt-313723c356.test
+++ b/test/tkt-313723c356.test
@@ -18,7 +18,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 
-ifcapable !wal { finish_test ; return }
+if {![wal_is_capable]} { finish_test ; return }
 
 do_execsql_test tkt-313723c356.1 {
   PRAGMA page_size = 1024;
diff --git a/test/tkt-5d863f876e.test b/test/tkt-5d863f876e.test
index 86024e30..9a2fa3f4 100644
--- a/test/tkt-5d863f876e.test
+++ b/test/tkt-5d863f876e.test
@@ -18,7 +18,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 set ::testprefix tkt-5d863f876e
-ifcapable !wal {finish_test ; return }
+if {![wal_is_capable]} {finish_test ; return }
 
 do_multiclient_test tn {
   do_test $tn.1 {
diff --git a/test/tkt-9d68c883.test b/test/tkt-9d68c883.test
index 18dc6ccd..ba91b39e 100644
--- a/test/tkt-9d68c883.test
+++ b/test/tkt-9d68c883.test
@@ -50,4 +50,6 @@ for {set i 0} {$i < 100} {incr i} {
   } {ok}
 }
 
+catch { db close } 
+unregister_devsim
 finish_test
diff --git a/test/tkt4018.test b/test/tkt4018.test
index 2bc41d47..77582a5c 100644
--- a/test/tkt4018.test
+++ b/test/tkt4018.test
@@ -16,6 +16,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+do_not_use_codec
 
 proc testsql {sql} {
   set fd [open tf_main.tcl w]
diff --git a/test/trace3.test b/test/trace3.test
new file mode 100644
index 00000000..271009a3
--- /dev/null
+++ b/test/trace3.test
@@ -0,0 +1,233 @@
+# 2016 July 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The focus of
+# this test file is the "sqlite3_trace_v2()" and "sqlite3_expanded_sql()"
+# APIs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+ifcapable !trace { finish_test ; return }
+set ::testprefix trace3
+
+proc trace_v2_error { args } {
+  lappend ::stmtlist(error) [string trim $args]
+  error "trace error"; # this will be ignored.
+}
+proc trace_v2_record { args } {
+  lappend ::stmtlist(record) [string trim $args]
+}
+proc trace_v2_nop { args } {}; # do nothing.
+
+do_test trace3-1.0 {
+  execsql {
+    CREATE TABLE t1(a,b);
+    INSERT INTO t1 VALUES(1,NULL);
+    INSERT INTO t1 VALUES(2,-1);
+    INSERT INTO t1 VALUES(3,0);
+    INSERT INTO t1 VALUES(4,1);
+    INSERT INTO t1 VALUES(5,-2147483648);
+    INSERT INTO t1 VALUES(6,2147483647);
+    INSERT INTO t1 VALUES(7,-9223372036854775808);
+    INSERT INTO t1 VALUES(8,9223372036854775807);
+    INSERT INTO t1 VALUES(9,-1.0);
+    INSERT INTO t1 VALUES(10,0.0);
+    INSERT INTO t1 VALUES(11,1.0);
+    INSERT INTO t1 VALUES(12,'');
+    INSERT INTO t1 VALUES(13,'1');
+    INSERT INTO t1 VALUES(14,'one');
+    INSERT INTO t1 VALUES(15,x'abcd0123');
+    INSERT INTO t1 VALUES(16,x'4567cdef');
+  }
+} {}
+
+do_test trace3-1.1 {
+  set rc [catch {db trace_v2 1 2 3} msg]
+  lappend rc $msg
+} {1 {wrong # args: should be "db trace_v2 ?CALLBACK? ?MASK?"}}
+do_test trace3-1.2 {
+  set rc [catch {db trace_v2 1 bad} msg]
+  lappend rc $msg
+} {1 {bad trace type "bad": must be statement, profile, row, or close}}
+
+do_test trace3-2.1 {
+  db trace_v2 trace_v2_nop
+  db trace_v2
+} {trace_v2_nop}
+
+do_test trace3-3.1 {
+  unset -nocomplain ::stmtlist
+  db trace_v2 trace_v2_nop
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  array get ::stmtlist
+} {}
+do_test trace3-3.2 {
+  set ::stmtlist(error) {}
+  db trace_v2 trace_v2_error
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(error)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+do_test trace3-3.3 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+do_test trace3-3.4 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record statement
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+do_test trace3-3.5 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 1
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+
+do_test trace3-4.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record profile
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ -?\d+\}$/}
+do_test trace3-4.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 2
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ -?\d+\}$/}
+
+do_test trace3-5.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record row
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^[string trim [string repeat {-?\d+ } 16]]\$/"
+do_test trace3-5.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 4
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^[string trim [string repeat {-?\d+ } 16]]\$/"
+
+do_test trace3-6.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record {profile row}
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^[string trim [string repeat {-?\d+ } 16]] \\\{-?\\d+ -?\\d+\\\}\$/"
+do_test trace3-6.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record {statement profile row}
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^\\\{-?\\d+ \\\{SELECT a, b FROM t1 ORDER BY a;\\\}\\\} [string trim \
+[string repeat {-?\d+ } 16]] \\\{-?\\d+ -?\\d+\\\}\$/"
+
+do_test trace3-7.1 {
+  set DB [sqlite3_connection_pointer db]
+
+  set STMT [sqlite3_prepare_v2 $DB \
+      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 TAIL]
+} {/^[0-9A-Fa-f]+$/}
+
+do_test trace3-8.1 {
+  list [sqlite3_bind_null $STMT 1] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = NULL ORDER BY a;}}
+do_test trace3-8.2 {
+  list [sqlite3_bind_int $STMT 1 123] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 123 ORDER BY a;}}
+do_test trace3-8.3 {
+  list [sqlite3_bind_int64 $STMT 1 123] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 123 ORDER BY a;}}
+do_test trace3-8.4 {
+  list [sqlite3_bind_text $STMT 1 "some string" 11] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 'some string' ORDER BY a;}}
+do_test trace3-8.5 {
+  list [sqlite3_bind_text $STMT 1 "some 'bad' string" 17] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 'some ''bad'' string' ORDER BY a;}}
+do_test trace3-8.6 {
+  list [sqlite3_bind_double $STMT 1 123] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 123.0 ORDER BY a;}}
+do_test trace3-8.7 {
+  list [sqlite3_bind_text16 $STMT 1 \
+      [encoding convertto unicode hi\000yall\000] 16] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 'hi' ORDER BY a;}}
+do_test trace3-8.8 {
+  list [sqlite3_bind_blob $STMT 1 "\x12\x34\x56" 3] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = x'123456' ORDER BY a;}}
+do_test trace3-8.9 {
+  list [sqlite3_bind_blob $STMT 1 "\xAB\xCD\xEF" 3] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = x'abcdef' ORDER BY a;}}
+
+do_test trace3-9.1 {
+  sqlite3_finalize $STMT
+} {SQLITE_OK}
+
+do_test trace3-10.1 {
+  db trace_v2 ""
+  db trace_v2
+} {}
+do_test trace3-10.2 {
+  unset -nocomplain ::stmtlist
+  db trace_v2 "" {statement profile row}
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  array get ::stmtlist
+} {}
+
+do_test trace3-11.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record close
+  db close
+  set ::stmtlist(record)
+} {/^-?\d+$/}
+
+reset_db
+
+do_test trace3-11.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 8
+  db close
+  set ::stmtlist(record)
+} {/^-?\d+$/}
+
+finish_test
diff --git a/test/types2.test b/test/types2.test
index 4a70aa5f..d6d84e65 100644
--- a/test/types2.test
+++ b/test/types2.test
@@ -333,7 +333,7 @@ ifcapable subquery {
   test_boolset types2-8.8 {o IN (SELECT t FROM t4)} {7}
   test_boolset types2-8.9 {i IN (SELECT o FROM t4)} {9 10 11 12}
   test_boolset types2-8.6 {n IN (SELECT o FROM t4)} {9 10 11 12}
-  test_boolset types2-8.7 {t IN (SELECT o FROM t4)} {9 11}
+  test_boolset types2-8.7 {t IN (SELECT o FROM t4)} {}
   test_boolset types2-8.8 {o IN (SELECT o FROM t4)} {9 10}
 }
 
diff --git a/test/unixexcl.test b/test/unixexcl.test
index d6762178..8e9c4644 100644
--- a/test/unixexcl.test
+++ b/test/unixexcl.test
@@ -87,6 +87,7 @@ do_multiclient_test tn {
     sql1 {
       PRAGMA auto_vacuum = 0;
       PRAGMA journal_mode = WAL;
+      PRAGMA synchronous = FULL;
       CREATE TABLE t1(a, b);
       INSERT INTO t1 VALUES(1, 2);
     }
diff --git a/test/vacuum5.test b/test/vacuum5.test
new file mode 100644
index 00000000..8e76a939
--- /dev/null
+++ b/test/vacuum5.test
@@ -0,0 +1,153 @@
+# 2016-08-19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file implements a test for VACUUM on attached databases.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vacuum5
+
+# If the VACUUM statement is disabled in the current build, skip all
+# the tests in this file.
+#
+ifcapable !vacuum {
+  finish_test
+  return
+}
+
+forcedelete test2.db test3.db
+do_execsql_test vacuum5-1.1 {
+  PRAGMA auto_vacuum = 0;
+  CREATE TABLE main.t1(a,b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t1(a,b) SELECT x, randomblob(1000) FROM c;
+  CREATE TEMP TABLE ttemp(x,y);
+  INSERT INTO ttemp SELECT * FROM t1;
+  ATTACH 'test2.db' AS x2;
+  ATTACH 'test3.db' AS x3;
+  CREATE TABLE x2.t2(c,d);
+  INSERT INTO t2 SELECT * FROM t1;
+  CREATE TABLE x3.t3(e,f);
+  INSERT INTO t3 SELECT * FROM t1;
+  DELETE FROM t1 WHERE (rowid%3)!=0;
+  DELETE FROM t2 WHERE (rowid%4)!=0;
+  DELETE FROM t3 WHERE (rowid%5)!=0;
+  PRAGMA main.integrity_check;
+  PRAGMA x2.integrity_check;
+  PRAGMA x3.integrity_check;
+} {ok ok ok}
+set size1 [file size test.db]
+set size2 [file size test2.db]
+set size3 [file size test3.db]
+
+do_execsql_test vacuum5-1.2.1 {
+  VACUUM main;
+} {}
+do_test vacuum5-1.2.2 {
+  expr {[file size test.db]<$size1}
+} {1}
+do_test vacuum5-1.2.3 {
+  file size test2.db
+} $size2
+do_test vacuum5-1.2.4 {
+  file size test3.db
+} $size3
+set size1 [file size test.db]
+do_execsql_test vacuum-1.2.5 {
+  DELETE FROM t1;
+  PRAGMA main.integrity_check;
+} {ok}
+
+do_execsql_test vacuum5-1.3.1 {
+  VACUUM x2;
+} {}
+do_test vacuum5-1.3.2 {
+  file size test.db
+} $size1
+do_test vacuum5-1.3.3 {
+  expr {[file size test2.db]<$size2}
+} 1
+do_test vacuum5-1.3.4 {
+  file size test3.db
+} $size3
+set size2 [file size test2.db]
+do_execsql_test vacuum-1.3.5 {
+  DELETE FROM t2;
+  PRAGMA x2.integrity_check;
+} {ok}
+
+do_execsql_test vacuum5-1.4.1 {
+  VACUUM x3;
+} {}
+do_test vacuum5-1.3.2 {
+  file size test.db
+} $size1
+do_test vacuum5-1.3.3 {
+  file size test2.db
+} $size2
+do_test vacuum5-1.3.4 {
+  expr {[file size test3.db]<$size3}
+} 1
+
+# VACUUM is a no-op on the TEMP table
+#
+set sizeTemp [db one {PRAGMA temp.page_count}]
+do_execsql_test vacuum5-1.4.1 {
+  VACUUM temp;
+} {}
+do_execsql_test vacuum5-1.4.2 {
+  PRAGMA temp.page_count;
+} $sizeTemp
+
+do_catchsql_test vacuum5-2.0 {
+  VACUUM olaf;
+} {1 {unknown database olaf}}
+
+#-------------------------------------------------------------------------
+# Test that a temp file is opened as part of VACUUM.
+#
+if {$::TEMP_STORE<3 && [permutation]!="inmemory_journal"} {
+  db close
+  testvfs tvfs 
+  tvfs filter xOpen
+  tvfs script open_cb
+  forcedelete test.db
+
+  set ::openfiles [list]
+  proc open_cb {method args} {
+    lappend ::openfiles [file tail [lindex $args 0]]
+  }
+  sqlite3 db test.db -vfs tvfs
+
+  do_execsql_test 3.0 {
+    PRAGMA temp_store = file;
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 50;
+    CREATE TABLE t1(i INTEGER PRIMARY KEY, j UNIQUE);
+    WITH s(i) AS (
+      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t1 SELECT NULL, randomblob(100) FROM s;
+  }
+
+  do_execsql_test 3.1 { VACUUM }
+
+  db close
+  tvfs delete
+  do_test 3.2 {
+    lrange $::openfiles 0 4
+  } {test.db test.db-journal test.db-journal {} test.db-journal}
+} 
+
+
+
+finish_test
diff --git a/test/vacuummem.test b/test/vacuummem.test
new file mode 100644
index 00000000..0f77c1b4
--- /dev/null
+++ b/test/vacuummem.test
@@ -0,0 +1,73 @@
+# 2005 February 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing that the VACUUM statement correctly
+# frees any memory used for a temporary cache.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vacuummem
+
+if {[permutation]=="memsubsys1"} {
+  finish_test
+  return
+}
+
+# If ENABLE_MEMORY_MANAGEMENT is defined, when VACUUM is run the temp db
+# is able to borrow space from the main db (and it does, because the
+# temp db is configure with a very small cache). When the VACUUM is
+# finished and the temp db closed, all the page-cache memory currently 
+# assigned to the temp db is freed. If ENABLE_MEMORY_MANAGEMENT is defined
+# this causes the total memory usage to drop much more than expected,
+# causing tests in this file to fail.
+#
+ifcapable memorymanage {
+  finish_test
+  return
+}
+
+
+proc memory_used {} { 
+  set stat [sqlite3_status SQLITE_STATUS_MEMORY_USED 1]  
+  lindex $stat 1
+}
+
+do_execsql_test 1.0 {
+  PRAGMA cache_size = -2000;
+  CREATE TABLE t1(a, b, c);
+
+  WITH r(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM r WHERE i<100000
+  )
+  INSERT INTO t1 SELECT randomblob(100),randomblob(100),randomblob(100) FROM r;
+
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+}
+set ans "#/[memory_used]/"
+
+do_test 1.1 { memory_used } $ans
+
+do_execsql_test 1.2 VACUUM
+
+do_test 1.3 { memory_used } $ans
+
+do_execsql_test 1.4 {
+  SELECT count(*) FROM t1 WHERE +a IS NOT NULL
+} {100000}
+
+do_test 1.5 { memory_used } $ans
+
+
+
+finish_test
diff --git a/test/view.test b/test/view.test
index 235f8605..694b3c6a 100644
--- a/test/view.test
+++ b/test/view.test
@@ -92,6 +92,26 @@ do_test view-1.8 {
   }
 } {2 3 5 6 8 9}
 
+do_execsql_test view-1.10 {
+  CREATE TABLE t9(x INTEGER);
+  CREATE VIEW v9a AS SELECT x FROM t9;
+  CREATE VIEW v9b AS SELECT * FROM t9;
+  CREATE VIEW v9c(x) AS SELECT x FROM t9;
+  CREATE VIEW v9d(x) AS SELECT * FROM t9;
+} {}
+do_execsql_test view-1.11 {
+  PRAGMA table_info(v9a);
+} {0 x INTEGER 0 {} 0}
+do_execsql_test view-1.12 {
+  PRAGMA table_info(v9b);
+} {0 x INTEGER 0 {} 0}
+do_execsql_test view-1.13 {
+  PRAGMA table_info(v9c);
+} {0 x INTEGER 0 {} 0}
+do_execsql_test view-1.14 {
+  PRAGMA table_info(v9d);
+} {0 x INTEGER 0 {} 0}
+
 do_test view-2.1 {
   execsql {
     CREATE VIEW v2 AS SELECT * FROM t1 WHERE a>5
diff --git a/test/vtab6.test b/test/vtab6.test
index f8e0935a..f4504b01 100644
--- a/test/vtab6.test
+++ b/test/vtab6.test
@@ -566,12 +566,12 @@ do_test vtab6-11.4.1 {
   catchsql {
     SELECT a, b, c FROM ab NATURAL JOIN bc;
   }
-} {1 {table ab: xBestIndex returned an invalid plan}}
+} {1 {ab.xBestIndex malfunction}}
 do_test vtab6-11.4.2 {
   catchsql {
     SELECT a, b, c FROM bc NATURAL JOIN ab;
   }
-} {1 {table bc: xBestIndex returned an invalid plan}}
+} {1 {bc.xBestIndex malfunction}}
 
 unset ::echo_module_ignore_usable
 
diff --git a/test/vtabF.test b/test/vtabF.test
index b9341ea5..7b11cb8c 100644
--- a/test/vtabF.test
+++ b/test/vtabF.test
@@ -21,7 +21,7 @@ ifcapable !vtab||!schema_pragmas { finish_test ; return }
 # Register the echo module
 register_echo_module [sqlite3_connection_pointer db]
 
-do_test vtabE-1.1 {
+do_test vtabF-1.1 {
   execsql {
     CREATE TABLE t1(a, b);
     CREATE INDEX i1 ON t1(a);
@@ -37,7 +37,7 @@ do_test vtabE-1.1 {
     SELECT b FROM t1 WHERE a IS NOT NULL;
   }
 } {110 111 112 113}
-do_test vtabE-1.2 {
+do_test vtabF-1.2 {
   execsql {SELECT b FROM tv1 WHERE a IS NOT NULL}
 } {110 111 112 113}
 
diff --git a/test/vtabH.test b/test/vtabH.test
index d16db136..f2a116f3 100644
--- a/test/vtabH.test
+++ b/test/vtabH.test
@@ -108,7 +108,12 @@ foreach ::tclvar_set_omit {0 1} {
 
 #-------------------------------------------------------------------------
 #
-if {1} {
+if {$tcl_platform(platform)=="windows"} {
+  set drive [string range [pwd] 0 1]
+  set ::env(fstreeDrive) $drive
+}
+if {$tcl_platform(platform)!="windows" || \
+    [regexp -nocase -- {^[A-Z]:} $drive]} {
   reset_db
   register_fs_module db
   do_execsql_test 3.0 {
@@ -119,7 +124,7 @@ if {1} {
   proc list_root_files {} {
     if {$::tcl_platform(platform) eq "windows"} {
       set res [list]
-      foreach name [glob -directory $::env(SystemDrive)/ -- *] {
+      foreach name [glob -directory $::env(fstreeDrive)/ -- *] {
         if {[string index [file tail $name] 0] eq "."} continue
         lappend res $name
       }
@@ -142,6 +147,25 @@ if {1} {
     }
   }
 
+  # Read the first 5 entries from the root directory.  Except, ignore
+  # files that contain the "$" character in their names as these are
+  # special files on some Windows platforms.
+  #
+  set res [list]
+  set root_files [list_root_files]
+  set num_root_files [llength $root_files]
+  set lim_root_files [expr {$num_root_files > 5 ? 5 : $num_root_files}]
+  foreach p [lrange $root_files 0 [expr {$lim_root_files - 1}]] {
+    if {$::tcl_platform(platform) eq "windows"} {
+      if {[regexp {\$} $p]} {incr lim_root_files -1} else {lappend res $p}
+    } else {
+      lappend res "/$p"
+    }
+  }
+  do_execsql_test 3.1 [subst {
+    SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $lim_root_files;
+  }] $res
+
   # Read all entries in the current directory.
   #
   proc contents {pattern} {
diff --git a/test/wal.test b/test/wal.test
index 92b65e66..bb164bb7 100644
--- a/test/wal.test
+++ b/test/wal.test
@@ -1378,6 +1378,7 @@ do_test wal-21.3 {
 #-------------------------------------------------------------------------
 # Test reading and writing of databases with different page-sizes.
 #
+incr ::do_not_use_codec
 foreach pgsz {512 1024 2048 4096 8192 16384 32768 65536} {
   do_multiclient_test tn [string map [list %PGSZ% $pgsz] {
     do_test wal-22.%PGSZ%.$tn.1 {
@@ -1398,6 +1399,7 @@ foreach pgsz {512 1024 2048 4096 8192 16384 32768 65536} {
     } {0}
   }]
 }
+incr ::do_not_use_codec -1
 
 #-------------------------------------------------------------------------
 # Test that when 1 or more pages are recovered from a WAL file, 
diff --git a/test/wal2.test b/test/wal2.test
index 9d45444d..0b15b154 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -1194,9 +1194,12 @@ foreach {tn sql reslist} {
   2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2}
   3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0}
 } {
+  ifcapable default_ckptfullfsync {
+    if {[string trim $sql]==""} continue
+  }
   faultsim_delete_and_reopen
 
-  execsql {PRAGMA auto_vacuum = 0}
+  execsql {PRAGMA auto_vacuum = 0; PRAGMA synchronous = FULL;}
   execsql $sql
   do_execsql_test wal2-14.$tn.0 { PRAGMA page_size = 4096 }   {}
   do_execsql_test wal2-14.$tn.1 { PRAGMA journal_mode = WAL } {wal}
diff --git a/test/wal3.test b/test/wal3.test
index da3d3187..56f40ab5 100644
--- a/test/wal3.test
+++ b/test/wal3.test
@@ -220,6 +220,7 @@ foreach {tn syncmode synccount} {
     sqlite3 db test.db -vfs T
   
     execsql "PRAGMA synchronous = $syncmode"
+    execsql "PRAGMA checkpoint_fullfsync = 0"
     execsql { PRAGMA journal_mode = WAL }
     execsql { CREATE TABLE filler(a,b,c); }
 
diff --git a/test/wal5.test b/test/wal5.test
index 360d9c91..50c51728 100644
--- a/test/wal5.test
+++ b/test/wal5.test
@@ -18,6 +18,7 @@ source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 source $testdir/wal_common.tcl
 ifcapable !wal {finish_test ; return }
+do_not_use_codec
 
 set testprefix wal5
 
@@ -140,13 +141,13 @@ foreach {testprefix do_wal_checkpoint} {
     do_test 1.$tn.7 {
       reopen_all
       list [db_page_count] [wal_page_count] $::nBusyHandler
-    } {7 0 0}
+    } [expr {[nonzero_reserved_bytes]?"/# # 0/":"7 0 0"}]
 
     do_test 1.$tn.8  { sql2 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5}
     do_test 1.$tn.9  {
       sql1 { INSERT INTO t1 VALUES(6, zeroblob(1200)) }
       list [db_page_count] [wal_page_count] $::nBusyHandler
-    } {7 5 0}
+    } [expr {[nonzero_reserved_bytes]?"/# # #/":"7 5 0"}]
     do_test 1.$tn.10 { sql3 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5 6}
 
     set ::busy_handler_script { 
@@ -157,7 +158,7 @@ foreach {testprefix do_wal_checkpoint} {
     do_test 1.$tn.11 {
       code1 { do_wal_checkpoint db -mode restart }
       list [db_page_count] [wal_page_count] $::nBusyHandler
-    } {10 5 8}
+    } [expr {[nonzero_reserved_bytes]?"/# # #/":"10 5 8"}]
     do_test 1.$tn.12 { set ::db_file_size } 10
   }
 
diff --git a/test/wal8.test b/test/wal8.test
index 0682fce3..3e5a0c26 100644
--- a/test/wal8.test
+++ b/test/wal8.test
@@ -27,6 +27,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set ::testprefix wal8
 ifcapable !wal {finish_test ; return }
+do_not_use_codec
 
 db close
 forcedelete test.db test.db-wal
diff --git a/test/walbak.test b/test/walbak.test
index 303a628a..0e0f9995 100644
--- a/test/walbak.test
+++ b/test/walbak.test
@@ -127,6 +127,7 @@ do_test walbak-2.1 {
   }
 } {}
 do_test walbak-2.2 {
+  forcedelete abc.db
   db backup abc.db
   sqlite3 db2 abc.db
   string compare [sig db] [sig db2]
@@ -239,6 +240,7 @@ foreach {tn setup} {
   }
 
 } {
+  if {$tn==4 && [sqlite3 -has-codec]} continue
   foreach f [glob -nocomplain test.db*] { forcedelete $f }
 
   eval $setup
diff --git a/test/walcrash4.test b/test/walcrash4.test
new file mode 100644
index 00000000..80839b39
--- /dev/null
+++ b/test/walcrash4.test
@@ -0,0 +1,85 @@
+# 2010 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+ifcapable !wal {finish_test ; return }
+set testprefix walcrash4
+do_not_use_codec
+
+#-------------------------------------------------------------------------
+# At one point, if "PRAGMA synchronous=full" is set and the platform
+# does not support POWERSAFE_OVERWRITE, and the last frame written to 
+# the wal file in a transaction is aligned with a sector boundary, the
+# xSync() call was omitted. 
+#
+# The following test verifies that this has been fixed.
+#
+do_execsql_test 1.0 {
+  PRAGMA autovacuum = 0;
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  PRAGMA main.synchronous = full;
+} {wal}
+
+faultsim_save_and_close
+
+# The error message is different on unix and windows
+#
+if {$::tcl_platform(platform)=="windows"} {
+ set msg "child killed: unknown signal"
+} else {
+ set msg "child process exited abnormally"
+}
+
+for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
+  for {set i 0} {$i < 10} {incr i} {
+    do_test 1.nExtra=$nExtra.i=$i.1 {
+      faultsim_restore_and_reopen
+    
+      set fd [open crash.tcl w]
+      puts $fd [subst -nocommands {
+        sqlite3_crash_enable 1
+        sqlite3_test_control_pending_byte $::sqlite_pending_byte
+        sqlite3 db test.db -vfs crash
+        db eval {
+          PRAGMA main.synchronous=FULL;
+          BEGIN;
+          CREATE TABLE t1(x UNIQUE);
+        }
+        for {set e 2} {[set e] < ($nExtra+2)} {incr e} {
+          db eval "CREATE TABLE t[set e] (x)"
+        }
+        db eval {
+          INSERT INTO t1 VALUES( randomblob(170000) );
+          COMMIT;
+        }
+        sqlite3_crash_now
+      }]
+      close $fd
+    
+      set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg]
+      list $r $msg
+    } "1 {$msg}"
+  
+    do_execsql_test 1.nExtra=$nExtra.i=$i.2 { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    } {1 ok}
+  } 
+}
+
+
+finish_test
diff --git a/test/walprotocol.test b/test/walprotocol.test
index d658de9d..ee8d0b72 100644
--- a/test/walprotocol.test
+++ b/test/walprotocol.test
@@ -69,7 +69,7 @@ proc lock_callback {method filename handle lock} {
   if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY }
   return SQLITE_OK
 }
-puts "# Warning: This next test case causes SQLite to call xSlee(1) 100 times."
+puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
 puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite"
 puts "# is built on unix without HAVE_USLEEP defined, it may be much longer."
 do_test 1.3 {
diff --git a/test/walro.test b/test/walro.test
index 6d920b1e..f46e44d4 100644
--- a/test/walro.test
+++ b/test/walro.test
@@ -212,7 +212,7 @@ do_multiclient_test tn {
       INSERT INTO t2 SELECT x||y, y||x FROM t2;
     }
     file size test.db-wal
-  } {147800}
+  } [expr {[nonzero_reserved_bytes]?148848:147800}]
   do_test 1.4.4.2 {
     csql1 { SELECT * FROM t1 }
   } {0 {a b c d e f g h i j k l 1 2 3 4 5 6}}
diff --git a/test/where.test b/test/where.test
index 0b5bb934..d4f04e80 100644
--- a/test/where.test
+++ b/test/where.test
@@ -573,11 +573,16 @@ do_test where-6.6 {
     SELECT * FROM t3 WHERE a>0 ORDER BY a LIMIT 3
   }
 } {1 100 4 2 99 9 3 98 16 nosort}
-do_test where-6.7 {
+do_test where-6.7.1 {
   cksort {
-    SELECT * FROM t3 WHERE b>0 ORDER BY a LIMIT 3
+    SELECT * FROM t3 WHERE b>0 ORDER BY a LIMIT 10
   }
-} {1 100 4 2 99 9 3 98 16 nosort}
+} {/1 100 4 2 99 9 3 98 16 .* nosort/}
+do_test where-6.7.2 {
+  cksort {
+    SELECT * FROM t3 WHERE b>0 ORDER BY a LIMIT 1
+  }
+} {1 100 4 sort}
 ifcapable subquery {
   do_test where-6.8a {
     cksort {
diff --git a/test/where2.test b/test/where2.test
index 434a7bcd..0a053350 100644
--- a/test/where2.test
+++ b/test/where2.test
@@ -765,4 +765,15 @@ do_execsql_test where2-13.1 {
   SELECT * FROM t13 WHERE (1=2 AND a=3) OR a=4;
 } {4 5}
 
+# https://www.sqlite.org/src/info/5e3c886796e5512e  (2016-03-09)
+# Correlated subquery on the RHS of an IN operator 
+#
+do_execsql_test where2-14.1 {
+  CREATE TABLE t14a(x INTEGER PRIMARY KEY);
+  INSERT INTO t14a(x) VALUES(1),(2),(3),(4);
+  CREATE TABLE t14b(y INTEGER PRIMARY KEY);
+  INSERT INTO t14b(y) VALUES(1);
+  SELECT x FROM t14a WHERE x NOT IN (SELECT x FROM t14b);
+} {}
+
 finish_test
diff --git a/test/where3.test b/test/where3.test
index c2804b55..6edbe2bc 100644
--- a/test/where3.test
+++ b/test/where3.test
@@ -45,7 +45,7 @@ do_test where3-1.1 {
   }
 } {222 two 2 222 {} {}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.1.1 {
      explain_no_trace {SELECT * FROM t1, t2 LEFT JOIN t3 ON q=x
                         WHERE p=2 AND a=q}
@@ -86,7 +86,7 @@ do_test where3-1.2 {
   }
 } {1 {Value for C1.1} {Value for C2.1} 2 {} {Value for C2.2} 3 {Value for C1.3} {Value for C2.3}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.2.1 {
      explain_no_trace {
        SELECT parent1.parent1key, child1.value, child2.value
diff --git a/test/whereD.test b/test/whereD.test
index de0f9618..8ced0ffc 100644
--- a/test/whereD.test
+++ b/test/whereD.test
@@ -337,4 +337,81 @@ do_searchcount_test 6.6.4 {
   SELECT c FROM x1 WHERE b=6 OR c=11 OR a=1
 } {7 11 3 search 7}
 
+#-------------------------------------------------------------------------
+#
+do_execsql_test 7.0 {
+  CREATE TABLE y1(a, b);
+  CREATE TABLE y2(x, y);
+  CREATE INDEX y2xy ON y2(x, y);
+  INSERT INTO y1 VALUES(1, 1);
+  INSERT INTO y2 VALUES(3, 3);
+}
+
+do_execsql_test 7.1 {
+  SELECT * FROM y1 LEFT JOIN y2 ON ((x=1 AND y=b) OR (x=2 AND y=b))
+} {1 1 {} {}}
+
+do_execsql_test 7.3 {
+  CREATE TABLE foo (Id INTEGER PRIMARY KEY, fa INTEGER, fb INTEGER); 
+  CREATE TABLE bar (Id INTEGER PRIMARY KEY, ba INTEGER, bb INTEGER);
+
+  INSERT INTO foo VALUES(1, 1, 1);
+  INSERT INTO foo VALUES(2, 1, 2);
+  INSERT INTO foo VALUES(3, 1, 3);
+  INSERT INTO foo VALUES(4, 1, 4);
+  INSERT INTO foo VALUES(5, 1, 5);
+  INSERT INTO foo VALUES(6, 1, 6);
+  INSERT INTO foo VALUES(7, 1, 7);
+  INSERT INTO foo VALUES(8, 1, 8);
+  INSERT INTO foo VALUES(9, 1, 9);
+
+  INSERT INTO bar VALUES(NULL, 1, 1);
+  INSERT INTO bar VALUES(NULL, 2, 2);
+  INSERT INTO bar VALUES(NULL, 3, 3);
+  INSERT INTO bar VALUES(NULL, 1, 4);
+  INSERT INTO bar VALUES(NULL, 2, 5);
+  INSERT INTO bar VALUES(NULL, 3, 6);
+  INSERT INTO bar VALUES(NULL, 1, 7);
+  INSERT INTO bar VALUES(NULL, 2, 8);
+  INSERT INTO bar VALUES(NULL, 3, 9);
+}
+
+do_execsql_test 7.4 {
+  SELECT 
+    bar.Id, bar.ba, bar.bb, foo.fb
+    FROM foo LEFT JOIN bar
+           ON (bar.ba = 1 AND bar.bb = foo.fb)
+           OR (bar.ba = 5 AND bar.bb = foo.fb);
+} {
+  1 1 1 1 
+  {} {} {} 2 
+  {} {} {} 3 
+  4 1 4 4 
+  {} {} {} 5 
+  {} {} {} 6 
+  7 1 7 7 
+  {} {} {} 8 
+  {} {} {} 9
+}
+
+do_execsql_test 7.5 {
+  CREATE INDEX idx_bar ON bar(ba, bb);
+  SELECT 
+    bar.Id, bar.ba, bar.bb, foo.fb
+    FROM foo LEFT JOIN bar
+           ON (bar.ba = 1 AND bar.bb = foo.fb)
+           OR (bar.ba = 5 AND bar.bb = foo.fb);
+} {
+  1 1 1 1 
+  {} {} {} 2 
+  {} {} {} 3 
+  4 1 4 4 
+  {} {} {} 5 
+  {} {} {} 6 
+  7 1 7 7 
+  {} {} {} 8 
+  {} {} {} 9
+}
+
+
 finish_test
diff --git a/test/win32nolock.test b/test/win32nolock.test
new file mode 100644
index 00000000..81288603
--- /dev/null
+++ b/test/win32nolock.test
@@ -0,0 +1,126 @@
+# 2016 July 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+if {$tcl_platform(platform)!="windows"} return
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix win32nolock
+
+do_test win32nolock-1.0 {
+  sqlite3 db test.db
+  execsql {
+    CREATE TABLE t1(a, b);
+    BEGIN;
+    INSERT INTO t1 VALUES(1, 2);
+  }
+} {}
+
+do_test win32nolock-1.1 {
+  execsql COMMIT
+  catchsql { SELECT * FROM t1 }
+} {0 {1 2}}
+
+db close
+
+do_test win32nolock-1.2 {
+  sqlite3 db test.db -vfs win32-none
+  sqlite3 db2 test.db -vfs win32-none
+  execsql { PRAGMA mmap_size = 0 } db2
+  execsql {
+    BEGIN;
+    INSERT INTO t1 VALUES(3, 4);
+  }
+} {}
+
+do_test win32nolock-1.3 {
+  execsql { SELECT * FROM t1 }
+} {1 2 3 4}
+
+do_test win32nolock-1.4 {
+  execsql { SELECT * FROM t1; } db2
+} {1 2}
+
+do_test win32nolock-1.5 {
+  execsql {
+    BEGIN;
+    SELECT * FROM t1;
+  } db2
+} {1 2}
+
+do_test win32nolock-1.6 {
+  execsql COMMIT
+  execsql {SELECT * FROM t1} db2
+} {1 2}
+
+ifcapable memorymanage {
+  do_test win32nolock-1.7 {
+    sqlite3_release_memory 1000000
+    execsql {SELECT * FROM t1} db2
+  } {1 2 3 4}
+}
+
+do_test win32nolock-1.8 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.9.1 {
+  sqlite3 db test.db
+  sqlite3 db2 test.db
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {1 {database is locked}}}
+
+do_test win32nolock-1.9.2 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.10.1 {
+  sqlite3 db test.db -vfs win32-none
+  sqlite3 db2 test.db
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {0 {}}}
+
+do_test win32nolock-1.10.2 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.11.1 {
+  sqlite3 db test.db
+  sqlite3 db2 test.db -vfs win32-none
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {0 {}}}
+
+do_test win32nolock-1.11.2 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.12.1 {
+  sqlite3 db test.db -vfs win32-none
+  sqlite3 db2 test.db -vfs win32-none
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {0 {}}}
+
+do_test win32nolock-1.12.2 {
+  db close
+  db2 close
+} {}
+
+finish_test
diff --git a/test/wordcount.c b/test/wordcount.c
index 72aa6b2f..5002918c 100644
--- a/test/wordcount.c
+++ b/test/wordcount.c
@@ -30,6 +30,7 @@
 **     --nosync             Use PRAGMA synchronous=OFF
 **     --journal MMMM       Use PRAGMA journal_mode=MMMM
 **     --timer              Time the operation of this program
+**     --tag NAME           Tag all output using NAME.  Use only stdout.
 **
 ** Modes:
 **
@@ -82,6 +83,9 @@
 #include "sqlite3.h"
 #define ISALPHA(X) isalpha((unsigned char)(X))
 
+/* Output tag */
+char *zTag = "--";
+
 /* Return the current wall-clock time */
 static sqlite3_int64 realTime(void){
   static sqlite3_vfs *clockVfs = 0;
@@ -115,7 +119,7 @@ static void traceCallback(void *NotUsed, const char *zSql){
 ** each column separated by a single space. */
 static int printResult(void *NotUsed, int nArg, char **azArg, char **azNm){
   int i;
-  printf("--");
+  printf("%s", zTag);
   for(i=0; i
+    /// This enumeration is used to represent all the possible exit codes from
+    /// this tool.
+    /// 
+    internal enum ExitCode
+    {
+        /// 
+        /// The file download was a success.
+        /// 
+        Success = 0,
+
+        /// 
+        /// The command line arguments are missing (i.e. null).  Generally,
+        /// this should not happen.
+        /// 
+        MissingArgs = 1,
+
+        /// 
+        /// The wrong number of command line arguments was supplied.
+        /// 
+        WrongNumArgs = 2,
+
+        /// 
+        /// The "matchingOnly" flag could not be converted to a value of the
+        ///  type.
+        /// 
+        BadMatchingOnlyFlag = 3,
+
+        /// 
+        /// An exception was caught in .  Generally, this
+        /// should not happen.
+        /// 
+        Exception = 4
+    }
+
+    ///////////////////////////////////////////////////////////////////////////
+
+    internal static class Replace
+    {
+        #region Private Support Methods
+        /// 
+        /// This method displays an error message to the console and/or
+        /// displays the command line usage information for this tool.
+        /// 
+        /// 
+        /// The error message to display, if any.
+        /// 
+        /// 
+        /// Non-zero to display the command line usage information.
+        /// 
+        private static void Error(
+            string message,
+            bool usage
+            )
+        {
+            if (message != null)
+                Console.WriteLine(message);
+
+            string fileName = Path.GetFileName(
+                Process.GetCurrentProcess().MainModule.FileName);
+
+            Console.WriteLine(String.Format(
+                "usage: {0}   ",
+                fileName));
+        }
+        #endregion
+
+        ///////////////////////////////////////////////////////////////////////
+
+        #region Program Entry Point
+        /// 
+        /// This is the entry-point for this tool.  It handles processing the
+        /// command line arguments, reading from the standard input channel,
+        /// replacing any matching lines of text, and writing to the standard
+        /// output channel.
+        /// 
+        /// 
+        /// The command line arguments.
+        /// 
+        /// 
+        /// Zero upon success; non-zero on failure.  This will be one of the
+        /// values from the  enumeration.
+        /// 
+        private static int Main(
+            string[] args
+            )
+        {
+            //
+            // NOTE: Sanity check the command line arguments.
+            //
+            if (args == null)
+            {
+                Error(null, true);
+                return (int)ExitCode.MissingArgs;
+            }
+
+            if (args.Length != 3)
+            {
+                Error(null, true);
+                return (int)ExitCode.WrongNumArgs;
+            }
+
+            try
+            {
+                //
+                // NOTE: Create a regular expression from the first command
+                //       line argument.  Then, grab the replacement string,
+                //       which is the second argument.
+                //
+                Regex regEx = new Regex(args[0]);
+                string replacement = args[1];
+
+                //
+                // NOTE: Attempt to convert the third argument to a boolean.
+                //
+                bool matchingOnly;
+
+                if (!bool.TryParse(args[2], out matchingOnly))
+                {
+                    Error(null, true);
+                    return (int)ExitCode.BadMatchingOnlyFlag;
+                }
+
+                //
+                // NOTE: Grab the standard input and output channels from the
+                //       console.
+                //
+                TextReader inputTextReader = Console.In;
+                TextWriter outputTextWriter = Console.Out;
+
+                //
+                // NOTE: Loop until end-of-file is hit on the standard input
+                //       stream.
+                //
+                while (true)
+                {
+                    //
+                    // NOTE: Read a line from the standard input channel.  If
+                    //       null is returned here, there is no more input and
+                    //       we are done.
+                    //
+                    string inputLine = inputTextReader.ReadLine();
+
+                    if (inputLine == null)
+                        break;
+
+                    //
+                    // NOTE: Perform regular expression replacements on this
+                    //       line, if any.  Then, write the modified line to
+                    //       the standard output channel.
+                    //
+                    string outputLine = regEx.Replace(inputLine, replacement);
+
+                    if (!matchingOnly || !String.Equals(
+                            inputLine, outputLine, StringComparison.Ordinal))
+                    {
+                        outputTextWriter.WriteLine(outputLine);
+                    }
+                }
+
+                //
+                // NOTE: At this point, everything has succeeded.
+                //
+                return (int)ExitCode.Success;
+            }
+            catch (Exception e)
+            {
+                //
+                // NOTE: An exception was caught.  Report it via the console
+                //       and return failure.
+                //
+                Error(e.ToString(), false);
+                return (int)ExitCode.Exception;
+            }
+        }
+        #endregion
+    }
+}
diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl
index 84e3994c..a6c58f1a 100644
--- a/tool/addopcodes.tcl
+++ b/tool/addopcodes.tcl
@@ -37,7 +37,10 @@ set extras {
   UMINUS
   UPLUS
   REGISTER
+  VECTOR
+  SELECT_COLUMN
   ASTERISK
+  SPAN
   SPACE
   ILLEGAL
 }
diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat
index 84528e37..497b05e3 100755
--- a/tool/build-all-msvc.bat
+++ b/tool/build-all-msvc.bat
@@ -22,7 +22,7 @@ REM
 REM Example:
 REM
 REM                        CD /D C:\dev\sqlite\core
-REM                        tool\build-all-msvc.bat C:\Temp
+REM                        CALL tool\build-all-msvc.bat C:\Temp
 REM
 REM In the example above, "C:\dev\sqlite\core" represents the root of the
 REM source tree for SQLite and "C:\Temp" represents the final destination
@@ -54,6 +54,11 @@ REM
 REM There are a few other environment variables that impact the build process
 REM when set ^(to anything^), they are:
 REM
+REM                        USE_AUTOCONF_MAKEFILE
+REM
+REM When set, the "autoconf" Makefile for MSVC will be used instead of the main
+REM Makefile for MSVC.  It must exist at "%ROOT%\autoconf\Makefile.msc".
+REM
 REM                        NOCLEAN
 REM
 REM When set, the "clean" target will not be used during each build iteration.
@@ -69,6 +74,11 @@ REM be skipped and they will not appear in the final destination directory.
 REM Setting this environment variable is never strictly needed and could cause
 REM issues in some circumstances; therefore, setting it is not recommended.
 REM
+REM                        NOMEMDEBUG
+REM
+REM When set, disables use of MEMDEBUG when building binaries for the "Debug"
+REM configuration.
+REM
 REM                        BUILD_ALL_SHELL
 REM
 REM When set, the command line shell will be built for each selected platform
@@ -88,17 +98,30 @@ REM on the WindowsSdkDir environment variable.  It causes this batch script to
 REM assume the Windows 10.0 SDK location should be used.
 REM
 REM                        NMAKE_ARGS
+REM                        NMAKE_ARGS_DEBUG
+REM                        NMAKE_ARGS_RETAIL
 REM
-REM When set, the value is expanded and passed to the NMAKE command line, after
-REM its other arguments.  This is used to specify additional NMAKE options, for
-REM example:
+REM When set, these values are expanded and passed to the NMAKE command line,
+REM after its other arguments.  These may be used to specify additional NMAKE
+REM options, for example:
 REM
 REM                        SET NMAKE_ARGS=FOR_WINRT=1
+REM                        SET NMAKE_ARGS_DEBUG=MEMDEBUG=1
+REM                        SET NMAKE_ARGS_RETAIL=WIN32HEAP=1
 REM
 REM Using the above command before running this tool will cause the compiled
 REM binaries to target the WinRT environment, which provides a subset of the
 REM Win32 API.
 REM
+REM                        DLL_FILE_NAME
+REM                        DLL_PDB_FILE_NAME
+REM                        LIB_FILE_NAME
+REM                        EXE_FILE_NAME
+REM                        EXE_PDB_FILE_NAME
+REM
+REM When set, these values will override the associated target file name used
+REM for the build.
+REM
 SETLOCAL
 
 REM SET __ECHO=ECHO
@@ -204,11 +227,17 @@ REM NOTE: If the command used to invoke NMAKE is not already set, use the
 REM       default.
 REM
 IF NOT DEFINED NMAKE_CMD (
-  SET NMAKE_CMD=nmake -B -f Makefile.msc
+  IF DEFINED USE_AUTOCONF_MAKEFILE (
+    SET NMAKE_CMD=nmake -B -f autoconf\Makefile.msc
+  ) ELSE (
+    SET NMAKE_CMD=nmake -B -f Makefile.msc
+  )
 )
 
 %_VECHO% NmakeCmd = '%NMAKE_CMD%'
 %_VECHO% NmakeArgs = '%NMAKE_ARGS%'
+%_VECHO% NmakeArgsDebug = '%NMAKE_ARGS_DEBUG%'
+%_VECHO% NmakeArgsRetail = '%NMAKE_ARGS_RETAIL%'
 
 REM
 REM NOTE: Setup environment variables to translate between the MSVC platform
@@ -239,7 +268,7 @@ REM
 IF DEFINED TCLSH_CMD (
   SET TCLSH_FILE=%TCLSH_CMD%
 ) ELSE (
-  SET TCLSH_FILE=tclsh85.exe
+  SET TCLSH_FILE=tclsh.exe
 )
 
 FOR %%T IN (%TCLSH_FILE%) DO (
@@ -255,6 +284,30 @@ IF NOT DEFINED %TCLSH_FILE%_PATH (
   GOTO errors
 )
 
+REM
+REM NOTE: Setup the default names for the build targets we are creating.  Any
+REM       ^(or all^) of these may end up being overridden.
+REM
+IF NOT DEFINED DLL_FILE_NAME (
+  SET DLL_FILE_NAME=sqlite3.dll
+)
+
+IF NOT DEFINED DLL_PDB_FILE_NAME (
+  SET DLL_PDB_FILE_NAME=sqlite3.pdb
+)
+
+IF NOT DEFINED LIB_FILE_NAME (
+  SET LIB_FILE_NAME=sqlite3.lib
+)
+
+IF NOT DEFINED EXE_FILE_NAME (
+  SET EXE_FILE_NAME=sqlite3.exe
+)
+
+IF NOT DEFINED EXE_PDB_FILE_NAME (
+  SET EXE_PDB_FILE_NAME=sqlite3sh.pdb
+)
+
 REM
 REM NOTE: Set the TOOLPATH variable to contain all the directories where the
 REM       external tools were found in the search above.
@@ -434,12 +487,20 @@ FOR %%P IN (%PLATFORMS%) DO (
         REM NOTE: Setting this to non-zero should enable the SQLITE_MEMDEBUG
         REM       define.
         REM
-        SET MEMDEBUG=1
+        IF NOT DEFINED NOMEMDEBUG (
+          SET MEMDEBUG=1
+        )
       ) ELSE (
         CALL :fn_UnsetVariable DEBUG
         CALL :fn_UnsetVariable MEMDEBUG
       )
 
+      REM
+      REM NOTE: Copy the extra NMAKE arguments for this configuration into the
+      REM       common variable used by the actual commands.
+      REM
+      CALL :fn_CopyVariable NMAKE_ARGS_%%B NMAKE_ARGS_CFG
+
       REM
       REM NOTE: Launch a nested command shell to perform the following steps:
       REM
@@ -546,7 +607,7 @@ FOR %%P IN (%PLATFORMS%) DO (
         REM       file, etc.
         REM
         IF NOT DEFINED NOCLEAN (
-          %__ECHO% %NMAKE_CMD% clean
+          CALL :fn_MakeClean %%D
 
           IF ERRORLEVEL 1 (
             ECHO Failed to clean for platform %%P.
@@ -559,7 +620,7 @@ FOR %%P IN (%PLATFORMS%) DO (
           REM       specifically wanting to build for each platform.
           REM
           %_AECHO% Cleaning final core library output files only...
-          %__ECHO% DEL /Q *.lo sqlite3.dll sqlite3.lib sqlite3.pdb 2%REDIRECT% NUL
+          %__ECHO% DEL /Q *.lo "%DLL_FILE_NAME%" "%LIB_FILE_NAME%" "%DLL_PDB_FILE_NAME%" 2%REDIRECT% NUL
         )
 
         REM
@@ -569,10 +630,10 @@ FOR %%P IN (%PLATFORMS%) DO (
         REM       Also, disable looking for and/or linking to the native Tcl
         REM       runtime library.
         REM
-        %__ECHO% %NMAKE_CMD% sqlite3.dll XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS%
+        CALL :fn_MakeDll %%D
 
         IF ERRORLEVEL 1 (
-          ECHO Failed to build %%B "sqlite3.dll" for platform %%P.
+          ECHO Failed to build %%B "%DLL_FILE_NAME%" for platform %%P.
           GOTO errors
         )
 
@@ -580,10 +641,10 @@ FOR %%P IN (%PLATFORMS%) DO (
         REM NOTE: Copy the "sqlite3.dll" file to the appropriate directory for
         REM       the build and platform beneath the binary directory.
         REM
-        %__ECHO% XCOPY sqlite3.dll "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+        %__ECHO% XCOPY "%DLL_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
 
         IF ERRORLEVEL 1 (
-          ECHO Failed to copy "sqlite3.dll" to "%BINARYDIRECTORY%\%%B\%%D\".
+          ECHO Failed to copy "%DLL_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
           GOTO errors
         )
 
@@ -591,10 +652,10 @@ FOR %%P IN (%PLATFORMS%) DO (
         REM NOTE: Copy the "sqlite3.lib" file to the appropriate directory for
         REM       the build and platform beneath the binary directory.
         REM
-        %__ECHO% XCOPY sqlite3.lib "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+        %__ECHO% XCOPY "%LIB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
 
         IF ERRORLEVEL 1 (
-          ECHO Failed to copy "sqlite3.lib" to "%BINARYDIRECTORY%\%%B\%%D\".
+          ECHO Failed to copy "%LIB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
           GOTO errors
         )
 
@@ -604,11 +665,13 @@ FOR %%P IN (%PLATFORMS%) DO (
         REM       are prevented from doing so.
         REM
         IF NOT DEFINED NOSYMBOLS (
-          %__ECHO% XCOPY sqlite3.pdb "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+          IF EXIST "%DLL_PDB_FILE_NAME%" (
+            %__ECHO% XCOPY "%DLL_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
 
-          IF ERRORLEVEL 1 (
-            ECHO Failed to copy "sqlite3.pdb" to "%BINARYDIRECTORY%\%%B\%%D\".
-            GOTO errors
+            IF ERRORLEVEL 1 (
+              ECHO Failed to copy "%DLL_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+              GOTO errors
+            )
           )
         )
 
@@ -627,7 +690,7 @@ FOR %%P IN (%PLATFORMS%) DO (
             REM       specifically wanting to build for each platform.
             REM
             %_AECHO% Cleaning final shell executable output files only...
-            %__ECHO% DEL /Q sqlite3.exe sqlite3sh.pdb 2%REDIRECT% NUL
+            %__ECHO% DEL /Q "%EXE_FILE_NAME%" "%EXE_PDB_FILE_NAME%" 2%REDIRECT% NUL
           )
 
           REM
@@ -637,10 +700,10 @@ FOR %%P IN (%PLATFORMS%) DO (
           REM       Also, disable looking for and/or linking to the native Tcl
           REM       runtime library.
           REM
-          %__ECHO% %NMAKE_CMD% sqlite3.exe XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS%
+          CALL :fn_MakeExe %%D
 
           IF ERRORLEVEL 1 (
-            ECHO Failed to build %%B "sqlite3.exe" for platform %%P.
+            ECHO Failed to build %%B "%EXE_FILE_NAME%" for platform %%P.
             GOTO errors
           )
 
@@ -648,10 +711,10 @@ FOR %%P IN (%PLATFORMS%) DO (
           REM NOTE: Copy the "sqlite3.exe" file to the appropriate directory
           REM       for the build and platform beneath the binary directory.
           REM
-          %__ECHO% XCOPY sqlite3.exe "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+          %__ECHO% XCOPY "%EXE_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
 
           IF ERRORLEVEL 1 (
-            ECHO Failed to copy "sqlite3.exe" to "%BINARYDIRECTORY%\%%B\%%D\".
+            ECHO Failed to copy "%EXE_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
             GOTO errors
           )
 
@@ -661,11 +724,13 @@ FOR %%P IN (%PLATFORMS%) DO (
           REM       unless we are prevented from doing so.
           REM
           IF NOT DEFINED NOSYMBOLS (
-            %__ECHO% XCOPY sqlite3sh.pdb "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
+            IF EXIST "%EXE_PDB_FILE_NAME%" (
+              %__ECHO% XCOPY "%EXE_PDB_FILE_NAME%" "%BINARYDIRECTORY%\%%B\%%D\" %FFLAGS% %DFLAGS%
 
-            IF ERRORLEVEL 1 (
-              ECHO Failed to copy "sqlite3sh.pdb" to "%BINARYDIRECTORY%\%%B\%%D\".
-              GOTO errors
+              IF ERRORLEVEL 1 (
+                ECHO Failed to copy "%EXE_PDB_FILE_NAME%" to "%BINARYDIRECTORY%\%%B\%%D\".
+                GOTO errors
+              )
             )
           )
         )
@@ -696,6 +761,18 @@ REM NOTE: If we get to this point, we have succeeded.
 REM
 GOTO no_errors
 
+:fn_MakeClean
+  %__ECHO% %NMAKE_CMD% clean "PLATFORM=%1" XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% %NMAKE_ARGS_CFG%
+  GOTO :EOF
+
+:fn_MakeDll
+  %__ECHO% %NMAKE_CMD% "%DLL_FILE_NAME%" "PLATFORM=%1" XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% %NMAKE_ARGS_CFG%
+  GOTO :EOF
+
+:fn_MakeExe
+  %__ECHO% %NMAKE_CMD% "%EXE_FILE_NAME%" "PLATFORM=%1" XCOMPILE=1 USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% %NMAKE_ARGS_CFG%
+  GOTO :EOF
+
 :fn_ShowVariable
   SETLOCAL
   SET __ECHO_CMD=ECHO %%%2%%
diff --git a/tool/dbhash.c b/tool/dbhash.c
new file mode 100644
index 00000000..b1c72b3e
--- /dev/null
+++ b/tool/dbhash.c
@@ -0,0 +1,506 @@
+/*
+** 2016-06-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that computes an SHA1 hash on the content
+** of an SQLite database.
+**
+** The hash is computed over just the content of the database.  Free
+** space inside of the database file, and alternative on-disk representations
+** of the same content (ex: UTF8 vs UTF16) do not affect the hash.  So,
+** for example, the database file page size, encoding, and auto_vacuum setting
+** can all be changed without changing the hash.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "sqlite3.h"
+
+/* Context for the SHA1 hash */
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+struct GlobalVars {
+  const char *zArgv0;       /* Name of program */
+  unsigned fDebug;          /* Debug flags */
+  sqlite3 *db;              /* The database connection */
+  SHA1Context cx;           /* SHA1 hash context */
+} g;
+
+/*
+** Debugging flags
+*/
+#define DEBUG_FULLTRACE   0x00000001   /* Trace hash to stderr */
+
+/******************************************************************************
+** The Hash Engine
+**
+** Modify these routines (and appropriate state fields in global variable 'g')
+** in order to compute a different (better?) hash of the database.
+*/
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
+/*
+ * GCC by itself only generates left rotates.  Use right rotates if
+ * possible to be kinder to dinky implementations with iterative rotate
+ * instructions.
+ */
+#define SHA_ROT(op, x, k) \
+        ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
+#define rol(x,k) SHA_ROT("roll", x, k)
+#define ror(x,k) SHA_ROT("rorl", x, k)
+
+#else
+/* Generic C equivalent */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#endif
+
+
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy g.cx.state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+
+/* Initialize the SHA1 hash */
+static void hash_init(void){
+  /* SHA1 initialization constants */
+  g.cx.state[0] = 0x67452301;
+  g.cx.state[1] = 0xEFCDAB89;
+  g.cx.state[2] = 0x98BADCFE;
+  g.cx.state[3] = 0x10325476;
+  g.cx.state[4] = 0xC3D2E1F0;
+  g.cx.count[0] = g.cx.count[1] = 0;
+}
+
+/* Add new content to the SHA1 hash */
+static void hash_step(const unsigned char *data,  unsigned int len){
+  unsigned int i, j;
+
+  j = g.cx.count[0];
+  if( (g.cx.count[0] += len << 3) < j ){
+    g.cx.count[1] += (len>>29)+1;
+  }
+  j = (j >> 3) & 63;
+  if( (j + len) > 63 ){
+    (void)memcpy(&g.cx.buffer[j], data, (i = 64-j));
+    SHA1Transform(g.cx.state, g.cx.buffer);
+    for(; i + 63 < len; i += 64){
+      SHA1Transform(g.cx.state, &data[i]);
+    }
+    j = 0;
+  }else{
+    i = 0;
+  }
+  (void)memcpy(&g.cx.buffer[j], &data[i], len - i);
+}
+
+
+/* Add padding and compute and output the message digest. */
+static void hash_finish(const char *zName){
+  unsigned int i;
+  unsigned char finalcount[8];
+  unsigned char digest[20];
+  static const char zEncode[] = "0123456789abcdef";
+  char zOut[41];
+
+  for (i = 0; i < 8; i++){
+    finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)]
+       >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+  }
+  hash_step((const unsigned char *)"\200", 1);
+  while ((g.cx.count[0] & 504) != 448){
+    hash_step((const unsigned char *)"\0", 1);
+  }
+  hash_step(finalcount, 8);  /* Should cause a SHA1Transform() */
+  for (i = 0; i < 20; i++){
+    digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+  }
+  for(i=0; i<20; i++){
+    zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
+    zOut[i*2+1] = zEncode[digest[i] & 0xf];
+  }
+  zOut[i*2]= 0;
+  printf("%s %s\n", zOut, zName);
+}
+/* End of the hashing logic
+*******************************************************************************/
+  
+/*
+** Print an error resulting from faulting command-line arguments and
+** abort the program.
+*/
+static void cmdlineError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  exit(1);
+}
+
+/*
+** Print an error message for an error that occurs at runtime, then
+** abort the program.
+*/
+static void runtimeError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Prepare a new SQL statement.  Print an error and abort if anything
+** goes wrong.
+*/
+static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
+  char *zSql;
+  int rc;
+  sqlite3_stmt *pStmt;
+
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  if( zSql==0 ) runtimeError("out of memory");
+  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
+  if( rc ){
+    runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
+                 zSql);
+  }
+  sqlite3_free(zSql);
+  return pStmt;
+}
+static sqlite3_stmt *db_prepare(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/*
+** Compute the hash for all rows of the query formed from the printf-style
+** zFormat and its argument.
+*/
+static void hash_one_query(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;        /* The query defined by zFormat and "..." */
+  int nCol;                   /* Number of columns in the result set */
+  int i;                      /* Loop counter */
+
+  /* Prepare the query defined by zFormat and "..." */
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  nCol = sqlite3_column_count(pStmt);
+
+  /* Compute a hash over the result of the query */
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    for(i=0; i=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"1",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_FLOAT: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          double r = sqlite3_column_double(pStmt,i);
+          memcpy(&u, &r, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"2",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_TEXT: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_text(pStmt, i);
+          hash_step((const unsigned char*)"3", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_BLOB: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_blob(pStmt, i);
+          hash_step((const unsigned char*)"4", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "BLOB (%d bytes)\n", n);
+          }
+          break;
+        }
+      }
+    }
+  }
+  sqlite3_finalize(pStmt);
+}
+
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(void){
+  printf("Usage: %s [options] FILE ...\n", g.zArgv0);
+  printf(
+"Compute a SHA1 hash on the content of database FILE.  System tables such as\n"
+"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n"
+"Options:\n"
+"   --debug N           Set debugging flags to N (experts only)\n"
+"   --like PATTERN      Only hash tables whose name is LIKE the pattern\n"
+"   --schema-only       Only hash the schema - omit table content\n"
+"   --without-schema    Only hash table content - omit the schema\n"
+  );
+}
+
+int main(int argc, char **argv){
+  const char *zDb = 0;         /* Name of the database currently being hashed */
+  int i;                       /* Loop counter */
+  int rc;                      /* Subroutine return code */
+  char *zErrMsg;               /* Error message when opening database */
+  sqlite3_stmt *pStmt;         /* An SQLite query */
+  const char *zLike = 0;       /* LIKE pattern of tables to hash */
+  int omitSchema = 0;          /* True to compute hash on content only */
+  int omitContent = 0;         /* True to compute hash on schema only */
+  int nFile = 0;               /* Number of input filenames seen */
+
+  g.zArgv0 = argv[0];
+  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+  for(i=1; ibase;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a series_cursor.
+*/
+static int seriesClose(sqlite3_vtab_cursor *cur){
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a series_cursor to its next row of output.
+*/
+static int seriesNext(sqlite3_vtab_cursor *cur){
+  series_cursor *pCur = (series_cursor*)cur;
+  if( pCur->isDesc ){
+    pCur->iValue -= pCur->iStep;
+  }else{
+    pCur->iValue += pCur->iStep;
+  }
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int seriesColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  series_cursor *pCur = (series_cursor*)cur;
+  sqlite3_int64 x = 0;
+  switch( i ){
+    case SERIES_COLUMN_START:  x = pCur->mnValue; break;
+    case SERIES_COLUMN_STOP:   x = pCur->mxValue; break;
+    case SERIES_COLUMN_STEP:   x = pCur->iStep;   break;
+    default:                   x = pCur->iValue;  break;
+  }
+  sqlite3_result_int64(ctx, x);
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  series_cursor *pCur = (series_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int seriesEof(sqlite3_vtab_cursor *cur){
+  series_cursor *pCur = (series_cursor*)cur;
+  if( pCur->isDesc ){
+    return pCur->iValue < pCur->mnValue;
+  }else{
+    return pCur->iValue > pCur->mxValue;
+  }
+}
+
+/* True to cause run-time checking of the start=, stop=, and/or step= 
+** parameters.  The only reason to do this is for testing the
+** constraint checking logic for virtual tables in the SQLite core.
+*/
+#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY
+# define SQLITE_SERIES_CONSTRAINT_VERIFY 0
+#endif
+
+/*
+** This method is called to "rewind" the series_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to seriesColumn() or seriesRowid() or 
+** seriesEof().
+**
+** The query plan selected by seriesBestIndex is passed in the idxNum
+** parameter.  (idxStr is not used in this implementation.)  idxNum
+** is a bitmask showing which constraints are available:
+**
+**    1:    start=VALUE
+**    2:    stop=VALUE
+**    4:    step=VALUE
+**
+** Also, if bit 8 is set, that means that the series should be output
+** in descending order rather than in ascending order.
+**
+** This routine should initialize the cursor and position it so that it
+** is pointing at the first row, or pointing off the end of the table
+** (so that seriesEof() will return true) if the table is empty.
+*/
+static int seriesFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  series_cursor *pCur = (series_cursor *)pVtabCursor;
+  int i = 0;
+  if( idxNum & 1 ){
+    pCur->mnValue = sqlite3_value_int64(argv[i++]);
+  }else{
+    pCur->mnValue = 0;
+  }
+  if( idxNum & 2 ){
+    pCur->mxValue = sqlite3_value_int64(argv[i++]);
+  }else{
+    pCur->mxValue = 0xffffffff;
+  }
+  if( idxNum & 4 ){
+    pCur->iStep = sqlite3_value_int64(argv[i++]);
+    if( pCur->iStep<1 ) pCur->iStep = 1;
+  }else{
+    pCur->iStep = 1;
+  }
+  if( idxNum & 8 ){
+    pCur->isDesc = 1;
+    pCur->iValue = pCur->mxValue;
+    if( pCur->iStep>0 ){
+      pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep;
+    }
+  }else{
+    pCur->isDesc = 0;
+    pCur->iValue = pCur->mnValue;
+  }
+  pCur->iRowid = 1;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the generate_series virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan.  idxStr is unused.
+**
+** The query plan is represented by bits in idxNum:
+**
+**  (1)  start = $value  -- constraint exists
+**  (2)  stop = $value   -- constraint exists
+**  (4)  step = $value   -- constraint exists
+**  (8)  output in descending order
+*/
+static int seriesBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                 /* Loop over constraints */
+  int idxNum = 0;        /* The query plan bitmask */
+  int startIdx = -1;     /* Index of the start= constraint, or -1 if none */
+  int stopIdx = -1;      /* Index of the stop= constraint, or -1 if none */
+  int stepIdx = -1;      /* Index of the step= constraint, or -1 if none */
+  int nArg = 0;          /* Number of arguments that seriesFilter() expects */
+
+  const struct sqlite3_index_constraint *pConstraint;
+  pConstraint = pIdxInfo->aConstraint;
+  for(i=0; inConstraint; i++, pConstraint++){
+    if( pConstraint->usable==0 ) continue;
+    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    switch( pConstraint->iColumn ){
+      case SERIES_COLUMN_START:
+        startIdx = i;
+        idxNum |= 1;
+        break;
+      case SERIES_COLUMN_STOP:
+        stopIdx = i;
+        idxNum |= 2;
+        break;
+      case SERIES_COLUMN_STEP:
+        stepIdx = i;
+        idxNum |= 4;
+        break;
+    }
+  }
+  if( startIdx>=0 ){
+    pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
+    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  }
+  if( stopIdx>=0 ){
+    pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
+    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  }
+  if( stepIdx>=0 ){
+    pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
+    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  }
+  if( (idxNum & 3)==3 ){
+    /* Both start= and stop= boundaries are available.  This is the 
+    ** the preferred case */
+    pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0));
+    pIdxInfo->estimatedRows = 1000;
+    if( pIdxInfo->nOrderBy==1 ){
+      if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8;
+      pIdxInfo->orderByConsumed = 1;
+    }
+  }else{
+    /* If either boundary is missing, we have to generate a huge span
+    ** of numbers.  Make this case very expensive so that the query
+    ** planner will work hard to avoid it. */
+    pIdxInfo->estimatedCost = (double)2147483647;
+    pIdxInfo->estimatedRows = 2147483647;
+  }
+  pIdxInfo->idxNum = idxNum;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** generate_series virtual table.
+*/
+static sqlite3_module seriesModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  seriesConnect,             /* xConnect */
+  seriesBestIndex,           /* xBestIndex */
+  seriesDisconnect,          /* xDisconnect */
+  0,                         /* xDestroy */
+  seriesOpen,                /* xOpen - open a cursor */
+  seriesClose,               /* xClose - close a cursor */
+  seriesFilter,              /* xFilter - configure scan constraints */
+  seriesNext,                /* xNext - advance a cursor */
+  seriesEof,                 /* xEof - check for end of scan */
+  seriesColumn,              /* xColumn - read data */
+  seriesRowid,               /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+};
+/* END the generate_series(START,END,STEP) implementation
+*********************************************************************************/
+
 /*
 ** Print sketchy documentation for this utility program
 */
@@ -729,6 +1049,7 @@ int main(int argc, char **argv){
     #endif
         sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
         sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
+        sqlite3_create_module(db, "generate_series", &seriesModule, 0);
         sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);
         if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding);
         if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
diff --git a/tool/lemon.c b/tool/lemon.c
index d704deb6..0fa3d63b 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -263,7 +263,8 @@ struct symbol {
   int useCnt;              /* Number of times used */
   char *destructor;        /* Code which executes whenever this symbol is
                            ** popped from the stack during error processing */
-  int destLineno;          /* Line number for start of destructor */
+  int destLineno;          /* Line number for start of destructor.  Set to
+                           ** -1 for duplicate destructors. */
   char *datatype;          /* The data type of information held by this
                            ** object. Only used if type==NONTERMINAL */
   int dtnum;               /* The data type number.  In the parser, the value
@@ -286,9 +287,15 @@ struct rule {
   const char **rhsalias;   /* An alias for each RHS symbol (NULL if none) */
   int line;                /* Line number at which code begins */
   const char *code;        /* The code executed when this rule is reduced */
+  const char *codePrefix;  /* Setup code before code[] above */
+  const char *codeSuffix;  /* Breakdown code after code[] above */
+  int noCode;              /* True if this rule has no associated C code */
+  int codeEmitted;         /* True if the code has been emitted already */
   struct symbol *precsym;  /* Precedence symbol for this rule */
   int index;               /* An index number for this rule */
+  int iRule;               /* Rule number as used in the generated tables */
   Boolean canReduce;       /* True if this rule is ever reduced */
+  Boolean doesReduce;      /* Reduce actions occur after optimization */
   struct rule *nextlhs;    /* Next rule with the same LHS */
   struct rule *next;       /* Next rule in the global list */
 };
@@ -336,6 +343,7 @@ struct action {
     struct state *stp;     /* The new state, if a shift */
     struct rule *rp;       /* The rule, if a reduce */
   } x;
+  struct symbol *spOpt;    /* SHIFTREDUCE optimization to this symbol */
   struct action *next;     /* Next action for this state */
   struct action *collide;  /* Next action with the same hash */
 };
@@ -346,7 +354,7 @@ struct state {
   struct config *bp;       /* The basis configurations for this state */
   struct config *cfp;      /* All configurations in this set */
   int statenum;            /* Sequential number for this state */
-  struct action *ap;       /* Array of actions for this state */
+  struct action *ap;       /* List of actions for this state */
   int nTknAct, nNtAct;     /* Number of actions on terminals and nonterminals */
   int iTknOfst, iNtOfst;   /* yy_action[] offset for terminals and nonterms */
   int iDfltReduce;         /* Default action is to REDUCE by this rule */
@@ -370,6 +378,7 @@ struct plink {
 struct lemon {
   struct state **sorted;   /* Table of states sorted by state number */
   struct rule *rule;       /* List of all rules */
+  struct rule *startRule;  /* First rule */
   int nstate;              /* Number of states */
   int nxstate;             /* nstate with tail degenerate states removed */
   int nrule;               /* Number of rules */
@@ -526,6 +535,7 @@ void Action_add(
   *app = newaction;
   newaction->type = type;
   newaction->sp = sp;
+  newaction->spOpt = 0;
   if( type==SHIFT ){
     newaction->x.stp = (struct state *)arg;
   }else{
@@ -856,12 +866,12 @@ void FindStates(struct lemon *lemp)
       ErrorMsg(lemp->filename,0,
 "The specified start symbol \"%s\" is not \
 in a nonterminal of the grammar.  \"%s\" will be used as the start \
-symbol instead.",lemp->start,lemp->rule->lhs->name);
+symbol instead.",lemp->start,lemp->startRule->lhs->name);
       lemp->errorcnt++;
-      sp = lemp->rule->lhs;
+      sp = lemp->startRule->lhs;
     }
   }else{
-    sp = lemp->rule->lhs;
+    sp = lemp->startRule->lhs;
   }
 
   /* Make sure the start symbol doesn't occur on the right-hand side of
@@ -1115,9 +1125,9 @@ void FindActions(struct lemon *lemp)
   /* Add the accepting token */
   if( lemp->start ){
     sp = Symbol_find(lemp->start);
-    if( sp==0 ) sp = lemp->rule->lhs;
+    if( sp==0 ) sp = lemp->startRule->lhs;
   }else{
-    sp = lemp->rule->lhs;
+    sp = lemp->startRule->lhs;
   }
   /* Add to the first state (which is always the starting state of the
   ** finite state machine) an action to ACCEPT if the lookahead is the
@@ -1495,6 +1505,54 @@ static void handle_T_option(char *z){
   lemon_strcpy(user_templatename, z);
 }
 
+/* Merge together to lists of rules ordered by rule.iRule */
+static struct rule *Rule_merge(struct rule *pA, struct rule *pB){
+  struct rule *pFirst = 0;
+  struct rule **ppPrev = &pFirst;
+  while( pA && pB ){
+    if( pA->iRuleiRule ){
+      *ppPrev = pA;
+      ppPrev = &pA->next;
+      pA = pA->next;
+    }else{
+      *ppPrev = pB;
+      ppPrev = &pB->next;
+      pB = pB->next;
+    }
+  }
+  if( pA ){
+    *ppPrev = pA;
+  }else{
+    *ppPrev = pB;
+  }
+  return pFirst;
+}
+
+/*
+** Sort a list of rules in order of increasing iRule value
+*/
+static struct rule *Rule_sort(struct rule *rp){
+  int i;
+  struct rule *pNext;
+  struct rule *x[32];
+  memset(x, 0, sizeof(x));
+  while( rp ){
+    pNext = rp->next;
+    rp->next = 0;
+    for(i=0; iname[0]); i++);
   lem.nterminal = i;
 
+  /* Assign sequential rule numbers.  Start with 0.  Put rules that have no
+  ** reduce action C-code associated with them last, so that the switch()
+  ** statement that selects reduction actions will have a smaller jump table.
+  */
+  for(i=0, rp=lem.rule; rp; rp=rp->next){
+    rp->iRule = rp->code ? i++ : -1;
+  }
+  for(rp=lem.rule; rp; rp=rp->next){
+    if( rp->iRule<0 ) rp->iRule = i++;
+  }
+  lem.startRule = lem.rule;
+  lem.rule = Rule_sort(lem.rule);
+
   /* Generate a reprint of the grammar, if requested on the command line */
   if( rpflag ){
     Reprint(&lem);
@@ -2148,6 +2220,7 @@ to follow the previous rule.");
         }else{
           psp->prevrule->line = psp->tokenlineno;
           psp->prevrule->code = &x[1];
+          psp->prevrule->noCode = 0;
         }
       }else if( x[0]=='[' ){
         psp->state = PRECEDENCE_MARK_1;
@@ -2254,6 +2327,7 @@ to follow the previous rule.");
           rp->lhsalias = psp->lhsalias;
           rp->nrhs = psp->nrhs;
           rp->code = 0;
+          rp->noCode = 1;
           rp->precsym = 0;
           rp->index = psp->gp->nrule++;
           rp->nextlhs = rp->lhs->rule;
@@ -3052,13 +3126,13 @@ int PrintAction(
     }
     case REDUCE: {
       struct rule *rp = ap->x.rp;
-      fprintf(fp,"%*s reduce       %-7d",indent,ap->sp->name,rp->index);
+      fprintf(fp,"%*s reduce       %-7d",indent,ap->sp->name,rp->iRule);
       RulePrint(fp, rp, -1);
       break;
     }
     case SHIFTREDUCE: {
       struct rule *rp = ap->x.rp;
-      fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index);
+      fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule);
       RulePrint(fp, rp, -1);
       break;
     }
@@ -3071,7 +3145,7 @@ int PrintAction(
     case SRCONFLICT:
     case RRCONFLICT:
       fprintf(fp,"%*s reduce       %-7d ** Parsing conflict **",
-        indent,ap->sp->name,ap->x.rp->index);
+        indent,ap->sp->name,ap->x.rp->iRule);
       break;
     case SSCONFLICT:
       fprintf(fp,"%*s shift        %-7d ** Parsing conflict **", 
@@ -3088,7 +3162,7 @@ int PrintAction(
     case RD_RESOLVED:
       if( showPrecedenceConflict ){
         fprintf(fp,"%*s reduce %-7d -- dropped by precedence",
-                indent,ap->sp->name,ap->x.rp->index);
+                indent,ap->sp->name,ap->x.rp->iRule);
       }else{
         result = 0;
       }
@@ -3097,6 +3171,9 @@ int PrintAction(
       result = 0;
       break;
   }
+  if( result && ap->spOpt ){
+    fprintf(fp,"  /* because %s==%s */", ap->sp->name, ap->spOpt->name);
+  }
   return result;
 }
 
@@ -3119,7 +3196,7 @@ void ReportOutput(struct lemon *lemp)
     while( cfp ){
       char buf[20];
       if( cfp->dot==cfp->rp->nrhs ){
-        lemon_sprintf(buf,"(%d)",cfp->rp->index);
+        lemon_sprintf(buf,"(%d)",cfp->rp->iRule);
         fprintf(fp,"    %5s ",buf);
       }else{
         fprintf(fp,"          ");
@@ -3220,8 +3297,8 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap)
   int act;
   switch( ap->type ){
     case SHIFT:  act = ap->x.stp->statenum;                        break;
-    case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate;        break;
-    case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break;
+    case SHIFTREDUCE: act = ap->x.rp->iRule + lemp->nstate;        break;
+    case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break;
     case ERROR:  act = lemp->nstate + lemp->nrule*2;               break;
     case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1;           break;
     default:     act = -1; break;
@@ -3430,6 +3507,7 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){
   int c;
   char zInt[40];
   if( zText==0 ){
+    if( used==0 && z!=0 ) z[0] = 0;
     used = 0;
     return z;
   }
@@ -3463,15 +3541,23 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){
 }
 
 /*
-** zCode is a string that is the action associated with a rule.  Expand
-** the symbols in this string so that the refer to elements of the parser
-** stack.
+** Write and transform the rp->code string so that symbols are expanded.
+** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate.
+**
+** Return 1 if the expanded code requires that "yylhsminor" local variable
+** to be defined.
 */
-PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){
+PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
   char *cp, *xp;
   int i;
-  char lhsused = 0;    /* True if the LHS element has been used */
-  char used[MAXRHS];   /* True for each RHS element which is used */
+  int rc = 0;            /* True if yylhsminor is used */
+  int dontUseRhs0 = 0;   /* If true, use of left-most RHS label is illegal */
+  const char *zSkip = 0; /* The zOvwrt comment within rp->code, or NULL */
+  char lhsused = 0;      /* True if the LHS element has been used */
+  char lhsdirect;        /* True if LHS writes directly into stack */
+  char used[MAXRHS];     /* True for each RHS element which is used */
+  char zLhs[50];         /* Convert the LHS symbol into this string */
+  char zOvwrt[900];      /* Comment that to allow LHS to overwrite RHS */
 
   for(i=0; inrhs; i++) used[i] = 0;
   lhsused = 0;
@@ -3480,25 +3566,89 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){
     static char newlinestr[2] = { '\n', '\0' };
     rp->code = newlinestr;
     rp->line = rp->ruleline;
+    rp->noCode = 1;
+  }else{
+    rp->noCode = 0;
+  }
+
+
+  if( rp->nrhs==0 ){
+    /* If there are no RHS symbols, then writing directly to the LHS is ok */
+    lhsdirect = 1;
+  }else if( rp->rhsalias[0]==0 ){
+    /* The left-most RHS symbol has no value.  LHS direct is ok.  But
+    ** we have to call the distructor on the RHS symbol first. */
+    lhsdirect = 1;
+    if( has_destructor(rp->rhs[0],lemp) ){
+      append_str(0,0,0,0);
+      append_str("  yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0,
+                 rp->rhs[0]->index,1-rp->nrhs);
+      rp->codePrefix = Strsafe(append_str(0,0,0,0));
+      rp->noCode = 0;
+    }
+  }else if( rp->lhsalias==0 ){
+    /* There is no LHS value symbol. */
+    lhsdirect = 1;
+  }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){
+    /* The LHS symbol and the left-most RHS symbol are the same, so 
+    ** direct writing is allowed */
+    lhsdirect = 1;
+    lhsused = 1;
+    used[0] = 1;
+    if( rp->lhs->dtnum!=rp->rhs[0]->dtnum ){
+      ErrorMsg(lemp->filename,rp->ruleline,
+        "%s(%s) and %s(%s) share the same label but have "
+        "different datatypes.",
+        rp->lhs->name, rp->lhsalias, rp->rhs[0]->name, rp->rhsalias[0]);
+      lemp->errorcnt++;
+    }    
+  }else{
+    lemon_sprintf(zOvwrt, "/*%s-overwrites-%s*/",
+                  rp->lhsalias, rp->rhsalias[0]);
+    zSkip = strstr(rp->code, zOvwrt);
+    if( zSkip!=0 ){
+      /* The code contains a special comment that indicates that it is safe
+      ** for the LHS label to overwrite left-most RHS label. */
+      lhsdirect = 1;
+    }else{
+      lhsdirect = 0;
+    }
+  }
+  if( lhsdirect ){
+    sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum);
+  }else{
+    rc = 1;
+    sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum);
   }
 
   append_str(0,0,0,0);
 
   /* This const cast is wrong but harmless, if we're careful. */
   for(cp=(char *)rp->code; *cp; cp++){
+    if( cp==zSkip ){
+      append_str(zOvwrt,0,0,0);
+      cp += lemonStrlen(zOvwrt)-1;
+      dontUseRhs0 = 1;
+      continue;
+    }
     if( ISALPHA(*cp) && (cp==rp->code || (!ISALNUM(cp[-1]) && cp[-1]!='_')) ){
       char saved;
       for(xp= &cp[1]; ISALNUM(*xp) || *xp=='_'; xp++);
       saved = *xp;
       *xp = 0;
       if( rp->lhsalias && strcmp(cp,rp->lhsalias)==0 ){
-        append_str("yygotominor.yy%d",0,rp->lhs->dtnum,0);
+        append_str(zLhs,0,0,0);
         cp = xp;
         lhsused = 1;
       }else{
         for(i=0; inrhs; i++){
           if( rp->rhsalias[i] && strcmp(cp,rp->rhsalias[i])==0 ){
-            if( cp!=rp->code && cp[-1]=='@' ){
+            if( i==0 && dontUseRhs0 ){
+              ErrorMsg(lemp->filename,rp->ruleline,
+                 "Label %s used after '%s'.",
+                 rp->rhsalias[0], zOvwrt);
+              lemp->errorcnt++;
+            }else if( cp!=rp->code && cp[-1]=='@' ){
               /* If the argument is of the form @X then substituted
               ** the token number of X, not the value of X */
               append_str("yymsp[%d].major",-1,i-rp->nrhs+1,0);
@@ -3523,6 +3673,11 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){
     append_str(cp, 1, 0, 0);
   } /* End loop */
 
+  /* Main code generation completed */
+  cp = append_str(0,0,0,0);
+  if( cp && cp[0] ) rp->code = Strsafe(cp);
+  append_str(0,0,0,0);
+
   /* Check to make sure the LHS has been used */
   if( rp->lhsalias && !lhsused ){
     ErrorMsg(lemp->filename,rp->ruleline,
@@ -3531,27 +3686,58 @@ PRIVATE void translate_code(struct lemon *lemp, struct rule *rp){
     lemp->errorcnt++;
   }
 
-  /* Generate destructor code for RHS symbols which are not used in the
-  ** reduce code */
+  /* Generate destructor code for RHS minor values which are not referenced.
+  ** Generate error messages for unused labels and duplicate labels.
+  */
   for(i=0; inrhs; i++){
-    if( rp->rhsalias[i] && !used[i] ){
-      ErrorMsg(lemp->filename,rp->ruleline,
-        "Label %s for \"%s(%s)\" is never used.",
-        rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]);
-      lemp->errorcnt++;
-    }else if( rp->rhsalias[i]==0 ){
-      if( has_destructor(rp->rhs[i],lemp) ){
-        append_str("  yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0,
-           rp->rhs[i]->index,i-rp->nrhs+1);
-      }else{
-        /* No destructor defined for this term */
+    if( rp->rhsalias[i] ){
+      if( i>0 ){
+        int j;
+        if( rp->lhsalias && strcmp(rp->lhsalias,rp->rhsalias[i])==0 ){
+          ErrorMsg(lemp->filename,rp->ruleline,
+            "%s(%s) has the same label as the LHS but is not the left-most "
+            "symbol on the RHS.",
+            rp->rhs[i]->name, rp->rhsalias);
+          lemp->errorcnt++;
+        }
+        for(j=0; jrhsalias[j] && strcmp(rp->rhsalias[j],rp->rhsalias[i])==0 ){
+            ErrorMsg(lemp->filename,rp->ruleline,
+              "Label %s used for multiple symbols on the RHS of a rule.",
+              rp->rhsalias[i]);
+            lemp->errorcnt++;
+            break;
+          }
+        }
       }
+      if( !used[i] ){
+        ErrorMsg(lemp->filename,rp->ruleline,
+          "Label %s for \"%s(%s)\" is never used.",
+          rp->rhsalias[i],rp->rhs[i]->name,rp->rhsalias[i]);
+        lemp->errorcnt++;
+      }
+    }else if( i>0 && has_destructor(rp->rhs[i],lemp) ){
+      append_str("  yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0,
+         rp->rhs[i]->index,i-rp->nrhs+1);
     }
   }
-  if( rp->code ){
-    cp = append_str(0,0,0,0);
-    rp->code = Strsafe(cp?cp:"");
+
+  /* If unable to write LHS values directly into the stack, write the
+  ** saved LHS value now. */
+  if( lhsdirect==0 ){
+    append_str("  yymsp[%d].minor.yy%d = ", 0, 1-rp->nrhs, rp->lhs->dtnum);
+    append_str(zLhs, 0, 0, 0);
+    append_str(";\n", 0, 0, 0);
   }
+
+  /* Suffix code generation complete */
+  cp = append_str(0,0,0,0);
+  if( cp && cp[0] ){
+    rp->codeSuffix = Strsafe(cp);
+    rp->noCode = 0;
+  }
+
+  return rc;
 }
 
 /* 
@@ -3566,6 +3752,12 @@ PRIVATE void emit_code(
 ){
  const char *cp;
 
+ /* Setup code prior to the #line directive */
+ if( rp->codePrefix && rp->codePrefix[0] ){
+   fprintf(out, "{%s", rp->codePrefix);
+   for(cp=rp->codePrefix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; }
+ }
+
  /* Generate code to do the reduce action */
  if( rp->code ){
    if( !lemp->nolinenosflag ){
@@ -3573,15 +3765,23 @@ PRIVATE void emit_code(
      tplt_linedir(out,rp->line,lemp->filename);
    }
    fprintf(out,"{%s",rp->code);
-   for(cp=rp->code; *cp; cp++){
-     if( *cp=='\n' ) (*lineno)++;
-   } /* End loop */
+   for(cp=rp->code; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; }
    fprintf(out,"}\n"); (*lineno)++;
    if( !lemp->nolinenosflag ){
      (*lineno)++;
      tplt_linedir(out,*lineno,lemp->outname);
    }
- } /* End if( rp->code ) */
+ }
+
+ /* Generate breakdown code that occurs after the #line directive */
+ if( rp->codeSuffix && rp->codeSuffix[0] ){
+   fprintf(out, "%s", rp->codeSuffix);
+   for(cp=rp->codeSuffix; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; }
+ }
+
+ if( rp->codePrefix ){
+   fprintf(out, "}\n"); (*lineno)++;
+ }
 
  return;
 }
@@ -3954,6 +4154,19 @@ void ReportTable(
   }
   free(ax);
 
+  /* Mark rules that are actually used for reduce actions after all
+  ** optimizations have been applied
+  */
+  for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE;
+  for(i=0; inxstate; i++){
+    struct action *ap;
+    for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){
+      if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){
+        ap->x.rp->doesReduce = i;
+      }
+    }
+  }
+
   /* Finish rendering the constants now that the action table has
   ** been computed */
   fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
@@ -4019,20 +4232,21 @@ void ReportTable(
   fprintf(out, "};\n"); lineno++;
 
   /* Output the yy_shift_ofst[] table */
-  fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++;
   n = lemp->nxstate;
   while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--;
-  fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++;
-  fprintf(out, "#define YY_SHIFT_MIN   (%d)\n", mnTknOfst); lineno++;
-  fprintf(out, "#define YY_SHIFT_MAX   (%d)\n", mxTknOfst); lineno++;
+  fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++;
+  fprintf(out, "#define YY_SHIFT_COUNT    (%d)\n", n-1); lineno++;
+  fprintf(out, "#define YY_SHIFT_MIN      (%d)\n", mnTknOfst); lineno++;
+  fprintf(out, "#define YY_SHIFT_MAX      (%d)\n", mxTknOfst); lineno++;
   fprintf(out, "static const %s yy_shift_ofst[] = {\n", 
-          minimum_size_type(mnTknOfst-1, mxTknOfst, &sz)); lineno++;
+       minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz));
+       lineno++;
   lemp->tablesize += n*sz;
   for(i=j=0; isorted[i];
     ofst = stp->iTknOfst;
-    if( ofst==NO_OFFSET ) ofst = mnTknOfst - 1;
+    if( ofst==NO_OFFSET ) ofst = lemp->nactiontab;
     if( j==0 ) fprintf(out," /* %5d */ ", i);
     fprintf(out, " %4d,", ofst);
     if( j==9 || i==n-1 ){
@@ -4122,7 +4336,7 @@ void ReportTable(
   ** when tracing REDUCE actions.
   */
   for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
-    assert( rp->index==i );
+    assert( rp->iRule==i );
     fprintf(out," /* %3d */ \"", i);
     writeRuleText(out, rp);
     fprintf(out,"\",\n"); lineno++;
@@ -4172,6 +4386,7 @@ void ReportTable(
   for(i=0; insymbol; i++){
     struct symbol *sp = lemp->symbols[i];
     if( sp==0 || sp->type==TERMINAL || sp->destructor==0 ) continue;
+    if( sp->destLineno<0 ) continue;  /* Already emitted */
     fprintf(out,"    case %d: /* %s */\n", sp->index, sp->name); lineno++;
 
     /* Combine duplicate destructors into a single case */
@@ -4182,7 +4397,7 @@ void ReportTable(
           && strcmp(sp->destructor,sp2->destructor)==0 ){
          fprintf(out,"    case %d: /* %s */\n",
                  sp2->index, sp2->name); lineno++;
-         sp2->destructor = 0;
+         sp2->destLineno = -1;  /* Avoid emitting this destructor again */
       }
     }
 
@@ -4206,38 +4421,51 @@ void ReportTable(
   tplt_xfer(lemp->name,in,out,&lineno);
 
   /* Generate code which execution during each REDUCE action */
+  i = 0;
   for(rp=lemp->rule; rp; rp=rp->next){
-    translate_code(lemp, rp);
+    i += translate_code(lemp, rp);
+  }
+  if( i ){
+    fprintf(out,"        YYMINORTYPE yylhsminor;\n"); lineno++;
   }
   /* First output rules other than the default: rule */
   for(rp=lemp->rule; rp; rp=rp->next){
     struct rule *rp2;               /* Other rules with the same action */
-    if( rp->code==0 ) continue;
-    if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */
-    fprintf(out,"      case %d: /* ", rp->index);
+    if( rp->codeEmitted ) continue;
+    if( rp->noCode ){
+      /* No C code actions, so this will be part of the "default:" rule */
+      continue;
+    }
+    fprintf(out,"      case %d: /* ", rp->iRule);
     writeRuleText(out, rp);
     fprintf(out, " */\n"); lineno++;
     for(rp2=rp->next; rp2; rp2=rp2->next){
-      if( rp2->code==rp->code ){
-        fprintf(out,"      case %d: /* ", rp2->index);
+      if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix
+             && rp2->codeSuffix==rp->codeSuffix ){
+        fprintf(out,"      case %d: /* ", rp2->iRule);
         writeRuleText(out, rp2);
-        fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->index); lineno++;
-        rp2->code = 0;
+        fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++;
+        rp2->codeEmitted = 1;
       }
     }
     emit_code(out,rp,lemp,&lineno);
     fprintf(out,"        break;\n"); lineno++;
-    rp->code = 0;
+    rp->codeEmitted = 1;
   }
   /* Finally, output the default: rule.  We choose as the default: all
   ** empty actions. */
   fprintf(out,"      default:\n"); lineno++;
   for(rp=lemp->rule; rp; rp=rp->next){
-    if( rp->code==0 ) continue;
-    assert( rp->code[0]=='\n' && rp->code[1]==0 );
-    fprintf(out,"      /* (%d) ", rp->index);
+    if( rp->codeEmitted ) continue;
+    assert( rp->noCode );
+    fprintf(out,"      /* (%d) ", rp->iRule);
     writeRuleText(out, rp);
-    fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->index); lineno++;
+    if( rp->doesReduce ){
+      fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++;
+    }else{
+      fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n",
+              rp->iRule); lineno++;
+    }
   }
   fprintf(out,"        break;\n"); lineno++;
   tplt_xfer(lemp->name,in,out,&lineno);
@@ -4308,7 +4536,7 @@ void ReportHeader(struct lemon *lemp)
 void CompressTables(struct lemon *lemp)
 {
   struct state *stp;
-  struct action *ap, *ap2;
+  struct action *ap, *ap2, *nextap;
   struct rule *rp, *rp2, *rbest;
   int nbest, n;
   int i;
@@ -4385,6 +4613,36 @@ void CompressTables(struct lemon *lemp)
       }
     }
   }
+
+  /* If a SHIFTREDUCE action specifies a rule that has a single RHS term
+  ** (meaning that the SHIFTREDUCE will land back in the state where it
+  ** started) and if there is no C-code associated with the reduce action,
+  ** then we can go ahead and convert the action to be the same as the
+  ** action for the RHS of the rule.
+  */
+  for(i=0; instate; i++){
+    stp = lemp->sorted[i];
+    for(ap=stp->ap; ap; ap=nextap){
+      nextap = ap->next;
+      if( ap->type!=SHIFTREDUCE ) continue;
+      rp = ap->x.rp;
+      if( rp->noCode==0 ) continue;
+      if( rp->nrhs!=1 ) continue;
+#if 1
+      /* Only apply this optimization to non-terminals.  It would be OK to
+      ** apply it to terminal symbols too, but that makes the parser tables
+      ** larger. */
+      if( ap->sp->indexnterminal ) continue;
+#endif
+      /* If we reach this point, it means the optimization can be applied */
+      nextap = ap;
+      for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){}
+      assert( ap2!=0 );
+      ap->spOpt = ap2->sp;
+      ap->type = ap2->type;
+      ap->x = ap2->x;
+    }
+  }
 }
 
 
diff --git a/tool/lempar.c b/tool/lempar.c
index 312507e5..5195d9c3 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -87,10 +87,6 @@
 %%
 /************* End control #defines *******************************************/
 
-/* The yyzerominor constant is used to initialize instances of
-** YYMINORTYPE objects to zero. */
-static const YYMINORTYPE yyzerominor = { 0 };
-
 /* Define the yytestcase() macro to be a no-op if is not already defined
 ** otherwise.
 **
@@ -120,7 +116,7 @@ static const YYMINORTYPE yyzerominor = { 0 };
 **
 **   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
 **     and YY_MAX_REDUCE
-
+**
 **   N == YY_ERROR_ACTION               A syntax error has occurred.
 **
 **   N == YY_ACCEPT_ACTION              The parser accepts its input.
@@ -129,16 +125,20 @@ static const YYMINORTYPE yyzerominor = { 0 };
 **                                      slots in the yy_action[] table.
 **
 ** The action table is constructed as a single large table named yy_action[].
-** Given state S and lookahead X, the action is computed as
+** Given state S and lookahead X, the action is computed as either:
 **
-**      yy_action[ yy_shift_ofst[S] + X ]
+**    (A)   N = yy_action[ yy_shift_ofst[S] + X ]
+**    (B)   N = yy_default[S]
 **
-** If the index value yy_shift_ofst[S]+X is out of range or if the value
-** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S]
-** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table
-** and that yy_default[S] should be used instead.  
+** The (A) formula is preferred.  The B formula is used instead if:
+**    (1)  The yy_shift_ofst[S]+X value is out of range, or
+**    (2)  yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or
+**    (3)  yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT.
+** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that
+** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X.
+** Hence only tests (1) and (2) need to be evaluated.)
 **
-** The formula above is for computing the action when the lookahead is
+** The formulas above are for computing the action when the lookahead is
 ** a terminal symbol.  If the lookahead is a non-terminal (as occurs after
 ** a reduce action) then the yy_reduce_ofst[] array is used in place of
 ** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of
@@ -207,15 +207,18 @@ typedef struct yyStackEntry yyStackEntry;
 /* The state of the parser is completely contained in an instance of
 ** the following structure */
 struct yyParser {
-  int yyidx;                    /* Index of top element in stack */
+  yyStackEntry *yytos;          /* Pointer to top element of the stack */
 #ifdef YYTRACKMAXSTACKDEPTH
-  int yyidxMax;                 /* Maximum value of yyidx */
+  int yyhwm;                    /* High-water mark of the stack */
 #endif
+#ifndef YYNOERRORRECOVERY
   int yyerrcnt;                 /* Shifts left before out of the error */
+#endif
   ParseARG_SDECL                /* A place to hold %extra_argument */
 #if YYSTACKDEPTH<=0
   int yystksz;                  /* Current side of the stack */
   yyStackEntry *yystack;        /* The parser's stack */
+  yyStackEntry yystk0;          /* First stack entry */
 #else
   yyStackEntry yystack[YYSTACKDEPTH];  /* The parser's stack */
 #endif
@@ -273,24 +276,34 @@ static const char *const yyRuleName[] = {
 
 #if YYSTACKDEPTH<=0
 /*
-** Try to increase the size of the parser stack.
+** Try to increase the size of the parser stack.  Return the number
+** of errors.  Return 0 on success.
 */
-static void yyGrowStack(yyParser *p){
+static int yyGrowStack(yyParser *p){
   int newSize;
+  int idx;
   yyStackEntry *pNew;
 
   newSize = p->yystksz*2 + 100;
-  pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
+  if( p->yystack==&p->yystk0 ){
+    pNew = malloc(newSize*sizeof(pNew[0]));
+    if( pNew ) pNew[0] = p->yystk0;
+  }else{
+    pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  }
   if( pNew ){
     p->yystack = pNew;
-    p->yystksz = newSize;
+    p->yytos = &p->yystack[idx];
 #ifndef NDEBUG
     if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sStack grows to %d entries!\n",
-              yyTracePrompt, p->yystksz);
+      fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
+              yyTracePrompt, p->yystksz, newSize);
     }
 #endif
+    p->yystksz = newSize;
   }
+  return pNew==0; 
 }
 #endif
 
@@ -319,15 +332,24 @@ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
   yyParser *pParser;
   pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
   if( pParser ){
-    pParser->yyidx = -1;
 #ifdef YYTRACKMAXSTACKDEPTH
-    pParser->yyidxMax = 0;
+    pParser->yyhwm = 0;
 #endif
 #if YYSTACKDEPTH<=0
+    pParser->yytos = NULL;
     pParser->yystack = NULL;
     pParser->yystksz = 0;
-    yyGrowStack(pParser);
+    if( yyGrowStack(pParser) ){
+      pParser->yystack = &pParser->yystk0;
+      pParser->yystksz = 1;
+    }
 #endif
+#ifndef YYNOERRORRECOVERY
+    pParser->yyerrcnt = -1;
+#endif
+    pParser->yytos = pParser->yystack;
+    pParser->yystack[0].stateno = 0;
+    pParser->yystack[0].major = 0;
   }
   return pParser;
 }
@@ -371,8 +393,9 @@ static void yy_destructor(
 */
 static void yy_pop_parser_stack(yyParser *pParser){
   yyStackEntry *yytos;
-  assert( pParser->yyidx>=0 );
-  yytos = &pParser->yystack[pParser->yyidx--];
+  assert( pParser->yytos!=0 );
+  assert( pParser->yytos > pParser->yystack );
+  yytos = pParser->yytos--;
 #ifndef NDEBUG
   if( yyTraceFILE ){
     fprintf(yyTraceFILE,"%sPopping %s\n",
@@ -399,9 +422,9 @@ void ParseFree(
 #ifndef YYPARSEFREENEVERNULL
   if( pParser==0 ) return;
 #endif
-  while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
+  while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
 #if YYSTACKDEPTH<=0
-  free(pParser->yystack);
+  if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
 #endif
   (*freeProc)((void*)pParser);
 }
@@ -412,7 +435,7 @@ void ParseFree(
 #ifdef YYTRACKMAXSTACKDEPTH
 int ParseStackPeak(void *p){
   yyParser *pParser = (yyParser*)p;
-  return pParser->yyidxMax;
+  return pParser->yyhwm;
 }
 #endif
 
@@ -420,61 +443,58 @@ int ParseStackPeak(void *p){
 ** Find the appropriate action for a parser given the terminal
 ** look-ahead token iLookAhead.
 */
-static int yy_find_shift_action(
+static unsigned int yy_find_shift_action(
   yyParser *pParser,        /* The parser */
   YYCODETYPE iLookAhead     /* The look-ahead token */
 ){
   int i;
-  int stateno = pParser->yystack[pParser->yyidx].stateno;
+  int stateno = pParser->yytos->stateno;
  
   if( stateno>=YY_MIN_REDUCE ) return stateno;
   assert( stateno <= YY_SHIFT_COUNT );
   do{
     i = yy_shift_ofst[stateno];
-    if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
     assert( iLookAhead!=YYNOCODE );
     i += iLookAhead;
     if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
-      if( iLookAhead>0 ){
 #ifdef YYFALLBACK
-        YYCODETYPE iFallback;            /* Fallback token */
-        if( iLookAhead %s\n",
-               yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
-          }
-#endif
-          assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
-          iLookAhead = iFallback;
-          continue;
+        if( yyTraceFILE ){
+          fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
+             yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
         }
+#endif
+        assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
+        iLookAhead = iFallback;
+        continue;
+      }
 #endif
 #ifdef YYWILDCARD
-        {
-          int j = i - iLookAhead + YYWILDCARD;
-          if( 
+      {
+        int j = i - iLookAhead + YYWILDCARD;
+        if( 
 #if YY_SHIFT_MIN+YYWILDCARD<0
-            j>=0 &&
+          j>=0 &&
 #endif
 #if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT
-            j0
+        ){
 #ifndef NDEBUG
-            if( yyTraceFILE ){
-              fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
-                 yyTracePrompt, yyTokenName[iLookAhead],
-                 yyTokenName[YYWILDCARD]);
-            }
-#endif /* NDEBUG */
-            return yy_action[j];
+          if( yyTraceFILE ){
+            fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
+               yyTracePrompt, yyTokenName[iLookAhead],
+               yyTokenName[YYWILDCARD]);
           }
+#endif /* NDEBUG */
+          return yy_action[j];
         }
-#endif /* YYWILDCARD */
       }
+#endif /* YYWILDCARD */
       return yy_default[stateno];
     }else{
       return yy_action[i];
@@ -516,15 +536,15 @@ static int yy_find_reduce_action(
 /*
 ** The following routine is called if the stack overflows.
 */
-static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
+static void yyStackOverflow(yyParser *yypParser){
    ParseARG_FETCH;
-   yypParser->yyidx--;
+   yypParser->yytos--;
 #ifndef NDEBUG
    if( yyTraceFILE ){
      fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
    }
 #endif
-   while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+   while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
    /* Here code is inserted which will execute if the parser
    ** stack every overflows */
 /******** Begin %stack_overflow code ******************************************/
@@ -541,11 +561,11 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){
   if( yyTraceFILE ){
     if( yyNewStateyystack[yypParser->yyidx].major],
+         yyTracePrompt,yyTokenName[yypParser->yytos->major],
          yyNewState);
     }else{
       fprintf(yyTraceFILE,"%sShift '%s'\n",
-         yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]);
+         yyTracePrompt,yyTokenName[yypParser->yytos->major]);
     }
   }
 }
@@ -560,33 +580,36 @@ static void yy_shift(
   yyParser *yypParser,          /* The parser to be shifted */
   int yyNewState,               /* The new state to shift in */
   int yyMajor,                  /* The major token to shift in */
-  YYMINORTYPE *yypMinor         /* Pointer to the minor token to shift in */
+  ParseTOKENTYPE yyMinor        /* The minor token to shift in */
 ){
   yyStackEntry *yytos;
-  yypParser->yyidx++;
+  yypParser->yytos++;
 #ifdef YYTRACKMAXSTACKDEPTH
-  if( yypParser->yyidx>yypParser->yyidxMax ){
-    yypParser->yyidxMax = yypParser->yyidx;
+  if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+    yypParser->yyhwm++;
+    assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
   }
 #endif
 #if YYSTACKDEPTH>0 
-  if( yypParser->yyidx>=YYSTACKDEPTH ){
-    yyStackOverflow(yypParser, yypMinor);
+  if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){
+    yyStackOverflow(yypParser);
     return;
   }
 #else
-  if( yypParser->yyidx>=yypParser->yystksz ){
-    yyGrowStack(yypParser);
-    if( yypParser->yyidx>=yypParser->yystksz ){
-      yyStackOverflow(yypParser, yypMinor);
+  if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
+    if( yyGrowStack(yypParser) ){
+      yyStackOverflow(yypParser);
       return;
     }
   }
 #endif
-  yytos = &yypParser->yystack[yypParser->yyidx];
+  if( yyNewState > YY_MAX_SHIFT ){
+    yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+  }
+  yytos = yypParser->yytos;
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
-  yytos->minor = *yypMinor;
+  yytos->minor.yy0 = yyMinor;
   yyTraceShift(yypParser, yyNewState);
 }
 
@@ -608,24 +631,47 @@ static void yy_accept(yyParser*);  /* Forward Declaration */
 */
 static void yy_reduce(
   yyParser *yypParser,         /* The parser */
-  int yyruleno                 /* Number of the rule by which to reduce */
+  unsigned int yyruleno        /* Number of the rule by which to reduce */
 ){
   int yygoto;                     /* The next state */
   int yyact;                      /* The next action */
-  YYMINORTYPE yygotominor;        /* The LHS of the rule reduced */
   yyStackEntry *yymsp;            /* The top of the parser's stack */
   int yysize;                     /* Amount to pop the stack */
   ParseARG_FETCH;
-  yymsp = &yypParser->yystack[yypParser->yyidx];
+  yymsp = yypParser->yytos;
 #ifndef NDEBUG
-  if( yyTraceFILE && yyruleno>=0 
-        && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
+  if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
     yysize = yyRuleInfo[yyruleno].nrhs;
     fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt,
       yyRuleName[yyruleno], yymsp[-yysize].stateno);
   }
 #endif /* NDEBUG */
-  yygotominor = yyzerominor;
+
+  /* Check that the stack is large enough to grow by a single entry
+  ** if the RHS of the rule is empty.  This ensures that there is room
+  ** enough on the stack to push the LHS value */
+  if( yyRuleInfo[yyruleno].nrhs==0 ){
+#ifdef YYTRACKMAXSTACKDEPTH
+    if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+      yypParser->yyhwm++;
+      assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
+    }
+#endif
+#if YYSTACKDEPTH>0 
+    if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){
+      yyStackOverflow(yypParser);
+      return;
+    }
+#else
+    if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
+      if( yyGrowStack(yypParser) ){
+        yyStackOverflow(yypParser);
+        return;
+      }
+      yymsp = yypParser->yytos;
+    }
+#endif
+  }
 
   switch( yyruleno ){
   /* Beginning here are the reduction cases.  A typical example
@@ -640,29 +686,22 @@ static void yy_reduce(
 %%
 /********** End reduce actions ************************************************/
   };
-  assert( yyruleno>=0 && yyrulenoyyidx -= yysize;
   yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
   if( yyact <= YY_MAX_SHIFTREDUCE ){
-    if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-    /* If the reduce action popped at least
-    ** one element off the stack, then we can push the new element back
-    ** onto the stack here, and skip the stack overflow test in yy_shift().
-    ** That gives a significant speed improvement. */
-    if( yysize ){
-      yypParser->yyidx++;
-      yymsp -= yysize-1;
-      yymsp->stateno = (YYACTIONTYPE)yyact;
-      yymsp->major = (YYCODETYPE)yygoto;
-      yymsp->minor = yygotominor;
-      yyTraceShift(yypParser, yyact);
-    }else{
-      yy_shift(yypParser,yyact,yygoto,&yygotominor);
+    if( yyact>YY_MAX_SHIFT ){
+      yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
     }
+    yymsp -= yysize-1;
+    yypParser->yytos = yymsp;
+    yymsp->stateno = (YYACTIONTYPE)yyact;
+    yymsp->major = (YYCODETYPE)yygoto;
+    yyTraceShift(yypParser, yyact);
   }else{
     assert( yyact == YY_ACCEPT_ACTION );
+    yypParser->yytos -= yysize;
     yy_accept(yypParser);
   }
 }
@@ -680,7 +719,7 @@ static void yy_parse_failed(
     fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
   }
 #endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+  while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
   /* Here code is inserted which will be executed whenever the
   ** parser fails */
 /************ Begin %parse_failure code ***************************************/
@@ -696,10 +735,10 @@ static void yy_parse_failed(
 static void yy_syntax_error(
   yyParser *yypParser,           /* The parser */
   int yymajor,                   /* The major type of the error token */
-  YYMINORTYPE yyminor            /* The minor type of the error token */
+  ParseTOKENTYPE yyminor         /* The minor type of the error token */
 ){
   ParseARG_FETCH;
-#define TOKEN (yyminor.yy0)
+#define TOKEN yyminor
 /************ Begin %syntax_error code ****************************************/
 %%
 /************ End %syntax_error code ******************************************/
@@ -718,7 +757,10 @@ static void yy_accept(
     fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
   }
 #endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+#ifndef YYNOERRORRECOVERY
+  yypParser->yyerrcnt = -1;
+#endif
+  assert( yypParser->yytos==yypParser->yystack );
   /* Here code is inserted which will be executed whenever the
   ** parser accepts */
 /*********** Begin %parse_accept code *****************************************/
@@ -753,7 +795,7 @@ void Parse(
   ParseARG_PDECL               /* Optional %extra_argument parameter */
 ){
   YYMINORTYPE yyminorunion;
-  int yyact;            /* The parser action. */
+  unsigned int yyact;   /* The parser action. */
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   int yyendofinput;     /* True if we are at the end of input */
 #endif
@@ -762,29 +804,8 @@ void Parse(
 #endif
   yyParser *yypParser;  /* The parser */
 
-  /* (re)initialize the parser, if necessary */
   yypParser = (yyParser*)yyp;
-  if( yypParser->yyidx<0 ){
-#if YYSTACKDEPTH<=0
-    if( yypParser->yystksz <=0 ){
-      /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/
-      yyminorunion = yyzerominor;
-      yyStackOverflow(yypParser, &yyminorunion);
-      return;
-    }
-#endif
-    yypParser->yyidx = 0;
-    yypParser->yyerrcnt = -1;
-    yypParser->yystack[0].stateno = 0;
-    yypParser->yystack[0].major = 0;
-#ifndef NDEBUG
-    if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n",
-              yyTracePrompt);
-    }
-#endif
-  }
-  yyminorunion.yy0 = yyminor;
+  assert( yypParser->yytos!=0 );
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   yyendofinput = (yymajor==0);
 #endif
@@ -799,14 +820,16 @@ void Parse(
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
     if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-      yy_shift(yypParser,yyact,yymajor,&yyminorunion);
+      yy_shift(yypParser,yyact,yymajor,yyminor);
+#ifndef YYNOERRORRECOVERY
       yypParser->yyerrcnt--;
+#endif
       yymajor = YYNOCODE;
     }else if( yyact <= YY_MAX_REDUCE ){
       yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
     }else{
       assert( yyact == YY_ERROR_ACTION );
+      yyminorunion.yy0 = yyminor;
 #ifdef YYERRORSYMBOL
       int yymx;
 #endif
@@ -836,9 +859,9 @@ void Parse(
       **
       */
       if( yypParser->yyerrcnt<0 ){
-        yy_syntax_error(yypParser,yymajor,yyminorunion);
+        yy_syntax_error(yypParser,yymajor,yyminor);
       }
-      yymx = yypParser->yystack[yypParser->yyidx].major;
+      yymx = yypParser->yytos->major;
       if( yymx==YYERRORSYMBOL || yyerrorhit ){
 #ifndef NDEBUG
         if( yyTraceFILE ){
@@ -846,26 +869,26 @@ void Parse(
              yyTracePrompt,yyTokenName[yymajor]);
         }
 #endif
-        yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion);
+        yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion);
         yymajor = YYNOCODE;
       }else{
-         while(
-          yypParser->yyidx >= 0 &&
-          yymx != YYERRORSYMBOL &&
-          (yyact = yy_find_reduce_action(
-                        yypParser->yystack[yypParser->yyidx].stateno,
+        while( yypParser->yytos >= yypParser->yystack
+            && yymx != YYERRORSYMBOL
+            && (yyact = yy_find_reduce_action(
+                        yypParser->yytos->stateno,
                         YYERRORSYMBOL)) >= YY_MIN_REDUCE
         ){
           yy_pop_parser_stack(yypParser);
         }
-        if( yypParser->yyidx < 0 || yymajor==0 ){
+        if( yypParser->yytos < yypParser->yystack || yymajor==0 ){
           yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
           yy_parse_failed(yypParser);
+#ifndef YYNOERRORRECOVERY
+          yypParser->yyerrcnt = -1;
+#endif
           yymajor = YYNOCODE;
         }else if( yymx!=YYERRORSYMBOL ){
-          YYMINORTYPE u2;
-          u2.YYERRSYMDT = 0;
-          yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2);
+          yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor);
         }
       }
       yypParser->yyerrcnt = 3;
@@ -878,7 +901,7 @@ void Parse(
       ** Applications can set this macro (for example inside %include) if
       ** they intend to abandon the parse upon the first syntax error seen.
       */
-      yy_syntax_error(yypParser,yymajor,yyminorunion);
+      yy_syntax_error(yypParser,yymajor, yyminor);
       yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
       yymajor = YYNOCODE;
       
@@ -893,24 +916,29 @@ void Parse(
       ** three input tokens have been successfully shifted.
       */
       if( yypParser->yyerrcnt<=0 ){
-        yy_syntax_error(yypParser,yymajor,yyminorunion);
+        yy_syntax_error(yypParser,yymajor, yyminor);
       }
       yypParser->yyerrcnt = 3;
       yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
       if( yyendofinput ){
         yy_parse_failed(yypParser);
+#ifndef YYNOERRORRECOVERY
+        yypParser->yyerrcnt = -1;
+#endif
       }
       yymajor = YYNOCODE;
 #endif
     }
-  }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
+  }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack );
 #ifndef NDEBUG
   if( yyTraceFILE ){
-    int i;
+    yyStackEntry *i;
+    char cDiv = '[';
     fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
-    for(i=1; i<=yypParser->yyidx; i++)
-      fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', 
-              yyTokenName[yypParser->yystack[i].major]);
+    for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){
+      fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]);
+      cDiv = ' ';
+    }
     fprintf(yyTraceFILE,"]\n");
   }
 #endif
diff --git a/tool/libvers.c b/tool/libvers.c
new file mode 100644
index 00000000..6911dbd0
--- /dev/null
+++ b/tool/libvers.c
@@ -0,0 +1,15 @@
+/*
+** Compile this program against an SQLite library of unknown version
+** and then run this program, and it will print out the SQLite version
+** information.
+*/
+#include 
+
+extern const char *sqlite3_libversion(void);
+extern const char *sqlite3_sourceid(void);
+
+int main(int argc, char **argv){
+  printf("SQLite version %s\n", sqlite3_libversion());
+  printf("SQLite source  %s\n", sqlite3_sourceid());
+  return 0;
+}
diff --git a/tool/logest.c b/tool/logest.c
index 347fa68a..e936e02c 100644
--- a/tool/logest.c
+++ b/tool/logest.c
@@ -147,7 +147,7 @@ int main(int argc, char **argv){
     }else if( strcmp(z,"inv")==0 ){
       if( n>0 ) a[n-1] = -a[n-1];
     }else if( z[0]=='^' ){
-      a[n++] = atoi(z+1);
+      a[n++] = (LogEst)atoi(z+1);
     }else if( isInteger(z) ){
       a[n++] = logEstFromInteger(atoi(z));
     }else if( isFloat(z) && z[0]!='-' ){
diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh
index 75a0b09a..4864ee85 100644
--- a/tool/mkautoconfamal.sh
+++ b/tool/mkautoconfamal.sh
@@ -53,6 +53,7 @@ cp $TOP/sqlite3.1         $TMPSPACE
 cp $TOP/sqlite3.pc.in     $TMPSPACE
 cp $TOP/src/shell.c       $TMPSPACE
 cp $TOP/src/sqlite3.rc    $TMPSPACE
+cp $TOP/tool/Replace.cs   $TMPSPACE
 
 cat $TMPSPACE/configure.ac |
 sed "s/--SQLITE-VERSION--/$VERSION/" > $TMPSPACE/tmp
diff --git a/tool/mkmsvcmin.tcl b/tool/mkmsvcmin.tcl
index da0cd628..88e7f918 100644
--- a/tool/mkmsvcmin.tcl
+++ b/tool/mkmsvcmin.tcl
@@ -72,6 +72,21 @@ $(LIBRESOBJS):	$(TOP)\sqlite3.rc rcver.vc $(SQLITE3H)
 	$(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc
 }]]
 
+#
+# NOTE: This block is used to replace the section marked <> in
+#       the Makefile, if it exists.
+#
+set blocks(2) [string trimleft [string map [list \\\\ \\] {
+Replace.exe:
+	$(CSC) /target:exe $(TOP)\Replace.cs
+
+sqlite3.def:	Replace.exe $(LIBOBJ)
+	echo EXPORTS > sqlite3.def
+	dumpbin /all $(LIBOBJ) \\
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\
+		| sort >> sqlite3.def
+}]]
+
 set data "#### DO NOT EDIT ####\n"
 append data "# This makefile is automatically "
 append data "generated from the [file tail $fromFileName] at\n"
@@ -90,8 +105,7 @@ foreach i [lsort -integer [array names blocks]] {
 }
 
 set data [string map [list " -I\$(TOP)\\src" ""] $data]
-set data [string map [list " /DEF:sqlite3.def" ""] $data]
-set data [string map [list " sqlite3.def" ""] $data]
+set data [string map [list " libsqlite3.lib" ""] $data]
 set data [string map [list " \$(ALL_TCL_TARGETS)" ""] $data]
 set data [string map [list "\$(TOP)\\src\\" "\$(TOP)\\"] $data]
 
diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl
index 053c7f89..3c2a5978 100644
--- a/tool/mkopcodeh.tcl
+++ b/tool/mkopcodeh.tcl
@@ -20,8 +20,7 @@
 # during code generation, we need to generate corresponding opcodes like
 # OP_Add and OP_Divide.  By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide,
 # code to translate from one to the other is avoided.  This makes the
-# code generator run (infinitesimally) faster and more importantly it makes
-# the library footprint smaller.
+# code generator smaller and faster.
 #
 # This script also scans for lines of the form:
 #
@@ -159,7 +158,29 @@ for {set i 0} {$i<$nOp} {incr i} {
   }
 }
 
-# Generate the numeric values for remaining opcodes
+# Assign the next group of values to JUMP opcodes
+#
+for {set i 0} {$i<$nOp} {incr i} {
+  set name $order($i)
+  if {$op($name)>=0} continue
+  if {!$jump($name)} continue
+  incr cnt
+  while {[info exists used($cnt)]} {incr cnt}
+  set op($name) $cnt
+  set used($cnt) 1
+  set def($cnt) $name
+}
+
+# Find the numeric value for the largest JUMP opcode
+#
+set mxJump -1
+for {set i 0} {$i<$nOp} {incr i} {
+  set name $order($i)
+  if {$jump($name) && $op($name)>$mxJump} {set mxJump $op($name)}
+}
+
+
+# Generate the numeric values for all remaining opcodes
 #
 for {set i 0} {$i<$nOp} {incr i} {
   set name $order($i)
@@ -232,3 +253,11 @@ for {set i 0} {$i<=$max} {incr i} {
   }
 }
 puts "\175"
+puts ""
+puts "/* The sqlite3P2Values() routine is able to run faster if it knows"
+puts "** the value of the largest JUMP opcode.  The smaller the maximum"
+puts "** JUMP opcode the better, so the mkopcodeh.tcl script that"
+puts "** generated this include file strives to group all JUMP opcodes"
+puts "** together near the beginning of the list."
+puts "*/"
+puts "#define SQLITE_MX_JUMP_OPCODE  $mxJump  /* Maximum JUMP opcode */"
diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl
index 601b8cce..a16b0591 100644
--- a/tool/mksqlite3c-noext.tcl
+++ b/tool/mksqlite3c-noext.tcl
@@ -1,7 +1,7 @@
 #!/usr/bin/tclsh
 #
 # To build a single huge source file holding all of SQLite (or at
-# least the core components - the test harness, shell, and TCL 
+# least the core components - the test harness, shell, and TCL
 # interface are omitted.) first do
 #
 #      make target_source
@@ -11,7 +11,7 @@
 # there and will not work if they are not found.)  There are a few
 # generated C code files that are also added to the tsrc directory.
 # For example, the "parse.c" and "parse.h" files to implement the
-# the parser are derived from "parse.y" using lemon.  And the 
+# the parser are derived from "parse.y" using lemon.  And the
 # "keywordhash.h" files is generated by a program named "mkkeywordhash".
 #
 # After the "tsrc" directory has been created and populated, run
@@ -26,15 +26,20 @@
 # from in this file.  The version number is needed to generate the header
 # comment of the amalgamation.
 #
-if {[lsearch $argv --nostatic]>=0} {
-  set addstatic 0
-} else {
-  set addstatic 1
-}
-if {[lsearch $argv --linemacros]>=0} {
-  set linemacros 1
-} else {
-  set linemacros 0
+set addstatic 1
+set linemacros 0
+set useapicall 0
+for {set i 0} {$i<[llength $argv]} {incr i} {
+  set x [lindex $argv $i]
+  if {[regexp {^-+nostatic$} $x]} {
+    set addstatic 0
+  } elseif {[regexp {^-+linemacros} $x]} {
+    set linemacros 1
+  } elseif {[regexp {^-+useapicall} $x]} {
+    set useapicall 1
+  } else {
+    error "unknown command-line option: $x"
+  }
 }
 set in [open tsrc/sqlite3.h]
 set cnt 0
@@ -57,7 +62,7 @@ set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
 puts $out [subst \
 {/******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version $VERSION.  By combining all the individual C code files into this 
+** version $VERSION.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -66,7 +71,7 @@ puts $out [subst \
 **
 ** This file is all you need to compile SQLite.  To use SQLite in other
 ** programs, you need this file and the "sqlite3.h" header file that defines
-** the programming interface to the SQLite library.  (If you do not have 
+** the programming interface to the SQLite library.  (If you do not have
 ** the "sqlite3.h" header file at hand, you will find a copy embedded within
 ** the text of this file.  Search for "Begin file sqlite3.h" to find the start
 ** of the embedded sqlite3.h header file.) Additional code files may be needed
@@ -83,7 +88,7 @@ if {$addstatic} {
 #endif}
 }
 
-# These are the header files used by SQLite.  The first time any of these 
+# These are the header files used by SQLite.  The first time any of these
 # files are seen in a #include statement in the C code, include the complete
 # text of the file in-line.  The file only needs to be included once.
 #
@@ -104,8 +109,8 @@ foreach hdr {
    parse.h
    pcache.h
    pragma.h
-   sqlite3ext.h
    sqlite3.h
+   sqlite3ext.h
    sqliteicu.h
    sqliteInt.h
    sqliteLimit.h
@@ -155,7 +160,8 @@ proc section_comment {text} {
 # process them appropriately.
 #
 proc copy_file {filename} {
-  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
+  global seen_hdr available_hdr varonly_hdr cdecllist out
+  global addstatic linemacros useapicall
   set ln 0
   set tail [file tail $filename]
   section_comment "Begin file $tail"
@@ -203,7 +209,8 @@ proc copy_file {filename} {
       puts $out "#if 0"
     } elseif {!$linemacros && [regexp {^#line} $line]} {
       # Skip #line directives.
-    } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {
+    } elseif {$addstatic
+               && ![regexp {^(static|typedef|SQLITE_PRIVATE)} $line]} {
       # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before
       # functions if this header file does not need it.
       if {![info exists varonly_hdr($tail)]
@@ -211,18 +218,20 @@ proc copy_file {filename} {
         regsub {^SQLITE_API } $line {} line
         # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
         # so that linkage can be modified at compile-time.
-        if {[regexp {^sqlite3_} $funcname]} {
+        if {[regexp {^sqlite3[a-z]*_} $funcname]} {
           set line SQLITE_API
           append line " " [string trim $rettype]
           if {[string index $rettype end] ne "*"} {
             append line " "
           }
-          if {[lsearch -exact $cdecllist $funcname] >= 0} {
-            append line SQLITE_CDECL
-          } else {
-            append line SQLITE_STDCALL
+          if {$useapicall} {
+            if {[lsearch -exact $cdecllist $funcname] >= 0} {
+              append line SQLITE_CDECL " "
+            } else {
+              append line SQLITE_APICALL " "
+            }
           }
-          append line " " $funcname $rest
+          append line $funcname $rest
           puts $out $line
         } else {
           puts $out "SQLITE_PRIVATE $line"
@@ -285,6 +294,7 @@ foreach file {
    mutex_w32.c
    malloc.c
    printf.c
+   treeview.c
    random.c
    threads.c
    utf.c
@@ -313,7 +323,6 @@ foreach file {
    vdbe.c
    vdbeblob.c
    vdbesort.c
-   journal.c
    memjournal.c
 
    walker.c
@@ -339,6 +348,8 @@ foreach file {
    update.c
    vacuum.c
    vtab.c
+   wherecode.c
+   whereexpr.c
    where.c
 
    parse.c
diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl
index 23241e27..55179c4b 100644
--- a/tool/mksqlite3c.tcl
+++ b/tool/mksqlite3c.tcl
@@ -1,7 +1,7 @@
 #!/usr/bin/tclsh
 #
 # To build a single huge source file holding all of SQLite (or at
-# least the core components - the test harness, shell, and TCL 
+# least the core components - the test harness, shell, and TCL
 # interface are omitted.) first do
 #
 #      make target_source
@@ -11,7 +11,7 @@
 # there and will not work if they are not found.)  There are a few
 # generated C code files that are also added to the tsrc directory.
 # For example, the "parse.c" and "parse.h" files to implement the
-# the parser are derived from "parse.y" using lemon.  And the 
+# the parser are derived from "parse.y" using lemon.  And the
 # "keywordhash.h" files is generated by a program named "mkkeywordhash".
 #
 # After the "tsrc" directory has been created and populated, run
@@ -28,12 +28,15 @@
 #
 set addstatic 1
 set linemacros 0
+set useapicall 0
 for {set i 0} {$i<[llength $argv]} {incr i} {
   set x [lindex $argv $i]
   if {[regexp {^-+nostatic$} $x]} {
     set addstatic 0
   } elseif {[regexp {^-+linemacros} $x]} {
     set linemacros 1
+  } elseif {[regexp {^-+useapicall} $x]} {
+    set useapicall 1
   } else {
     error "unknown command-line option: $x"
   }
@@ -59,7 +62,7 @@ set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
 puts $out [subst \
 {/******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version $VERSION.  By combining all the individual C code files into this 
+** version $VERSION.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -68,7 +71,7 @@ puts $out [subst \
 **
 ** This file is all you need to compile SQLite.  To use SQLite in other
 ** programs, you need this file and the "sqlite3.h" header file that defines
-** the programming interface to the SQLite library.  (If you do not have 
+** the programming interface to the SQLite library.  (If you do not have
 ** the "sqlite3.h" header file at hand, you will find a copy embedded within
 ** the text of this file.  Search for "Begin file sqlite3.h" to find the start
 ** of the embedded sqlite3.h header file.) Additional code files may be needed
@@ -85,7 +88,7 @@ if {$addstatic} {
 #endif}
 }
 
-# These are the header files used by SQLite.  The first time any of these 
+# These are the header files used by SQLite.  The first time any of these
 # files are seen in a #include statement in the C code, include the complete
 # text of the file in-line.  The file only needs to be included once.
 #
@@ -111,6 +114,7 @@ foreach hdr {
    pcache.h
    pragma.h
    rtree.h
+   sqlite3session.h
    sqlite3.h
    sqlite3ext.h
    sqlite3rbu.h
@@ -126,6 +130,7 @@ foreach hdr {
   set available_hdr($hdr) 1
 }
 set available_hdr(sqliteInt.h) 0
+set available_hdr(sqlite3session.h) 0
 
 # These headers should be copied into the amalgamation without modifying any
 # of their function declarations or definitions.
@@ -163,7 +168,8 @@ proc section_comment {text} {
 # process them appropriately.
 #
 proc copy_file {filename} {
-  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
+  global seen_hdr available_hdr varonly_hdr cdecllist out
+  global addstatic linemacros useapicall
   set ln 0
   set tail [file tail $filename]
   section_comment "Begin file $tail"
@@ -220,18 +226,20 @@ proc copy_file {filename} {
         regsub {^SQLITE_API } $line {} line
         # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
         # so that linkage can be modified at compile-time.
-        if {[regexp {^sqlite3(_|rbu_)} $funcname]} {
+        if {[regexp {^sqlite3[a-z]*_} $funcname]} {
           set line SQLITE_API
           append line " " [string trim $rettype]
           if {[string index $rettype end] ne "*"} {
             append line " "
           }
-          if {[lsearch -exact $cdecllist $funcname] >= 0} {
-            append line SQLITE_CDECL
-          } else {
-            append line SQLITE_STDCALL
+          if {$useapicall} {
+            if {[lsearch -exact $cdecllist $funcname] >= 0} {
+              append line SQLITE_CDECL " "
+            } else {
+              append line SQLITE_APICALL " "
+            }
           }
-          append line " " $funcname $rest
+          append line $funcname $rest
           puts $out $line
         } else {
           puts $out "SQLITE_PRIVATE $line"
@@ -323,7 +331,6 @@ foreach file {
    vdbe.c
    vdbeblob.c
    vdbesort.c
-   journal.c
    memjournal.c
 
    walker.c
@@ -379,6 +386,7 @@ foreach file {
    fts3_icu.c
    sqlite3rbu.c
    dbstat.c
+   sqlite3session.c
    json1.c
    fts5.c
 } {
diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl
index 3f59aef4..4f8a1fd4 100644
--- a/tool/mksqlite3h.tcl
+++ b/tool/mksqlite3h.tcl
@@ -10,17 +10,19 @@
 #
 # Run this script by specifying the root directory of the source tree
 # on the command-line.
-# 
+#
 # This script performs processing on src/sqlite.h.in. It:
 #
 #   1) Adds SQLITE_EXTERN in front of the declaration of global variables,
 #   2) Adds SQLITE_API in front of the declaration of API functions,
-#   3) Replaces the string --VERS-- with the current library version, 
+#   3) Replaces the string --VERS-- with the current library version,
 #      formatted as a string (e.g. "3.6.17"), and
 #   4) Replaces the string --VERSION-NUMBER-- with current library version,
 #      formatted as an integer (e.g. "3006017").
-#   5) Replaces the string --SOURCE-ID-- with the date and time and sha1 
+#   5) Replaces the string --SOURCE-ID-- with the date and time and sha1
 #      hash of the fossil-scm manifest for the source tree.
+#   6) Adds the SQLITE_CALLBACK calling convention macro in front of all
+#      callback declarations.
 #
 # This script outputs to stdout.
 #
@@ -34,6 +36,14 @@
 #
 set TOP [lindex $argv 0]
 
+# Enable use of SQLITE_APICALL macros at the right points?
+#
+set useapicall 0
+
+if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} {
+  set useapicall 1
+}
+
 # Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file.
 #
 set in [open $TOP/VERSION]
@@ -71,6 +81,7 @@ fconfigure stdout -translation lf
 set filelist [subst {
   $TOP/src/sqlite.h.in
   $TOP/ext/rtree/sqlite3rtree.h
+  $TOP/ext/session/sqlite3session.h
   $TOP/ext/fts5/fts5.h
 }]
 
@@ -91,15 +102,18 @@ set cdecllist {
 #
 foreach file $filelist {
   set in [open $file]
+  if {![regexp {sqlite\.h\.in} $file]} {
+    puts "/******** Begin file [file tail $file] *********/"
+  }
   while {![eof $in]} {
-  
+
     set line [gets $in]
 
     # File sqlite3rtree.h contains a line "#include ". Omit this
     # line when copying sqlite3rtree.h into sqlite3.h.
     #
     if {[string match {*#include*[<"]sqlite3.h[>"]*} $line]} continue
-  
+
     regsub -- --VERS--           $line $zVersion line
     regsub -- --VERSION-NUMBER-- $line $nVersion line
     regsub -- --SOURCE-ID--      $line "$zDate $zUuid" line
@@ -113,15 +127,25 @@ foreach file $filelist {
         if {[string index $rettype end] ne "*"} {
           append line " "
         }
-        if {[lsearch -exact $cdecllist $funcname] >= 0} {
-          append line SQLITE_CDECL
-        } else {
-          append line SQLITE_STDCALL
+        if {$useapicall} {
+          if {[lsearch -exact $cdecllist $funcname] >= 0} {
+            append line SQLITE_CDECL " "
+          } else {
+            append line SQLITE_APICALL " "
+          }
         }
-        append line " " $funcname $rest
+        append line $funcname $rest
       }
     }
+    if {$useapicall} {
+      set line [string map [list (*sqlite3_syscall_ptr) \
+          "(SQLITE_SYSAPI *sqlite3_syscall_ptr)"] $line]
+      regsub {\(\*} $line {(SQLITE_CALLBACK *} line
+    }
     puts $line
   }
   close $in
+  if {![regexp {sqlite\.h\.in} $file]} {
+    puts "/******** End of [file tail $file] *********/"
+  }
 }
diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl
index a14fd230..c874d538 100644
--- a/tool/mkvsix.tcl
+++ b/tool/mkvsix.tcl
@@ -77,7 +77,7 @@
 # command prompt:
 #
 #                         CD /D C:\dev\sqlite\core
-#                         tclsh85 tool\mkvsix.tcl C:\Temp
+#                         tclsh tool\mkvsix.tcl C:\Temp
 #
 # In the example above, "C:\dev\sqlite\core" represents the root of the source
 # tree for SQLite and "C:\Temp" represents the top-level directory containing
@@ -463,7 +463,7 @@ if {[string equal $packageFlavor WinRT]} then {
   }
   set shortName $shortNames($packageFlavor,$vsVersion)
   set displayName $displayNames($packageFlavor,$vsVersion)
-  set targetPlatformIdentifier UWP
+  set targetPlatformIdentifier UAP; # NOTE: Not "UWP".
   set targetPlatformVersion v0.8.0.0
   set minVsVersion [getMinVsVersionXmlChunk $vsVersion]
   set maxPlatformVersion \
diff --git a/tool/opcodesum.tcl b/tool/opcodesum.tcl
new file mode 100644
index 00000000..47dff32b
--- /dev/null
+++ b/tool/opcodesum.tcl
@@ -0,0 +1,34 @@
+#!/usr/bin/tclsh
+#
+# Run this script, redirecting input from cachegrind output, to compute the
+# number of CPU cycles used by each VDBE opcode.
+#
+# The cachegrind output should be configured so that it reports a single
+# column of Ir at the left margin. Ex:
+#
+#    cg_annotation --show=Ir --auto=yes cachegrind.out.* | tclsh opcodesum.tcl
+#
+set currentop x
+set ncycle(x) 0
+while {![eof stdin]} {
+  set line [string map {\173 x \175 x \042 x} [gets stdin]]
+  if {[regexp {  \.  case OP_.*:} $line]} {
+    regexp {OP_(.+):} $line all currentop
+    set ncycle($currentop) 0
+  } elseif {[lindex $line 1]=="default:"
+            && [regexp {really OP_Noop and OP_Explain} $line]} {
+    break
+  } elseif {[lindex $line 0]!="."} {
+    regsub -all {[^0-9]} [lindex $line 0] {} n
+    if {$n!=""} {incr ncycle($currentop) $n}
+  }
+}
+unset ncycle(x)
+set results {}
+foreach op [lsort [array names ncycle]] {
+  if {$ncycle($op)==0} continue
+  lappend results [list $ncycle($op) $op]
+}
+foreach entry [lsort -index 0 -int -decr $results] {
+  puts [format {%-16s %10d} [lindex $entry 1] [lindex $entry 0]]
+}
diff --git a/tool/replace.tcl b/tool/replace.tcl
index b01a83ac..5a1ac598 100644
--- a/tool/replace.tcl
+++ b/tool/replace.tcl
@@ -4,16 +4,19 @@
 # only lines successfully modified with a regular
 # expression.
 #
+fconfigure stdout -translation binary -encoding binary
+fconfigure stderr -translation binary -encoding binary
 set mode [string tolower [lindex $argv 0]]
 set from [lindex $argv 1]
 set to [lindex $argv 2]
-if {$mode ni [list exact include]} {exit 1}
+if {$mode ni [list exact regsub include]} {exit 1}
 if {[string length $from]==0} {exit 2}
 while {![eof stdin]} {
   set line [gets stdin]
   if {[eof stdin]} break
   switch -exact $mode {
     exact {set line [string map [list $from $to] $line]}
+    regsub {regsub -all -- $from $line $to line}
     include {if {[regsub -all -- $from $line $to line]==0} continue}
   }
   puts stdout $line
diff --git a/tool/run-speed-test.sh b/tool/run-speed-test.sh
index ee2ceac6..0e970ea0 100644
--- a/tool/run-speed-test.sh
+++ b/tool/run-speed-test.sh
@@ -12,7 +12,7 @@
 #
 #     summary-$BASE.txt           # Copy of standard output
 #     cout-$BASE.txt              # cachegrind output
-#     explain-$BASE.txt           # EXPLAIN listings
+#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
 #
 if test "$1" = ""
 then
@@ -21,9 +21,10 @@ then
 fi
 NAME=$1
 shift
-CC_OPTS="-DSQLITE_ENABLE_RTREE"
-SPEEDTEST_OPTS="--shrink-memory --reprepare"
+CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
+SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64"
 SIZE=5
+doExplain=0
 while test "$1" != ""; do
   case $1 in
     --reprepare)
@@ -35,12 +36,29 @@ while test "$1" != ""; do
     --utf16be)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
         ;;
+    --stats)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
     --without-rowid)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
         ;;
+    --nomemstat)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --wal)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        ;;
     --size)
         shift; SIZE=$1
         ;;
+    --explain)
+        doExplain=1
+        ;;
+    --heap)
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
+        ;;
     *)
         CC_OPTS="$CC_OPTS $1"
         ;;
@@ -54,9 +72,11 @@ echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
 rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
 gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
 size sqlite3.o | tee -a summary-$NAME.txt
-gcc -g -Os -Wall -I. $CC_OPTS \
-   -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-   ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+if test $doExplain -eq 1; then
+  gcc -g -Os -Wall -I. $CC_OPTS \
+     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+fi
 SRC=./speedtest1.c
 gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
 ls -l speedtest1 | tee -a summary-$NAME.txt
@@ -65,4 +85,6 @@ valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
 size sqlite3.o | tee -a summary-$NAME.txt
 wc sqlite3.c
 cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
-./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+if test $doExplain -eq 1; then
+  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+fi
diff --git a/tool/showdb.c b/tool/showdb.c
index 06cd36cd..d51a2fd8 100644
--- a/tool/showdb.c
+++ b/tool/showdb.c
@@ -151,7 +151,7 @@ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){
       exit(1);
     }
   }else{
-    lseek(g.dbfd, ofst, SEEK_SET);
+    lseek(g.dbfd, (long)ofst, SEEK_SET);
     got = read(g.dbfd, aData, nByte);
     if( got>0 && got "
+    } else {
+      puts -nonewline "% "
+    }
+    flush stdout
+    append line [gets stdin]
+    if {[info complete $line]} {
+      if {[catch {uplevel #0 $line} result]} {
+        puts stderr "Error: $result"
+      } elseif {$result!=""} {
+        puts $result
+      }
+      set line {}
+    } else {
+      append line \n
+    }
+  }
+}
+
+
 # Get the name of the database to analyze
 #
 proc usage {} {
@@ -34,22 +61,37 @@ information for the database and its constituent tables and indexes.
 
 Options:
 
-   --stats        Output SQL text that creates a new database containing
-                  statistics about the database that was analyzed
+   --pageinfo   Show how each page of the database-file is used
 
-   --pageinfo     Show how each page of the database-file is used
+   --stats      Output SQL text that creates a new database containing
+                statistics about the database that was analyzed
+
+   --tclsh      Run the built-in TCL interpreter interactively (for debugging)
+
+   --version    Show the version number of SQLite
 }
   exit 1
 }
 set file_to_analyze {}
 set flags(-pageinfo) 0
 set flags(-stats) 0
+set flags(-debug) 0
 append argv {}
 foreach arg $argv {
   if {[regexp {^-+pageinfo$} $arg]} {
     set flags(-pageinfo) 1
   } elseif {[regexp {^-+stats$} $arg]} {
     set flags(-stats) 1
+  } elseif {[regexp {^-+debug$} $arg]} {
+    set flags(-debug) 1
+  } elseif {[regexp {^-+tclsh$} $arg]} {
+    tclsh
+    exit 0
+  } elseif {[regexp {^-+version$} $arg]} {
+    sqlite3 mem :memory:
+    puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}]
+    mem close
+    exit 0
   } elseif {[regexp {^-} $arg]} {
     puts stderr "Unknown option: $arg"
     usage
@@ -100,6 +142,10 @@ if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} {
   puts stderr "error trying to open $file_to_analyze: $msg"
   exit 1
 }
+if {$flags(-debug)} {
+  proc dbtrace {txt} {puts $txt; flush stdout;}
+  db trace ::dbtrace
+}
 
 db eval {SELECT count(*) FROM sqlite_master}
 set pageSize [expr {wide([db one {PRAGMA page_size}])}]
@@ -142,16 +188,22 @@ if {$flags(-stats)} {
   exit 0
 }
 
+
 # In-memory database for collecting statistics. This script loops through
 # the tables and indices in the database being analyzed, adding a row for each
 # to an in-memory database (for which the schema is shown below). It then
 # queries the in-memory db to produce the space-analysis report.
 #
 sqlite3 mem :memory:
+if {$flags(-debug)} {
+  proc dbtrace {txt} {puts $txt; flush stdout;}
+  mem trace ::dbtrace
+}
 set tabledef {CREATE TABLE space_used(
    name clob,        -- Name of a table or index in the database file
    tblname clob,     -- Name of associated table
    is_index boolean, -- TRUE if it is an index, false for a table
+   is_without_rowid boolean, -- TRUE if WITHOUT ROWID table  
    nentry int,       -- Number of entries in the BTree
    leaf_entries int, -- Number of leaf entries
    depth int,        -- Depth of the b-tree
@@ -184,7 +236,7 @@ set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 }
 foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
 
   set is_index [expr {$name!=$tblname}]
-  set idx_btree [expr {$is_index || [is_without_rowid $name]}]
+  set is_without_rowid [is_without_rowid $name]
   db eval {
     SELECT 
       sum(ncell) AS nentry,
@@ -235,6 +287,7 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
       $name,
       $tblname,
       $is_index,
+      $is_without_rowid,
       $nentry,
       $leaf_entries,
       $depth,
@@ -330,12 +383,15 @@ proc subreport {title where showFrag} {
   # following query returns exactly one row (because it is an aggregate).
   #
   # The results of the query are stored directly by SQLite into local 
-  # variables (i.e. $nentry, $nleaf etc.).
+  # variables (i.e. $nentry, $payload etc.).
   #
   mem eval "
     SELECT
-      int(sum(nentry)) AS nentry,
-      int(sum(leaf_entries)) AS nleaf,
+      int(sum(
+        CASE WHEN (is_without_rowid OR is_index) THEN nentry 
+             ELSE leaf_entries 
+        END
+      )) AS nentry,
       int(sum(payload)) AS payload,
       int(sum(ovfl_payload)) AS ovfl_payload,
       max(mx_payload) AS mx_payload,
@@ -375,8 +431,8 @@ proc subreport {title where showFrag} {
   set storage [expr {$total_pages*$pageSize}]
   set payload_percent [percent $payload $storage {of storage consumed}]
   set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
-  set avg_payload [divide $payload $nleaf]
-  set avg_unused [divide $total_unused $nleaf]
+  set avg_payload [divide $payload $nentry]
+  set avg_unused [divide $total_unused $nentry]
   if {$int_pages>0} {
     # TODO: Is this formula correct?
     set nTab [mem eval "
@@ -390,12 +446,12 @@ proc subreport {title where showFrag} {
     "]
     set avg_fanout [format %.2f $avg_fanout]
   }
-  set ovfl_cnt_percent [percent $ovfl_cnt $nleaf {of all entries}]
+  set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}]
 
   # Print out the sub-report statistics.
   #
   statline {Percentage of total database} $total_pages_percent
-  statline {Number of entries} $nleaf
+  statline {Number of entries} $nentry
   statline {Bytes of storage consumed} $storage
   if {$compressed_size!=$storage} {
     set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
new file mode 100644
index 00000000..57f2ca07
--- /dev/null
+++ b/tool/speed-check.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+#
+# This is a template for a script used for day-to-day size and 
+# performance monitoring of SQLite.  Typical usage:
+#
+#     sh run-speed-test.sh trunk  #  Baseline measurement of trunk
+#     sh run-speed-test.sh x1     # Measure some experimental change
+#     fossil test-diff --tk cout-trunk.txt cout-x1.txt   # View chanages
+#
+# There are multiple output files, all with a base name given by
+# the first argument:
+#
+#     summary-$BASE.txt           # Copy of standard output
+#     cout-$BASE.txt              # cachegrind output
+#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
+#
+if test "$1" = ""
+then
+  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
+  exit
+fi
+NAME=$1
+shift
+#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
+CC_OPTS="-DSQLITE_ENABLE_MEMSYS5"
+SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64"
+SIZE=5
+LEAN_OPTS="-DSQLITE_THREADSAFE=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOB"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
+doExplain=0
+doCachegrind=1
+while test "$1" != ""; do
+  case $1 in
+    --reprepare)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --autovacuum)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --utf16be)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --stats)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --without-rowid)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --nomemstat)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --temp)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
+        ;;
+    --wal)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        ;;
+    --size)
+        shift; SIZE=$1
+        ;;
+    --cachesize)
+        shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1"
+        ;;
+    --explain)
+        doExplain=1
+        ;;
+    --vdbeprofile)
+        rm -f vdbe_profile.out
+        CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
+        doCachegrind=0
+        ;;
+    --lean)
+        CC_OPTS="$CC_OPTS $LEAN_OPTS"
+        ;;
+    --heap)
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
+        ;;
+    *)
+        CC_OPTS="$CC_OPTS $1"
+        ;;
+  esac
+  shift
+done
+SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
+echo "NAME           = $NAME" | tee summary-$NAME.txt
+echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
+echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
+rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
+gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
+size sqlite3.o | tee -a summary-$NAME.txt
+if test $doExplain -eq 1; then
+  gcc -g -Os -Wall -I. $CC_OPTS \
+     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+fi
+SRC=./speedtest1.c
+gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
+ls -l speedtest1 | tee -a summary-$NAME.txt
+if test $doCachegrind -eq 1; then
+  valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
+      $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+else
+  ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+fi
+size sqlite3.o | tee -a summary-$NAME.txt
+wc sqlite3.c
+if test $doCachegrind -eq 1; then
+  cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
+fi
+if test $doExplain -eq 1; then
+  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+fi
diff --git a/tool/sqldiff.c b/tool/sqldiff.c
index ae01cd3c..67f3197b 100644
--- a/tool/sqldiff.c
+++ b/tool/sqldiff.c
@@ -33,6 +33,7 @@ struct GlobalVars {
   const char *zArgv0;       /* Name of program */
   int bSchemaOnly;          /* Only show schema differences */
   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
+  int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
   unsigned fDebug;          /* Debug flags */
   sqlite3 *db;              /* The database connection */
 } g;
@@ -402,7 +403,8 @@ static void printQuoted(FILE *out, sqlite3_value *X){
         }
         fprintf(out, "'");
       }else{
-        fprintf(out, "NULL");
+        /* Could be an OOM, could be a zero-byte blob */
+        fprintf(out, "X''");
       }
       break;
     }
@@ -683,7 +685,7 @@ static void diff_one_table(const char *zTab, FILE *out){
 
   /* Run the query and output differences */
   if( !g.bSchemaOnly ){
-    pStmt = db_prepare(sql.z);
+    pStmt = db_prepare("%s", sql.z);
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       int iType = sqlite3_column_int(pStmt, nPk);
       if( iType==1 || iType==2 ){
@@ -1178,8 +1180,9 @@ static void getRbudiffQuery(
   strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
-  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n)");
+  strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) AND ");
+  strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Deleted rows: */
   strPrintf(pSql, "\nUNION ALL\nSELECT ");
@@ -1193,8 +1196,9 @@ static void getRbudiffQuery(
   strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
-  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n) ");
+  strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) AND ");
+  strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Updated rows. If all table columns are part of the primary key, there 
   ** can be no updates. In this case this part of the compound SELECT can
@@ -1225,7 +1229,7 @@ static void getRbudiffQuery(
     );
 
     strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
-    strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
+    strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
     strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
   }
 
@@ -1244,6 +1248,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
   Str sql = {0, 0, 0};            /* Query to find differences */
   Str insert = {0, 0, 0};         /* First part of output INSERT statement */
   sqlite3_stmt *pStmt = 0;
+  int nRow = 0;                   /* Total rows in data_xxx table */
 
   /* --rbu mode must use real primary keys. */
   g.bSchemaPK = 1;
@@ -1289,6 +1294,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
 
     /* Output the first part of the INSERT statement */
     fprintf(out, "%s", insert.z);
+    nRow++;
 
     if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
       for(i=0; i<=nCol; i++){
@@ -1299,7 +1305,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
       char *zOtaControl;
       int nOtaControl = sqlite3_column_bytes(pStmt, nCol);
 
-      zOtaControl = (char*)sqlite3_malloc(nOtaControl);
+      zOtaControl = (char*)sqlite3_malloc(nOtaControl+1);
       memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1);
 
       for(i=0; i0 ){
+    Str cnt = {0, 0, 0};
+    strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
+    fprintf(out, "%s\n", cnt.z);
+    strFree(&cnt);
+  }
 
   strFree(&ct);
   strFree(&sql);
@@ -1452,7 +1464,7 @@ static void summarize_one_table(const char *zTab, FILE *out){
   }
 
   /* Run the query and output difference summary */
-  pStmt = db_prepare(sql.z);
+  pStmt = db_prepare("%s", sql.z);
   nUpdate = 0;
   nInsert = 0;
   nDelete = 0;
@@ -1727,6 +1739,144 @@ end_changeset_one_table:
   sqlite3_free(zId);
 }
 
+/*
+** Extract the next SQL keyword or quoted string from buffer zIn and copy it
+** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
+** Return a pointer to the character within zIn immediately following 
+** the token or quoted string just extracted.
+*/
+const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
+  const char *p = zIn;
+  char *pOut = zBuf;
+  char *pEnd = &pOut[nBuf-1];
+  char q = 0;                     /* quote character, if any */
+
+  if( p==0 ) return 0;
+  while( *p==' ' ) p++;
+  switch( *p ){
+    case '"': q = '"'; break;
+    case '\'': q = '\''; break;
+    case '`': q = '`'; break;
+    case '[': q = ']'; break;
+  }
+
+  if( q ){
+    p++;
+    while( *p && pOut
+
+
diff --git a/vsixtest/App.xaml.cpp b/vsixtest/App.xaml.cpp
new file mode 100644
index 00000000..da8f327f
--- /dev/null
+++ b/vsixtest/App.xaml.cpp
@@ -0,0 +1,120 @@
+//
+// App.xaml.cpp
+// Implementation of the App class.
+//
+
+#include "pch.h"
+#include "MainPage.xaml.h"
+
+using namespace vsixtest;
+
+using namespace Platform;
+using namespace Windows::ApplicationModel;
+using namespace Windows::ApplicationModel::Activation;
+using namespace Windows::Foundation;
+using namespace Windows::Foundation::Collections;
+using namespace Windows::UI::Xaml;
+using namespace Windows::UI::Xaml::Controls;
+using namespace Windows::UI::Xaml::Controls::Primitives;
+using namespace Windows::UI::Xaml::Data;
+using namespace Windows::UI::Xaml::Input;
+using namespace Windows::UI::Xaml::Interop;
+using namespace Windows::UI::Xaml::Media;
+using namespace Windows::UI::Xaml::Navigation;
+
+/// 
+/// Initializes the singleton application object.  This is the first line of authored code
+/// executed, and as such is the logical equivalent of main() or WinMain().
+/// 
+App::App()
+{
+	InitializeComponent();
+	Suspending += ref new SuspendingEventHandler(this, &App::OnSuspending);
+}
+
+/// 
+/// Invoked when the application is launched normally by the end user.	Other entry points
+/// will be used such as when the application is launched to open a specific file.
+/// 
+/// Details about the launch request and process.
+void App::OnLaunched(Windows::ApplicationModel::Activation::LaunchActivatedEventArgs^ e)
+{
+
+#if _DEBUG
+		// Show graphics profiling information while debugging.
+		if (IsDebuggerPresent())
+		{
+			// Display the current frame rate counters
+			 DebugSettings->EnableFrameRateCounter = true;
+		}
+#endif
+
+	auto rootFrame = dynamic_cast(Window::Current->Content);
+
+	// Do not repeat app initialization when the Window already has content,
+	// just ensure that the window is active
+	if (rootFrame == nullptr)
+	{
+		// Create a Frame to act as the navigation context and associate it with
+		// a SuspensionManager key
+		rootFrame = ref new Frame();
+
+		rootFrame->NavigationFailed += ref new Windows::UI::Xaml::Navigation::NavigationFailedEventHandler(this, &App::OnNavigationFailed);
+
+		if (e->PreviousExecutionState == ApplicationExecutionState::Terminated)
+		{
+			// TODO: Restore the saved session state only when appropriate, scheduling the
+			// final launch steps after the restore is complete
+
+		}
+
+		if (rootFrame->Content == nullptr)
+		{
+			// When the navigation stack isn't restored navigate to the first page,
+			// configuring the new page by passing required information as a navigation
+			// parameter
+			rootFrame->Navigate(TypeName(MainPage::typeid), e->Arguments);
+		}
+		// Place the frame in the current Window
+		Window::Current->Content = rootFrame;
+		// Ensure the current window is active
+		Window::Current->Activate();
+	}
+	else
+	{
+		if (rootFrame->Content == nullptr)
+		{
+			// When the navigation stack isn't restored navigate to the first page,
+			// configuring the new page by passing required information as a navigation
+			// parameter
+			rootFrame->Navigate(TypeName(MainPage::typeid), e->Arguments);
+		}
+		// Ensure the current window is active
+		Window::Current->Activate();
+	}
+}
+
+/// 
+/// Invoked when application execution is being suspended.	Application state is saved
+/// without knowing whether the application will be terminated or resumed with the contents
+/// of memory still intact.
+/// 
+/// The source of the suspend request.
+/// Details about the suspend request.
+void App::OnSuspending(Object^ sender, SuspendingEventArgs^ e)
+{
+	(void) sender;	// Unused parameter
+	(void) e;	// Unused parameter
+
+	//TODO: Save application state and stop any background activity
+}
+
+/// 
+/// Invoked when Navigation to a certain page fails
+/// 
+/// The Frame which failed navigation
+/// Details about the navigation failure
+void App::OnNavigationFailed(Platform::Object ^sender, Windows::UI::Xaml::Navigation::NavigationFailedEventArgs ^e)
+{
+	throw ref new FailureException("Failed to load Page " + e->SourcePageType.Name);
+}
\ No newline at end of file
diff --git a/vsixtest/App.xaml.h b/vsixtest/App.xaml.h
new file mode 100644
index 00000000..5fa8837d
--- /dev/null
+++ b/vsixtest/App.xaml.h
@@ -0,0 +1,27 @@
+//
+// App.xaml.h
+// Declaration of the App class.
+//
+
+#pragma once
+
+#include "App.g.h"
+
+namespace vsixtest
+{
+	/// 
+	/// Provides application-specific behavior to supplement the default Application class.
+	/// 
+	ref class App sealed
+	{
+	protected:
+		virtual void OnLaunched(Windows::ApplicationModel::Activation::LaunchActivatedEventArgs^ e) override;
+
+	internal:
+		App();
+
+	private:
+		void OnSuspending(Platform::Object^ sender, Windows::ApplicationModel::SuspendingEventArgs^ e);
+		void OnNavigationFailed(Platform::Object ^sender, Windows::UI::Xaml::Navigation::NavigationFailedEventArgs ^e);
+	};
+}
diff --git a/vsixtest/Assets/LockScreenLogo.scale-200.png b/vsixtest/Assets/LockScreenLogo.scale-200.png
new file mode 100644
index 0000000000000000000000000000000000000000..735f57adb5dfc01886d137b4e493d7e97cf13af3
GIT binary patch
literal 1430
zcmaJ>TTC2P7~aKltDttVHYH6u8Io4i*}3fO&d$gd*bA_<3j~&e7%8(eXJLfhS!M@!
zKrliY>>6yT4+Kr95$!DoD(Qn-5TP|{V_KS`k~E6(LGS@#`v$hQo&^^BKsw3HIsZBT
z_y6C2n`lK@apunKojRQ^(_P}Mgewt$(^BBKCTZ;*xa?J3wQ7~@S0lUvbcLeq1Bg4o
zH-bvQi|wt~L7q$~a-gDFP!{&TQfc3fX*6=uHv*
zT&1&U(-)L%Xp^djI2?~eBF2cxC@YOP$+9d?P&h?lPy-9M2UT9fg5jKm1t$m#iWE{M
zIf%q9@;fyT?0UP>tcw-bLkz;s2LlKl2qeP0w
zECS7Ate+Awk|KQ+DOk;fl}Xsy4o^CY=pwq%QAAKKl628_yNPsK>?A>%D8fQG6IgdJ
ztnxttBz#NI_a@fk7SU`WtrpsfZsNs9^0(2a
z@C3#YO3>k~w7?2hipBf{#b6`}Xw1hlG$yi?;1dDs7k~xDAw@jiI*+tc;t2Lflg&bM)0!Y;0_@=w%`LW^8DsYpS#-bLOklX9r?Ei}TScw|4DbpW%+7
zFgAI)f51s}{y-eWb|vrU-Ya!GuYKP)J7z#*V_k^Xo>4!1Yqj*m)x&0L^tg3GJbVAJ
zJ-Pl$R=NAabouV=^z_t;^K*0AvFs!vYU>_<|I^#c?>>CR<(T?=%{;U=aI*SbZADLH
z&(f2wz_Y0??Tf|g;?|1Znw6}6U43Q#qNRwv1vp9uFn1)V#*4p&%$mP9x&15^OaBiDS(XppT|z^>;B{PLVEbS3IFYV
yGvCsSX*m

literal 0
HcmV?d00001

diff --git a/vsixtest/Assets/SplashScreen.scale-200.png b/vsixtest/Assets/SplashScreen.scale-200.png
new file mode 100644
index 0000000000000000000000000000000000000000..023e7f1feda78d5100569825acedfd213a0d84e9
GIT binary patch
literal 7700
zcmeHLYj~4Yw%(;oxoEH#Kxq-eR|+VkP17b#Vk;?4QwkI+A{L04G+#<<(x#Un1#+h5>eArRq
zTw$)ZvTWW_Y?bDho0nPVTh08+s`sp!j74rJTTtXIDww0SILedFv?sZ?yb@@}GN;#8
znk_b~Q(A0YR#uV4ef!osoV1M3;vQ8N$O|fStfgf$S5;ddUNv`tWtGjM;koG#N;7M<
zP*84lnx(bn_KF&9Z5Ai$)#Cs3a|$OFw>WKCT$of*L7_CqQEinflT|W{JT+aKp-E0v
zsxmYg)1(T>DROm+LN1eQw8}KCTp=C!$H7`PU!t9_Hw@TsTI2`udRZv*!a5`#A9hK6Y95L(CDUX&_@QxKV
z_feX{UhA#ZWlvgpL$#w^D#lq`_A4AzDqd|Zv6y9PX&DNcN|l}_D^{q@GG&H^Pg583
z8FI6N8^H7b5WjGp;urW)d7F+_lcp%KsLX0viCmE(OHH+=%ZfD_=`voUuoUxFO^L;-
z;!;2{g-YiiO6m4bs89OuF9!p{FGtH-f%8<2gY!h9s)4ciN%{Kh1+`}{^}M~+TDH9N
z^Z5PlgVXMC&2&k*Hw^Lb9gny#ro$MOIxIt{+r)EA10$VR3
zanN8D{TUkl+v0CQ_>ZoHP<M-x#8@8ZiT#$Kh`(uRaX1g$Bg|qy$<#7
zSSAi{Nb8Y=lvNVeio+UGLCAtoLBfL`iOv`)yoJMDJBN>4IH@(l7YRF;61@>qq1iM9
zr@b#OC~SAxSle?5Pp8Z78{VO0YFr1x7kZU64Z23eLf2T2#6J_t;-E}DkB?NufZ0Ug
zi?J&byXeaB-uTNVhuiM!UVQw}bZrJ3GtAETYp->!{q#zfN7D3AS9@Q7*V^85jGx#R
z(QxYV(wW#F0XF9^^s>>H8pPlVJ>)3Oz
z&_X8Sf@~?cH_O*cgi$U#`v`RRfv#y3m(ZpKk^5uLup+lVs$~}FZU$r_+}#hl%?g5m
z-u-}-666ssp-xWQak~>PPy$mRc|~?pVSs1_@mBEXpPVfLF6(Ktf1S*
zPPh@QZ=tFMs?LM2(5P3L2;l_6XX6s&cYsP1ip#eg0`ZEP0HGYh{UmS@o`MihLLvkU
zgyAG0G`b1|qjxxh1(ODKFE%AP}Dq=3vK$P7TXP4GrM1kQ72!GUVMDl`rDC&2;TA}*nF
z8$nQD&6ys_nc1*E7$*1S@R8$ymy(sQV}imGSedB@{!QR5P&N_H=-^o!?LsWs+2|mH
z-e=)T^SvI)=_JIm7}j4;@*Z17=(#}m=~YF~z~CLI+vdAGlJDcdF$TM?CVI1%LhUrN
zaa6DJ=Yh$)$k&Oz{-~8yw^GM^8prYxSxo
zvI4k#ibryMa%%*8oI-5m61Koa_A_xg=(fwp0aBX{;X4Q;NXUhtaoJDo1>TqhWtn=_
zd5~chq#&6~c%8JZK#t_&J(9EVUU&upYeIovLt1>vaHe}UUq>#RGQj!EN#5+0@T`(@
z^g~>*c`VGRiSt;!$_4+0hk^I!@O3``5=sZ8IwlxWW7km1B&_t&E*u0_9UBa#VqwY*
zz>nxv?FAsVnRaD(Bui=6i==BFUw0k4n$>`umU`F2l?7CYTD^)c2X+d9X&ddS9|gj?
zM?knGkGCX&W8offw8aLC2$D{PjC3nVZwd4k?eZH8*mZ)U@3Qk8RDFOz_#WUA#vnzy
zyP>KrCfKwSXea7}jgJjBc}PGY+4#6%lbZyjhy`5sZd_Vy6Wz;ixa?czkN}J9It1K6
zY!eu>|AwF^fwZlLAYyQI*lM@^>O>Iu6Vf6i>Q$?v!SeUS<{>UYMwz$*%Aq?w^`j{h
z!$GZbhu=^D{&ET8;))LL%ZBDZkQqRd2;u~!d9bHGmLRhLDctNgYyjsuvoSZ#iVdoB
z2!f--UUA#U;<{je#?cYt^{PIyKa%hW>}uepWMyAI{{Zo7?2>?$c9;whJae%oN|I-kpTQSx_C$Z&;f
zi2i)qmEn=y4U0uvk)$m;zKfjPK@oc?I`}1Jzl$Q~aoKBd3kt7L#7gyt|A_qgz6ai<
z=X%D1i!d2h?rHR^R8SUj&G||dkC?DT>{o#Yau<@uqVT{Xef&XG}5*E4aPk{}~
zplx&XhaV)&1EfI3Em;Bw#O5SV^c;{twb-1Rw)+=0!e_BLbd7tYmXCH0wrlOSS+~`7He8Iqx0{CN+DVit9;*6L~JAN
zD&cyT)2?h}xnYmL?^)<7YyzZ3$FHU^Eg;DLqAV{#wv#Wj7S`Jdl1pX&{3(uZ?!uh}
zDc$ZTNV*7le_W6}Hju~GMTxZQ1aWCeUc%!jv3MHAzt>Y-nQK%zfT*3ebDQA5b?iGn;
zBjv3B+GhLTexd_(CzZDP4|#n5^~scvB6#Pk%Ho!kQ>yYw((Dv{6=$g3jT1!u6gORW
zx5#`7Wy-ZHRa~IxGHdrp(bm%lf>2%J660nj$fCqN(epv@y!l9s7@k6EvxS{AMP>WY
zX4$@F8^kayphIx-RGO$+LYl9YdoI5d|4#q9##`_F5Xnx`&GPzp2fB{-{P@ATw=X@~
z_|&^UMWAKD;jjBKTK(~o?cUFRK8EX=6>cXpfzg4ZpMB>*w_^8GSiT-Jp|xBOnzM+j
z*09-@-~qJ(eqWq5@R4i^u4^{McCP(!3}C|v_WsTR*bIUxN(Nx`u##3B4{sE`Z`v8w
zAwIG`?1~PkID~W{uDzmqH98Pew_1(;x2%8r^vY{)_&J2K)cN{W+h5+g)ZcjP&Ci#O
zgy|8K@4kyMfwilHd&6TDlhb%++Pk!>9HRld6HT7gwyZGrxS$}CsD6`>6!!2K1@Mjf
z(P0WYB7V_OFZyeWrbOFb>O54BNXf~K&?}3=^v;v_wT{DKr?jN^DtN&DXwX%u?s*c6`%8>WFz
z7}YW^tp0bp^NriE)AB6M2l<7rn7fzePtR*omOevpfm9n?}2V*+0iW;S)C
zhg`NAjL?D=W#k*$aR{>pGf~lD-rVtD;5jW1_*Jn1j1=es@Kcx4ySM_bwcQCT=d+DV
z>Sz~L=Hj@(X%31nK$mWI@7d>}ORB`K(p=+`UD)+99YUGQc7y^bHZ1F(8|tL0
zdK*DT0kSXG_{BKTpP2*2PecdKV9;dq$^ZZDP;Nyq1kp-&GI5eAyZsK!e3V
zK@rPy*{(`KIfo+lc878mDKk^V#`VT05}64kBtk%DgwLrOvLMj5-;*GNKv6c6pzMuL
z6EP%ob|_0IW}lLRXCP2!9wWhEw3LA7iF#1O1mIZ@Z=6&bz41F;@S_GvYAG-#CW3z{
zP3+6vHhvP&A3$##Vo9$dT^#MoGg^|MDm=Bt1d2RRwSZ<;ZHICpLBv5Xs!D?BH^(9_
z7`H=N&^v|Z-%mP}wNzG{aiFCsRgwzwq!N6obW9+7(R;
z(SZ=23`|`>qil!LMGG{_Heq!BD>(Y-zV9wD)}hz25JA37YR%39;kI4y9pgtcUass6
zP24}ZY$vvYeI`zy&)A_X#nY3017ap*0&jx|mVwyGhg3;!keU53a}Uhm3BZI$N$6Se
zLWlAmy1S0xKJm4G_U@sN_Tm=`$xWJSEwKU98rZ&)1R^*$$1vA3oG#&*%SMxY_~oGP
zP&PFJatFLM-Ps%84IV-+Ow)T{C7cqUAvauy4C
z(FRz&?6$Rypj{xO!`y=*J5o4@U8Q-(y5(*=YoKeZ+-1YdljXxkA#B)zo=FeQH#?Le
zycNUmEEHWO9a=X^pb#&cOq7-`7UA87#|S22)<7RUtZo|(zibX=w;K3qur9vy#`MNV
z6UUcf9ZwEnKCCp+OoBnF@OdbvH)ANXO0o~Pi9l8=x3))}L<#vO0-~O4!~--Ket?d}
zJaqsj<@CD1%S2cTW%rOP{Vto%0sGW~1RMa_j^)5nil0Yw-
z0EE#bP+l4#P^%PQ+N*oxu1Zq05xZ!bXfYTg>9c{(Iw*lnjR^>kz%lAN^zFce7rppy
zY8zA~3GD=A6d*hze&l4D_wA~+O!56)BZTe_rEu}Ezi<4!kG|W#amBZ5{&XS2@6R~H
z{9o^y*BkH4$~yX9U&@CgbOzX1bn9xqF|zh$Dh0Y5y*E0e90*$!ObrHY3Ok0`2=O~r
zCuke6KrP9KOf?V(YDsM<6pX2nVoN%M$LT^q#FmtaF?1^27F*IcNX~XRB(|hCFvdcc
zc)$=S-)acdk$g4?_>jRqxpI6M3vHZk?0c^3=byamYDNf;uB{3NlKW5IhnOS3DNkMV
z?tK8?kJ}pmvp%&&eTVOVjHP`q34hN1@!aK}H(K!vI`~gf|Gv+FNEQD5Yd<~yX7k_l
h&G-K)@HZb3BABY{)U1?^%I#E6`MGoTtustd{~yM6srvu`

literal 0
HcmV?d00001

diff --git a/vsixtest/Assets/Square150x150Logo.scale-200.png b/vsixtest/Assets/Square150x150Logo.scale-200.png
new file mode 100644
index 0000000000000000000000000000000000000000..af49fec1a5484db1d52a7f9b5ec90a27c7030186
GIT binary patch
literal 2937
zcma)84OCO-8BSud5)jwMLRVKgX(S?$n?Ld|vrsm<$CF7)&zTbyy1FE5bU`Q17MRv`9ue$;R(@8kR;#vJ*IM0>cJIAOte!d7oRgdH
zd%ySjdB6L9=gX^A6)VzH7p2l@v~3zJAMw|DFy#^)F@@F*`mqUn=Il>l)8_+ab;nOW{%+iPx
z+s{Eu|&pIs)Z7{La9~?xKfyl
z#43?gjEL15d4WbOZo#SiP%>DB^+BcnJ=7dHEe;r#G=tuw|ka
z%q@}##Uh7;tc%L_64m(kHtw74ty%BJMb)_1)#S0j`)F8_1jF7vScpsnH=0V19bO8y
zR`0SjIdCUo&=>JwMQF8KHA<{ODHTiQh}0^@5QRmCA?gOH6_H3K^-_sNB^RrdNuK-R
zOO*vOrKCVvDwgUck`kF(E7j{I#iiN;b*ZdCt4m@HPA`EuEqGGf4%!K<;(=I=&Vyrw
z%TwcWtxa}8mCZ%Cyf&ActJ6_$ox5z6-D!0-dvnRx6t7y3d+h6QYpKWO;8OdnvERo7
zuEf>ih5`wqY)~o@OeVt-wM?Q!>QzdGRj!bz6fzYrfw$hZfAKzr2-M+D+R>}~oT574c;_3zquHcElqKIsryILt3g8n3jcMb+j?i?-L3FpZJ
z2WRVBRdDPc+G5aaYg#5hpE+6nQ|(VSoxT3|biF;BUq#==-27Xi=gihDPYP$7?=9cP
zYKE$jeQ|3~_L0VG-(F~2ZPyD0=k{J4Q~h(t__{-mz_w8{JDY9{`1ouzz!Vr5!ECdE
z6U~O1k8c}24V7~zzXWTV-Pe4)y}wQJS&q%H5`Fo_f_JvIU489aCX$;P`u#!I-=^4ijC2{&9!O&h>mi?9oYD=GC#%)6{GzN6nQYw+Fal50!#x^asjBBR50i`+mho*ttoqV)ubM2KD9S~k7+FR4>{29?6
z{!l6kDdyTN0YJ9LgkPWeXm|gyi@zM3?0@{&pXT12w|78&W-q!RRF)&iLCEZVH<|fR
zN0fr2^t8H(>L?>K#>^+jWROLral(Qy-xoBq1U7A&DV||wClb)Otd9?(gZ|8znMF}D
zf<1haWz^s0qgecz;RFGt0C-B4g`jNGHsFU+;{<%t65v^sjk^h$lmWn#B0#_)9ij&d
z-~lc`A)YYExi^7sBuPM^Y|wA2g*5?`K?#7tzELQYNxGo$UB$4J8RJp1k(8Jj+~hMT
zlN~>M@KTTh^--8y3PK_NZ@AC!{PT=CziBzGd+wTJ^@icH!Bd}%)g8V)%K?|c&WTUk
zy}qv1C%(fjRoZ4ozC3{O%@5?)XzH35zHns$pgU*Q?fj4v?fp1Qbm+j;3l;9jam9Da
zXVcKjPlQ73x78QPu|Ffm6x?`~e3oD=gl=4kYK?={kD5j~QCXU)`HSdduNNENzA*2$
zOm3PzF!lN5e*06-f1Uot67wY#{o-S1!KZ7E=!~7ynnk9_iJR#kFoNbAOT#^2Gd17F
zMmvU6>lndZQGd|ax9kUoXXO+$N?|j@6qpsF&_j7YXvwo_C{JpmLw5&#e6k>atv%es
z5)7r*Wvv_JkUpT}M!_o!nVlEk1Zbl=a*2hQ*<|%*K1Glj^FcF`6kTzGQ3lz~2tCc@
z&x|tj;aH&1&9HwcJBcT`;{?a+pnej;M1HO(6Z{#J!cZA04hnFl;NXA+&`=7bjW_^o
zfC40u3LMG?NdPtwGl>Tq6u}*QG)}-y;)lu-_>ee3kibW(69n0$0Zy!}9rQz%*v1iO
zT9_H>99yIrSPYVy6^);rR}7Yo=J_T@hi+qhTZXnVWyf;JDYm5#eYLTxr*?kiNn!+Y
zQ+LUkBafNJ#rH#C(?d5^;gw9o#%daEI{mA*LHPIHPU`#|H$hD
zwm>0&+kahQ)E#%~k>&5@&#Vg82H?s%71=)(soi@174pi9--2{w{1$}Sz4zGn3Du&x
bht0Iza^2ykEt4(epJ78uh5nDlX8(TxzDYwP

literal 0
HcmV?d00001

diff --git a/vsixtest/Assets/Square44x44Logo.scale-200.png b/vsixtest/Assets/Square44x44Logo.scale-200.png
new file mode 100644
index 0000000000000000000000000000000000000000..ce342a2ec8a61291ba76c54604aea7e9d20af11b
GIT binary patch
literal 1647
zcmaJ?eM}Q)7(e+G1Q(|`V9JhTI2>MkceK4;p;PR&$Pi?ejk3YQ_3o`S&|W_dsOZ8#
zWPTt69g`t$ab`0cj-Y0yiBSOqmd)tG7G(}M5aP0_%&9TijB#&)I{zSE^4@#z^FF`l
z`8{8`o%wlL(UI|y2!cdsuVamHH~H86F!*-15em4)NqUpCQM5?aoC_eCf@lV4wvF2a
zjDQn1JBL69f&@2M3rvzJcfE!eZ8FZUBlFlC5RD)it33{mF9#B82AiyQE%w)`vlwa>
zv{<1sm&kSKK$&%2jSFn7$t&P%%6Ue>R=EAnG8N7fqynWG8L3p!4801a;8{+nliO(qd(jNJ_?+9W3#hLIDLoT6~3fx9=`CC-D}-AMrpEO7HK
zt3$GicGPc?GmDjy7K2P@La;eu4!$zWCZ`ym{Z$b
zu-O6RM&K4JT|BIZB`E-gxqG%FzanI#+2FFmqHqXG7yxWB=w55RGOM)$xMb(>kSNR
z2w=1AZi%z=AmG~yea~XaXJR!v7vLn(RUnELfiB1|6D84ICOS}^Zo2AdN}<&*h}G_u
z{xZ!(%>tLT3J3<5XhWy-tg+6)0nmUUENLW8TWA{R6bgVd3X;anYFZ^IRis*_P-C-r
z;i>%1^eL3UI2-{w8nuFFcs0e~7J{O2k^~Ce%+Ly4U?|=!0LH=t6()xi<^I-rs+9sF
z*q{E-CxZbGPeu#a;XJwE;9S1?#R&uns>^0G3p`hEUF*v`M?@h%T%J%RChmD|EVydq
zmHWh*_=S%emRC*mhxaVLzT@>Z2SX0u9v*DIJ@WC^kLVdlGV6LpK$KIrlJqc
zpJ921)+3JJdTx|<`G&kXpKkjGJv=76R`yYIQ{#c-`%+`#V(7}Q;&@6U8!Td1`d;?N
z_9mnI#?AA}4J!r)LN4!E-@H5eXauuB7TOawS>Y|{-P?NNx-lq+z1W-+y(;39P&&LP
zL{N80?&=C*qKmdA^moMZRuPcD!B<*mq$ch=0Cnlitw#txRWhb3%TQvPqjkC`F69G4b!
ze7z9MZ#+;_#l?H37UqUhDFb^l&s2{oM$3I0o^Q!yx;;V)QmCMo)Tb_ui|mit8MS?U
zm##6$sZZ1$@|s%?l@>4Z<*Q}sRBSKMhb4I{e5LdEhsHIHTe8Bod5c>6QtT>$XgUBz
z6MK`kO$=jmt@FqggOhJ5j~e@ygRbG;<{Vu)*+nn9aQeo0;$#j;|MS=S$&L?BeV25z
xs3B`@=#`5TF{^6(A1rvdY@|-RtQ|iS5{tyX+wH?;n8E)G$kykv-D^wh{{!TZT%7;_

literal 0
HcmV?d00001

diff --git a/vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png b/vsixtest/Assets/Square44x44Logo.targetsize-24_altform-unplated.png
new file mode 100644
index 0000000000000000000000000000000000000000..f6c02ce97e0a802b85f6021e822c89f8bf57d5cd
GIT binary patch
literal 1255
zcmaJ>TWs4@7*5+{G#S+&C!qC#>
zf>5N3P6jO*Cz>ug*(_DmW=)kea&m$gZ^+nyiF`;j%w@}y8)>p*SH}C`m?DXeieF2U
zyQHecc_L%Gh!7GMt+hG06y;+|p4>m~}PjA}rKViGiEnn7G0ZO<>G|7q;2?NwGCM3s?eued6%hd$B+
z*kQJ{#~$S=DFE(%=E+UkmlEI*%3llUf~8Ja9YU1Vui0IbGBkW_gHB%Rd&!!ioX
zs40O?i9I{};kle7GMvE7(rk`la=gTI)47=>%?q@^iL-nUo3}h4S}N-KHn8t5mVP8w
z&bSErwp+37
zNJJ8?a|{r5Q3R0Z5s-LB1WHOwYC@7pCHWND#cL1cZ?{kJ368_*(UDWUDyb<}0y@o#
zfMF016iMWPCb6obAxT$JlB6(2DrlXDTB&!0`!m??4F(qWMhjVZo?JXQmz`1*58Z=&
zcDmB|S-E@j?BoFGix0flckqdS4jsPNzhfWyWIM98GxcLs89C(~dw%$_t;JjX-SD}E
zfiGV;{8Q%8r}w9x>EEigW81>`kvnU@pK)4+xk9@+bNj9L!AAZ@SZ@q|)&BmY3+HZx
zul~BeG4|}-;L%cHViQGQX?^zFfO0&#cHwel=d`lH9sJ-@Sl@n*(8J2>%Ac`IxyY?Q
z{=GhWvC#gu-~Ia7*n{=+;qM?Ul_wy1+u7ho;=`>EwP^g~R@{unBds`!#@}tluZQpS
zm)M~nYEifJWJGx?_6DcTy>#uh%>!H9=hb^(v`=m3F1{L>db=<5_tm+_&knAQ2EU$s
Mu9UqpbNZeC0BbUo^Z)<=

literal 0
HcmV?d00001

diff --git a/vsixtest/Assets/StoreLogo.png b/vsixtest/Assets/StoreLogo.png
new file mode 100644
index 0000000000000000000000000000000000000000..7385b56c0e4d3c6b0efe3324aa1194157d837826
GIT binary patch
literal 1451
zcmaJ>eN5D57_Z|bH;{0+1#mbl)eTU3{h)Wf7EZV?;HD@XL@{B`Ui%(2aMxQ~xdXSv
z5nzWi(LW)U2=Vc-cY@s7nPt{i0hc6!7xN4NNHI#EQl>YNBy8l4%x9gr_W-j
zEZMQmmTIy(>;lblRfh`dIyTgc9W5d!VP$L4(kKrN1c5G~(O_#xG
zAJCNTstD^5SeXFB+&$h=ToJP2H>xr$iqPs-#O*;4(!Fjw25-!gEb*)mU}=)J;Iu>w
zxK(5XoD0wrPSKQ~rbL^Cw6O_03*l*}i=ydbu7adJ6y;%@tjFeXIXT+ms30pmbOP%Q
zX}S;+LBh8Tea~TSkHzvX6$rYb)+n&{kSbIqh|c7hmlxmwSiq5iVhU#iEQ<>a18|O^Sln-8t&+t`*{qBWo5M?wFM(JuimAOb5!K#D}XbslM@#1ZVz_;!9U
zpfEpLAOz=0g@bd6Xj_ILi-x^!M}73h^o@}hM$1jflTs|Yuj9AL@A3<-?MV4!^4q`e
z)fO@A;{9K^?W?DbnesnPr6kK>$zaKo&;FhFd(GYFCIU^T+OIMb%Tqo+P%oq(IdX7S
zf6+HLO?7o0m+p>~Tp5UrXWh!UH!wZ5kv!E`_w)PTpI(#Iw{AS`gH4^b(bm^ZCq^FZ
zY9DD7bH}rq9mg88+KgA$Zp!iWncuU2n1AuIa@=sWvUR-s`Qb{R*kk(SPU^`$6BXz8
zn#7yaFOIK%qGxyi`dYtm#&qqox0$h=pNi#u=M8zUG@bpiZ=3sT=1}Trr}39cC)H|v
zbL?W)=&s4zrh)7>L(|cc%$1#!zfL?HjpeP%T+x_a+jZ16b^iKOHxFEX$7d|8${H-*
zIrOJ5w&i$>*D>AKaIoYg`;{L@jM((Kt?$N$5OnuPqVvq**Nm}(f0wwOF%iX_Pba;V
z;m@wxX&NcV3?<1+u?A{y_DIj7#m3Af1rCE)o`D&Y3}0%7E;iX1yMDiS)sh0wKi!36
zL!Wmq?P^Ku&rK~HJd97KkLTRl>ScGFYZNlYytWnhmuu|)L&ND8_PmkayQb{HOY640
bno1(wj@u8DCVuFR|31B*4ek@pZJqxCDDe1x

literal 0
HcmV?d00001

diff --git a/vsixtest/Assets/Wide310x150Logo.scale-200.png b/vsixtest/Assets/Wide310x150Logo.scale-200.png
new file mode 100644
index 0000000000000000000000000000000000000000..288995b397fdbef1fb7e85afd71445d5de1952c5
GIT binary patch
literal 3204
zcmbVPeQXow8NYmBd90>}0NP?GhXW~VaeThm=a0tV#EwJMI!)6M3}|c4_Bl3=Kd>G0
z(GHx1wl<7(tP?FsOQkTilSo*iIvF%uArExJ73~P
zSv1xEy!U(Wd4A9D`FQV@W3@F^qJ@PEF$@z`Z!*BbFsS(^?B
zyiAzJ+q})bkgiQHWqEb*jJD-coHYr1^iocg)l!Qa{Xqs-l~6J}p-|##ZHYofskQ3$
zI0;xzXyhazBeXhIsg5A=%ufo@f)1yy&ScKS0;HF^!r_2UE^lpZEom(+@duma3awTv
zCrCL-%D_SvYWIcdHkmI}#50(fkUi)Qgx!80ju>g1za^}ff>JI8Z@^-iCiaCgg@TgF
z+vtE?Q9{VQUX&MW9SYYmGcxA14%N2@7FwBTD4N<(2{nWgV8$e3?-F=L^&FrtWn~(U_Q~~^uYiyeY6-KoTnfh9AWz@
zIKje0)u!_Lw)E}G!#kEfwKVdNt(UAf9*f>tEL_(=xco-T%jTi@7YlC3hs2ik%Le0H
ztj}RTeCF(5mwvi3_56>-yB?l;J>-1%!9~=fs|QcNG3J~a@JCu`4SB460s0ZO+##4fFUSGLcj_ja^fL4&BKALfb#$6$O?>P@qx2Agl^x0i&ugt
zsy5Pyu=()`7HRMG3IB7F1@`_
z+-!J%#i6e^U$e#+C%Q>_qVRzWRsG^W_n+@OcX@vzI&z;mzHNb!GQ?LWA(wtpqHqTM
z1OFw_{Zn?fD)p)`c`kOgv{de=v@suGRqY{N^U7gI1VF3*F=obwaXI6ob5__Yn
zVTguS!%(NI09J8x#AO_aW!9W7k*UvB;IWDFC3srwftr{kHj%g)fvnAm;&h_dnl~
    MY-
zf+K}sCe8qU6Ujs`3ua{U0Of$R_gVQBuUA
za0v=mu#vIOqiiAZOr&h*$WyOw&k-xr$;G4Ixa!#TJNr>95(h>l%)PUy4p+^SgR(uR
zta%k*?ny-+nAr8spEk1fo{J4i!b^Fia`N{_F6@zidA2ZTTrjl#^5Z-2KfB@Cu}l9s
z(*|Z2jc?p~vn2f)3y9i*7zJV1L{$?|&q)4oaT;uXi6>1GkRXVTOzAz(RHEmr=eFIi
z`}<>-Q?K0GN8!IYxeP1XKXO+jsJbp~o^);Bc;%b7Flpe7;1`Ny@3r7ZR;?R)aJt8C
ziNlEC<@3f_lIV4TwV}&e;D!Ee5_|e#g0LUh=5vmYWYm7&2h*M>QPKvGh9-)wfMMW3
z8J9b%1k7dzPzO0_NGQy92BZ^FR6R~6;^6?lqO;-QUP4BY%cG%3vEhbm#>4vIhPBh3
z-+pZGjh$x%Hp{?=FHsMp0&wNPlj00us{&`1ZOZTqs8%4X&xH=UDr*xyBW(Zp&Em94
zf)ZSfn#yg0N)>!1kWdkqJ^S*z0FF5|fj&qcE#Na|%OY0$uO>!&hP+1ywfD_WXk@4J(?MBftK7>$Nvqh@tDuarN%PrTLQ2Uzysx>UV=V
zk^RrDSvdQ?0;=hY67EgII-f4`t=+i*yS=Y~!XlqIy_4x&%+OdfbKOFPXS2X5%4R{N
z$SQMX^AK6(fA
+
+    
+
+    
+
diff --git a/vsixtest/MainPage.xaml.cpp b/vsixtest/MainPage.xaml.cpp
new file mode 100644
index 00000000..e67dcb83
--- /dev/null
+++ b/vsixtest/MainPage.xaml.cpp
@@ -0,0 +1,53 @@
+//
+// MainPage.xaml.cpp
+// Implementation of the MainPage class.
+//
+
+#include "pch.h"
+#include "MainPage.xaml.h"
+#include "sqlite3.h"
+
+using namespace vsixtest;
+
+using namespace Platform;
+using namespace Windows::Foundation;
+using namespace Windows::Foundation::Collections;
+using namespace Windows::UI::Xaml;
+using namespace Windows::UI::Xaml::Controls;
+using namespace Windows::UI::Xaml::Controls::Primitives;
+using namespace Windows::UI::Xaml::Data;
+using namespace Windows::UI::Xaml::Input;
+using namespace Windows::UI::Xaml::Media;
+using namespace Windows::UI::Xaml::Navigation;
+
+// The Blank Page item template is documented at http://go.microsoft.com/fwlink/?LinkId=402352&clcid=0x409
+
+MainPage::MainPage()
+{
+	InitializeComponent();
+	UseSQLite();
+}
+
+void MainPage::UseSQLite(void)
+{
+    int rc = SQLITE_OK;
+    sqlite3 *pDb = nullptr;
+
+    rc = sqlite3_open_v2("test.db", &pDb,
+	SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
+
+    if (rc != SQLITE_OK)
+	throw ref new FailureException("Failed to open database.");
+
+    rc = sqlite3_exec(pDb, "VACUUM;", nullptr, nullptr, nullptr);
+
+    if (rc != SQLITE_OK)
+	throw ref new FailureException("Failed to vacuum database.");
+
+    rc = sqlite3_close(pDb);
+
+    if (rc != SQLITE_OK)
+	throw ref new FailureException("Failed to close database.");
+
+    pDb = nullptr;
+}
diff --git a/vsixtest/MainPage.xaml.h b/vsixtest/MainPage.xaml.h
new file mode 100644
index 00000000..ea327a3e
--- /dev/null
+++ b/vsixtest/MainPage.xaml.h
@@ -0,0 +1,22 @@
+//
+// MainPage.xaml.h
+// Declaration of the MainPage class.
+//
+
+#pragma once
+
+#include "MainPage.g.h"
+
+namespace vsixtest
+{
+	/// 
    
+	/// An empty page that can be used on its own or navigated to within a Frame.
+	/// 
+	public ref class MainPage sealed
+	{
+	public:
+		MainPage();
+		void UseSQLite(void);
+
+	};
+}
diff --git a/vsixtest/Package.appxmanifest b/vsixtest/Package.appxmanifest
new file mode 100644
index 00000000..106b3f1e
--- /dev/null
+++ b/vsixtest/Package.appxmanifest
@@ -0,0 +1,49 @@
+
+
+
+
+  
+
+  
+
+  
+    vsixtest
+    mistachkin
+    Assets\StoreLogo.png
+  
+
+  
+    
+  
+
+  
+    
+  
+
+  
+    
+      
+        
+        
+      
+    
+  
+
+  
+    
+  
+
\ No newline at end of file
diff --git a/vsixtest/pch.cpp b/vsixtest/pch.cpp
new file mode 100644
index 00000000..97b544ec
--- /dev/null
+++ b/vsixtest/pch.cpp
@@ -0,0 +1,6 @@
+//
+// pch.cpp
+// Include the standard header and generate the precompiled header.
+//
+
+#include "pch.h"
diff --git a/vsixtest/pch.h b/vsixtest/pch.h
new file mode 100644
index 00000000..b793236d
--- /dev/null
+++ b/vsixtest/pch.h
@@ -0,0 +1,11 @@
+//
+// pch.h
+// Header for standard system include files.
+//
+
+#pragma once
+
+#include 
+#include 
+
+#include "App.xaml.h"
diff --git a/vsixtest/vsixtest.sln b/vsixtest/vsixtest.sln
new file mode 100644
index 00000000..1ab6e064
--- /dev/null
+++ b/vsixtest/vsixtest.sln
@@ -0,0 +1,39 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.24720.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "vsixtest", "vsixtest.vcxproj", "{60BB14A5-0871-4656-BC38-4F0958230F9A}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|ARM = Debug|ARM
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|ARM = Release|ARM
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|ARM.ActiveCfg = Debug|ARM
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|ARM.Build.0 = Debug|ARM
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|ARM.Deploy.0 = Debug|ARM
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x64.ActiveCfg = Debug|x64
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x64.Build.0 = Debug|x64
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x64.Deploy.0 = Debug|x64
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x86.ActiveCfg = Debug|Win32
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x86.Build.0 = Debug|Win32
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Debug|x86.Deploy.0 = Debug|Win32
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|ARM.ActiveCfg = Release|ARM
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|ARM.Build.0 = Release|ARM
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|ARM.Deploy.0 = Release|ARM
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x64.ActiveCfg = Release|x64
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x64.Build.0 = Release|x64
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x64.Deploy.0 = Release|x64
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x86.ActiveCfg = Release|Win32
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x86.Build.0 = Release|Win32
+		{60BB14A5-0871-4656-BC38-4F0958230F9A}.Release|x86.Deploy.0 = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
diff --git a/vsixtest/vsixtest.tcl b/vsixtest/vsixtest.tcl
new file mode 100644
index 00000000..5dce821d
--- /dev/null
+++ b/vsixtest/vsixtest.tcl
@@ -0,0 +1,373 @@
+#!/usr/bin/tclsh
+#
+# This script is used to quickly test a VSIX (Visual Studio Extension) file
+# with Visual Studio 2015 on Windows.
+#
+# PREREQUISITES
+#
+# 1. This tool is Windows only.
+#
+# 2. This tool must be executed with "elevated administrator" privileges.
+#
+# 3. Tcl 8.4 and later are supported, earlier versions have not been tested.
+#
+# 4. The "sqlite-UWP-output.vsix" file is assumed to exist in the parent
+#    directory of the directory containing this script.  The [optional] first
+#    command line argument to this script may be used to specify an alternate
+#    file.  However, currently, the file must be compatible with both Visual
+#    Studio 2015 and the Universal Windows Platform.
+#
+# 5. The "VERSION" file is assumed to exist in the parent directory of the
+#    directory containing this script.  It must contain a version number that
+#    matches the VSIX file being tested.
+#
+# 6. The temporary directory specified in the TEMP or TMP environment variables
+#    must refer to an existing directory writable by the current user.
+#
+# 7. The VS140COMNTOOLS environment variable must refer to the Visual Studio
+#    2015 common tools directory.
+#
+# USAGE
+#
+# The first argument to this script is optional.  If specified, it must be the
+# name of the VSIX file to test.
+#
+package require Tcl 8.4
+
+proc fail { {error ""} {usage false} } {
+  if {[string length $error] > 0} then {
+    puts stdout $error
+    if {!$usage} then {exit 1}
+  }
+
+  puts stdout "usage:\
+[file tail [info nameofexecutable]]\
+[file tail [info script]] \[vsixFile\]"
+
+  exit 1
+}
+
+proc isWindows {} {
+  #
+  # NOTE: Returns non-zero only when running on Windows.
+  #
+  return [expr {[info exists ::tcl_platform(platform)] && \
+      $::tcl_platform(platform) eq "windows"}]
+}
+
+proc isAdministrator {} {
+  #
+  # NOTE: Returns non-zero only when running as "elevated administrator".
+  #
+  if {[isWindows]} then {
+    if {[catch {exec -- whoami /groups} groups] == 0} then {
+      set groups [string map [list \r\n \n] $groups]
+
+      foreach group [split $groups \n] {
+        #
+        # NOTE: Match this group line against the "well-known" SID for
+        #       the "Administrators" group on Windows.
+        #
+        if {[regexp -- {\sS-1-5-32-544\s} $group]} then {
+          #
+          # NOTE: Match this group line against the attributes column
+          #       sub-value that should be present when running with
+          #       elevated administrator credentials.
+          #
+          if {[regexp -- {\sEnabled group(?:,|\s)} $group]} then {
+            return true
+          }
+        }
+      }
+    }
+  }
+
+  return false
+}
+
+proc getEnvironmentVariable { name } {
+  #
+  # NOTE: Returns the value of the specified environment variable or an empty
+  #       string for environment variables that do not exist in the current
+  #       process environment.
+  #
+  return [expr {[info exists ::env($name)] ? $::env($name) : ""}]
+}
+
+proc getTemporaryPath {} {
+  #
+  # NOTE: Returns the normalized path to the first temporary directory found
+  #       in the typical set of environment variables used for that purpose
+  #       or an empty string to signal a failure to locate such a directory.
+  #
+  set names [list]
+
+  foreach name [list TEMP TMP] {
+    lappend names [string toupper $name] [string tolower $name] \
+        [string totitle $name]
+  }
+
+  foreach name $names {
+    set value [getEnvironmentVariable $name]
+
+    if {[string length $value] > 0} then {
+      return [file normalize $value]
+    }
+  }
+
+  return ""
+}
+
+proc appendArgs { args } {
+  #
+  # NOTE: Returns all passed arguments joined together as a single string
+  #       with no intervening spaces between arguments.
+  #
+  eval append result $args
+}
+
+proc readFile { fileName } {
+  #
+  # NOTE: Reads and returns the entire contents of the specified file, which
+  #       may contain binary data.
+  #
+  set file_id [open $fileName RDONLY]
+  fconfigure $file_id -encoding binary -translation binary
+  set result [read $file_id]
+  close $file_id
+  return $result
+}
+
+proc writeFile { fileName data } {
+  #
+  # NOTE: Writes the entire contents of the specified file, which may contain
+  #       binary data.
+  #
+  set file_id [open $fileName {WRONLY CREAT TRUNC}]
+  fconfigure $file_id -encoding binary -translation binary
+  puts -nonewline $file_id $data
+  close $file_id
+  return ""
+}
+
+proc putsAndEval { command } {
+  #
+  # NOTE: Outputs a command to the standard output channel and then evaluates
+  #       it in the callers context.
+  #
+  catch {
+    puts stdout [appendArgs "Running: " [lrange $command 1 end] ...\n]
+  }
+
+  return [uplevel 1 $command]
+}
+
+proc isBadDirectory { directory } {
+  #
+  # NOTE: Returns non-zero if the directory is empty, does not exist, -OR- is
+  #       not a directory.
+  #
+  catch {
+    puts stdout [appendArgs "Checking directory \"" $directory \"...\n]
+  }
+
+  return [expr {[string length $directory] == 0 || \
+      ![file exists $directory] || ![file isdirectory $directory]}]
+}
+
+proc isBadFile { fileName } {
+  #
+  # NOTE: Returns non-zero if the file name is empty, does not exist, -OR- is
+  #       not a regular file.
+  #
+  catch {
+    puts stdout [appendArgs "Checking file \"" $fileName \"...\n]
+  }
+
+  return [expr {[string length $fileName] == 0 || \
+      ![file exists $fileName] || ![file isfile $fileName]}]
+}
+
+#
+# NOTE: This is the entry point for this script.
+#
+set script [file normalize [info script]]
+
+if {[string length $script] == 0} then {
+  fail "script file currently being evaluated is unknown" true
+}
+
+if {![isWindows]} then {
+  fail "this tool only works properly on Windows"
+}
+
+if {![isAdministrator]} then {
+  fail "this tool must run with \"elevated administrator\" privileges"
+}
+
+set path [file normalize [file dirname $script]]
+set argc [llength $argv]; if {$argc > 1} then {fail "" true}
+
+if {$argc == 1} then {
+  set vsixFileName [lindex $argv 0]
+} else {
+  set vsixFileName [file join \
+      [file dirname $path] sqlite-UWP-output.vsix]
+}
+
+###############################################################################
+
+if {[isBadFile $vsixFileName]} then {
+  fail [appendArgs \
+      "VSIX file \"" $vsixFileName "\" does not exist"]
+}
+
+set versionFileName [file join [file dirname $path] VERSION]
+
+if {[isBadFile $versionFileName]} then {
+  fail [appendArgs \
+      "Version file \"" $versionFileName "\" does not exist"]
+}
+
+set projectTemplateFileName [file join $path vsixtest.vcxproj.data]
+
+if {[isBadFile $projectTemplateFileName]} then {
+  fail [appendArgs \
+      "Project template file \"" $projectTemplateFileName \
+      "\" does not exist"]
+}
+
+set envVarName VS140COMNTOOLS
+set vsDirectory [getEnvironmentVariable $envVarName]
+
+if {[isBadDirectory $vsDirectory]} then {
+  fail [appendArgs \
+      "Visual Studio 2015 directory \"" $vsDirectory \
+      "\" from environment variable \"" $envVarName \
+      "\" does not exist"]
+}
+
+set vsixInstaller [file join \
+    [file dirname $vsDirectory] IDE VSIXInstaller.exe]
+
+if {[isBadFile $vsixInstaller]} then {
+  fail [appendArgs \
+      "Visual Studio 2015 VSIX installer \"" $vsixInstaller \
+      "\" does not exist"]
+}
+
+set envVarName ProgramFiles
+set programFiles [getEnvironmentVariable $envVarName]
+
+if {[isBadDirectory $programFiles]} then {
+  fail [appendArgs \
+      "Program Files directory \"" $programFiles \
+      "\" from environment variable \"" $envVarName \
+      "\" does not exist"]
+}
+
+set msBuild [file join $programFiles MSBuild 14.0 Bin MSBuild.exe]
+
+if {[isBadFile $msBuild]} then {
+  fail [appendArgs \
+      "MSBuild v14.0 executable file \"" $msBuild \
+      "\" does not exist"]
+}
+
+set temporaryDirectory [getTemporaryPath]
+
+if {[isBadDirectory $temporaryDirectory]} then {
+  fail [appendArgs \
+      "Temporary directory \"" $temporaryDirectory \
+      "\" does not exist"]
+}
+
+###############################################################################
+
+set installLogFileName [appendArgs \
+    [file rootname [file tail $vsixFileName]] \
+    -install- [pid] .log]
+
+set commands(1) [list exec [file nativename $vsixInstaller]]
+
+lappend commands(1) /quiet /norepair
+lappend commands(1) [appendArgs /logFile: $installLogFileName]
+lappend commands(1) [file nativename $vsixFileName]
+
+###############################################################################
+
+set buildLogFileName [appendArgs \
+    [file rootname [file tail $vsixFileName]] \
+    -build-%configuration%-%platform%- [pid] .log]
+
+set commands(2) [list exec [file nativename $msBuild]]
+
+lappend commands(2) [file nativename [file join $path vsixtest.sln]]
+lappend commands(2) /target:Rebuild
+lappend commands(2) /property:Configuration=%configuration%
+lappend commands(2) /property:Platform=%platform%
+
+lappend commands(2) [appendArgs \
+    /logger:FileLogger,Microsoft.Build.Engine\;Logfile= \
+    [file nativename [file join $temporaryDirectory \
+    $buildLogFileName]] \;Verbosity=diagnostic]
+
+###############################################################################
+
+set uninstallLogFileName [appendArgs \
+    [file rootname [file tail $vsixFileName]] \
+    -uninstall- [pid] .log]
+
+set commands(3) [list exec [file nativename $vsixInstaller]]
+
+lappend commands(3) /quiet /norepair
+lappend commands(3) [appendArgs /logFile: $uninstallLogFileName]
+lappend commands(3) [appendArgs /uninstall:SQLite.UWP.2015]
+
+###############################################################################
+
+if {1} then {
+  catch {
+    puts stdout [appendArgs \
+        "Install log: \"" [file nativename [file join \
+        $temporaryDirectory $installLogFileName]] \"\n]
+  }
+
+  catch {
+    puts stdout [appendArgs \
+        "Build logs: \"" [file nativename [file join \
+        $temporaryDirectory $buildLogFileName]] \"\n]
+  }
+
+  catch {
+    puts stdout [appendArgs \
+        "Uninstall log: \"" [file nativename [file join \
+        $temporaryDirectory $uninstallLogFileName]] \"\n]
+  }
+}
+
+###############################################################################
+
+if {1} then {
+  putsAndEval $commands(1)
+
+  set versionNumber [string trim [readFile $versionFileName]]
+  set data [readFile $projectTemplateFileName]
+  set data [string map [list %versionNumber% $versionNumber] $data]
+
+  set projectFileName [file join $path vsixtest.vcxproj]
+  writeFile $projectFileName $data
+
+  set platforms [list x86 x64 ARM]
+  set configurations [list Debug Release]
+
+  foreach platform $platforms {
+    foreach configuration $configurations {
+      putsAndEval [string map [list \
+          %platform% $platform %configuration% $configuration] \
+          $commands(2)]
+    }
+  }
+
+  putsAndEval $commands(3)
+}
diff --git a/vsixtest/vsixtest.vcxproj.data b/vsixtest/vsixtest.vcxproj.data
new file mode 100644
index 00000000..a64584c3
--- /dev/null
+++ b/vsixtest/vsixtest.vcxproj.data
@@ -0,0 +1,198 @@
+
+
+  
+    {60bb14a5-0871-4656-bc38-4f0958230f9a}
+    vsixtest
+    en-US
+    14.0
+    true
+    Windows Store
+    10.0.10586.0
+    10.0.10586.0
+    10.0
+  
+  
+  
+    
+      Debug
+      ARM
+    
+    
+      Debug
+      Win32
+    
+    
+      Debug
+      x64
+    
+    
+      Release
+      ARM
+    
+    
+      Release
+      Win32
+    
+    
+      Release
+      x64
+    
+  
+  
+    Application
+    true
+    v140
+  
+  
+    Application
+    true
+    v140
+  
+  
+    Application
+    true
+    v140
+  
+  
+    Application
+    false
+    true
+    v140
+    true
+  
+  
+    Application
+    false
+    true
+    v140
+    true
+  
+  
+    Application
+    false
+    true
+    v140
+    true
+  
+  
+  
+  
+  
+    
+    
+  
+  
+    
+    
+  
+  
+    
+    
+  
+  
+    
+    
+  
+  
+    
+    
+  
+  
+    
+    
+  
+  
+  
+    vsixtest_TemporaryKey.pfx
+  
+  
+    
+      /bigobj %(AdditionalOptions)
+      4453;28204
+    
+  
+  
+    
+      /bigobj %(AdditionalOptions)
+      4453;28204
+    
+  
+  
+    
+      /bigobj %(AdditionalOptions)
+      4453;28204
+    
+  
+  
+    
+      /bigobj %(AdditionalOptions)
+      4453;28204
+    
+  
+  
+    
+      /bigobj %(AdditionalOptions)
+      4453;28204
+    
+  
+  
+    
+      /bigobj %(AdditionalOptions)
+      4453;28204
+    
+  
+  
+    
+    
+      App.xaml
+    
+    
+      MainPage.xaml
+    
+  
+  
+    
+      Designer
+    
+    
+      Designer
+    
+  
+  
+    
+      Designer
+    
+    
+  
+  
+    
+    
+    
+    
+    
+    
+    
+  
+  
+    
+      App.xaml
+    
+    
+      MainPage.xaml
+    
+    
+      Create
+      Create
+      Create
+      Create
+      Create
+      Create
+    
+  
+  
+    
+  
+  
+  
+  
+
\ No newline at end of file
diff --git a/vsixtest/vsixtest.vcxproj.filters b/vsixtest/vsixtest.vcxproj.filters
new file mode 100644
index 00000000..ac1dfca4
--- /dev/null
+++ b/vsixtest/vsixtest.vcxproj.filters
@@ -0,0 +1,57 @@
+
+
+  
+    
+      60bb14a5-0871-4656-bc38-4f0958230f9a
+    
+    
+      e6271362-8f96-476d-907f-4da227b02435
+      bmp;fbx;gif;jpg;jpeg;tga;tiff;tif;png
+    
+  
+  
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+      Assets
+    
+    
+      Assets
+    
+    
+      Assets
+    
+    
+      Assets
+    
+    
+      Assets
+    
+    
+      Assets
+    
+    
+      Assets
+    
+  
+  
+    
+  
+  
+    
+  
+  
+    
+  
+
\ No newline at end of file
diff --git a/vsixtest/vsixtest_TemporaryKey.pfx b/vsixtest/vsixtest_TemporaryKey.pfx
new file mode 100644
index 0000000000000000000000000000000000000000..e6787bcad0527ecc86a43ea007ba8776c4b26215
GIT binary patch
literal 2520
zcmY*ac|6o>7ypeJ!=Nx|G_o{dY||aaG7*=uRdx}&bTP8McB-MVjL1zQ`_@>78g~pM
zA?Xf@v>;?lA^YyizFzOI`+48{e%{ade4q22?>Xl=&p*%i!BY{vTo5Rpib#PYl;Ta}
zw~jz~Aca&!42+7PfHWCT<=*|j6!!v*$~^
zY4Gr)FkYQeF~j*M(BHYZc)DOz*q}&)S9!g{H0k1#-rUN(n(H$|6{LPDexJiZ-kU&FbKR@6F34qKTFux@xniVPiG3Im6+bkmhv?r))l&Dh
z+rGAjRwjvXTA0-DCAE3)udY3mp1vfqB3r%W%g$US-xNB~H$WB(SFOM2DM~Wb`WBSD
zrsmnx6crY!JKAQdDOCEnQ5#8nJ>48`5K7FJFeu*hS-7x2Y$2yv`B>Nx-3hmP;(RFy
z>4r0O+foVl^mk0SOzpMr@0RX8mCUL5tlyl{MKF0b;i|tRD>LlIMi~&oU!IG7AxM`F
zd_#_lI(}YT;&5QhZ+qcdpq3+<;;1m9=S>c>udYxaep@Dvv^}7m$fO>mSS38a5RhJ6
zp-1GMa^ics&P9wCvgLhqgvMPaCFq6dHGY*&sET#^P??s~ts;S1JG!ou^>D}dQmSjt1sMDg_CQC%AyT@e
zMd7E`TfZE>FFm5?)Q$>8wZZ0VB>N!J@)r83mQBqaF}Y{Uwi%mz)NkpyE3B;@=v5}}d$e3g6>CQ4uG3CVAPb8E{o(WDWx64dZr
zp~GyP4l06CQ6(2E;~nvGUC7z+hVM-7p?<&URkIz5MT@xX$>*0mA1g!F3`(i~we1?>
zCme&g0u{bUVnnHIuSxUgYmd71`BL_R*$o++|bK}tOlc$|~C=(DW
z&rQ$s(beUf`fbWSPAX_@_3*gC*Pkr9Yphb#z6pFcxnVw45Y=$<_o5u#Z_=V`n9!zz
zu)AsO#IDlGltwGD^nU@JbmfQa8rJ={g`Km$J)Ta=Pd1Z6YsPH0D+QJE2dCV$Z;mRI
zoyfEnsZE4uRpk571wJ}vL|(m}H{2-3%p*&Vn2nbA%Z=y0XEDc((lKYlJv)TC>;1l{
z4O+%tJU7;gv*DgXg{FE!g~fLkvE1lRk_skAi?6>_chzTNljJ+_axug1@=%PeWB%0Y
zZx${;(*_#DG_0;p@A?Y9U^3>$)UmrQ#fxUW;;Vns_?~!>9+eAzIa9V4%IASYmApcF
z%JBQ$bLiT+QY3$;Vb67wcR@TzA`POm7enC3ZEnyb6f!7AO1VbsZpaJYTo8C3S`57|
zib#6NF2;0k*oxm5IQb~biC%Ilb4$wJT?c9XTEws}tY+sLtuz1`yKZw@cF0ubssp1c
zsrJo*BF$6j?XPg1Y{hG@mGJXpv)&=crtdfV3DmC&tEcKL<)ULX(L`jYyE!8px{zbP
zdcV+{=k8NPMs9i?<7q~8ki-zKG0b$`U)-gMXFKhc0}U67EBYJ~Z%a(oY5lr`<-5oy
zH+1oHbv!E+fu0J{^=?;<>A;KpBbNvP0D2z#VV}p$e#h-WBvX
zKm#;QaE}AjK_h_~9B>A}10Eo701>PvfpupppN3TP0xDgON^?RXv4sJk^KVx0KFp@SC#Ri>(&9VxcHC3dS?;)F)TWn|xj%(VJ3
z?BI|MHp9_6I#{*17;l37liej`h=-Ca6?L&By%5V;q{3x*D>k(vPJVCyEv&7uJXYo%
z`;aL*(Xf#x!O0$>BRgIW^>r~tePV%+pyjw**
zk#KpMa)N4+fIjYL{witoF=GL3nQ#>6VURDreL&
z;+XWM^-=}m0p0s-z#CP%=~x~n{!7Wma!&{A9~LhUX0X)%-UCX(CLz2T92^D@sI+U@p4S%bFf
zuBKCWyS+m`-R;8L+8BouJXjmaBoWznxBFaMql1&t9#(R1%2LWMEzjnN&_X$kt647=
z?@6{FoIXEqpW|P0Yan0FI)Mqy{4^nXZ?_B}Y!}4;hL|`Wwnw?p8qjp^wxiS`Z8(W%
ze_U458SQ|L%qoqZZ!)U>GH3Z<}rm!iAu)PJfu6%(D`ZS;c)h+(>7zq-s~-0TiS8^=Zq#g60fDwvpi)UWfH^
zk-&0}S@!2@#YsE()`jPgMQ-2rjQtJ5^3%|_Ch^?iMYS>zd>0rju3O#f6!sHHJ!^tBb
zDZ7Q;qZ(Sillt1*f-02@<4O^;t9Am&{hY+1%H+{^ubidQN*j-*o#kUV$N5(&)pqV<
z>m2PDzKKBfp8fGUMJXRfnKRBDv6{3k@yF}nrSUv)Zsns;E;LLu%ZHQS71R@}Q`~3B
lyiSSy3ZJrriAr|rpzb`|H|fj2WbtYE^sH7-kL-^J_itHiY_0$R

literal 0
HcmV?d00001