From 12069ec91327d2374ed92afd5377adaa7fbcc294 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 17 Aug 2011 13:17:32 -0400 Subject: [PATCH] track 3.7.7.1 --- Makefile.in | 11 +- Makefile.msc | 875 ++++++++++ VERSION | 2 +- config.h.in | 3 + configure | 108 +- configure.ac | 3 +- ext/fts3/fts3.c | 3103 ++++++++++++++++++++++-------------- ext/fts3/fts3Int.h | 227 ++- ext/fts3/fts3_aux.c | 14 +- ext/fts3/fts3_expr.c | 127 +- ext/fts3/fts3_hash.c | 1 + ext/fts3/fts3_icu.c | 4 +- ext/fts3/fts3_porter.c | 3 +- ext/fts3/fts3_snippet.c | 186 +-- ext/fts3/fts3_term.c | 369 +++++ ext/fts3/fts3_test.c | 321 ++++ ext/fts3/fts3_tokenizer.c | 8 +- ext/fts3/fts3_tokenizer1.c | 3 +- ext/fts3/fts3_write.c | 1173 ++++++++++---- ext/rtree/rtree.c | 312 ++-- ext/rtree/rtree1.test | 81 + main.mk | 15 +- manifest | 262 +-- manifest.uuid | 2 +- src/alter.c | 4 +- src/attach.c | 18 +- src/btree.c | 71 +- src/btree.h | 1 + src/build.c | 15 +- src/date.c | 164 +- src/delete.c | 9 +- src/expr.c | 90 +- src/fkey.c | 14 +- src/func.c | 30 +- src/global.c | 6 +- src/insert.c | 13 + src/main.c | 303 +++- src/malloc.c | 11 +- src/os_unix.c | 73 +- src/os_win.c | 242 +-- src/pager.c | 32 +- src/pcache.c | 7 + src/pcache1.c | 2 +- src/pragma.c | 20 +- src/select.c | 4 +- src/shell.c | 17 +- src/sqlite.h.in | 433 ++++- src/sqliteInt.h | 31 +- src/tclsqlite.c | 14 +- src/test1.c | 157 ++ src/test3.c | 2 +- src/test_config.c | 10 +- src/test_demovfs.c | 13 +- src/test_journal.c | 12 +- src/test_malloc.c | 30 + src/test_multiplex.c | 149 +- src/test_quota.c | 4 +- src/test_thread.c | 6 +- src/test_vfs.c | 133 +- src/test_vfstrace.c | 94 ++ src/tokenize.c | 19 +- src/trigger.c | 5 +- src/update.c | 11 +- src/utf.c | 2 +- src/util.c | 32 +- src/vdbe.c | 117 +- src/vdbe.h | 4 +- src/vdbeInt.h | 2 +- src/vdbeapi.c | 44 +- src/vdbeaux.c | 275 ++-- src/vdbeblob.c | 5 +- src/vtab.c | 166 +- src/wal.c | 85 +- src/wal.h | 6 +- test/8_3_names.test | 197 +++ test/alter.test | 2 +- test/cast.test | 10 +- test/e_createtable.test | 6 +- test/e_delete.test | 2 +- test/e_expr.test | 5 +- test/e_insert.test | 2 +- test/e_select.test | 6 +- test/e_update.test | 5 +- test/e_uri.test | 456 ++++++ test/enc4.test | 276 ++-- test/expr.test | 111 +- test/fkey3.test | 106 ++ test/fts3atoken.test | 13 + test/fts3auto.test | 658 ++++++++ test/fts3aux1.test | 6 +- test/fts3conf.test | 139 ++ test/fts3corrupt.test | 7 + test/fts3defer.test | 29 +- test/fts3defer2.test | 16 +- test/fts3matchinfo.test | 14 +- test/fts3prefix.test | 203 +++ test/fts3rnd.test | 68 +- test/fts3sort.test | 162 ++ test/fts4aa.test | 134 +- test/hook.test | 28 + test/insert4.test | 60 + test/lock_common.tcl | 4 +- test/malloc.test | 12 + test/multiplex.test | 41 + test/nan.test | 16 +- test/oserror.test | 1 + test/pager1.test | 12 +- test/pagerfault.test | 5 + test/permutations.test | 5 +- test/quota.test | 3 + test/sync.test | 30 +- test/tester.tcl | 40 +- test/tkt-2d1a5c67d.test | 127 ++ test/tkt-91e2e8ba6f.test | 52 + test/tkt-bd484a090c.test | 38 + test/tkt3838.test | 2 +- test/tkt3922.test | 8 +- test/trace2.test | 6 +- test/triggerC.test | 70 +- test/types3.test | 2 +- test/unordered.test | 3 + test/uri.test | 310 ++++ test/wal2.test | 13 + test/wal5.test | 9 +- test/wal7.test | 118 ++ test/walro.test | 161 ++ tool/build-shell.sh | 21 + tool/getlock.c | 134 ++ tool/lemon.c | 43 +- tool/mksqlite3c.tcl | 2 + tool/mksqlite3h.tcl | 3 + tool/omittest.tcl | 22 +- tool/shell1.test | 10 +- tool/showdb.c | 2 +- tool/symbols.sh | 34 + tool/tostr.awk | 9 + tool/warnings.sh | 14 + 137 files changed, 11121 insertions(+), 3202 deletions(-) create mode 100644 Makefile.msc create mode 100644 ext/fts3/fts3_term.c create mode 100644 ext/fts3/fts3_test.c create mode 100644 test/8_3_names.test create mode 100644 test/e_uri.test create mode 100644 test/fts3auto.test create mode 100644 test/fts3conf.test create mode 100644 test/fts3prefix.test create mode 100644 test/fts3sort.test create mode 100644 test/tkt-2d1a5c67d.test create mode 100644 test/tkt-91e2e8ba6f.test create mode 100644 test/tkt-bd484a090c.test create mode 100644 test/uri.test create mode 100644 test/wal7.test create mode 100644 test/walro.test create mode 100644 tool/build-shell.sh create mode 100644 tool/getlock.c create mode 100644 tool/symbols.sh create mode 100644 tool/tostr.awk create mode 100644 tool/warnings.sh diff --git a/Makefile.in b/Makefile.in index f3239f3d..4a81e156 100644 --- a/Makefile.in +++ b/Makefile.in @@ -378,7 +378,9 @@ TESTSRC = \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_wholenumber.c \ - $(TOP)/src/test_wsd.c + $(TOP)/src/test_wsd.c \ + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c # Source code to the library files needed by the test fixture # @@ -420,6 +422,7 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ + $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c @@ -828,12 +831,12 @@ fts3_hash.lo: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) fts3_icu.lo: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c -fts3_snippet.lo: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) - $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c - fts3_porter.lo: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c +fts3_snippet.lo: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c + fts3_tokenizer.lo: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c diff --git a/Makefile.msc b/Makefile.msc new file mode 100644 index 00000000..a6d1bf4c --- /dev/null +++ b/Makefile.msc @@ -0,0 +1,875 @@ +# +# nmake Makefile for SQLite +# + +# The toplevel directory of the source tree. This is the directory +# that contains this "Makefile.msc". +# +TOP = . + +# Set this non-0 to create and use the SQLite amalgamation file. +# +USE_AMALGAMATION = 1 + +# Version numbers and release number for the SQLite being compiled. +# +VERSION = 3.7 +VERSION_NUMBER = 3007007 +RELEASE = 3.7.7 + +# C Compiler and options for use in building executables that +# will run on the platform that is doing the build. +# +BCC = cl.exe -O2 + +# C Compile and options for use in building executables that +# will run on the target platform. (BCC and TCC are usually the +# same unless your are cross-compiling.) +# +TCC = cl.exe -W3 -O2 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src -fp:precise + +# The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in +# any extension header files by default. For non-amalgamation +# builds, we need to make sure the compiler can find these. +# +!IF $(USE_AMALGAMATION)==0 +TCC = $(TCC) -I$(TOP)\ext\fts3 +TCC = $(TCC) -I$(TOP)\ext\rtree +!ENDIF + +# Define -DNDEBUG to compile without debugging (i.e., for production usage) +# Omitting the define will cause extra debugging code to be inserted and +# includes extra comments when "EXPLAIN stmt" is used. +# +TCC = $(TCC) -DNDEBUG + +# The library that programs using TCL must link against. +# +LIBTCL = tcl85.lib +TCLINCDIR = c:\tcl\include +TCLLIBDIR = c:\tcl\lib + +# This is the command to use for tclsh - normally just "tclsh", but we may +# know the specific version we want to use +# +TCLSH_CMD = tclsh85 + +# Compiler options needed for programs that use the readline() library. +# +READLINE_FLAGS = -DHAVE_READLINE=0 + +# The library that programs using readline() must link against. +# +LIBREADLINE = + +# Should the database engine be compiled threadsafe +# +TCC = $(TCC) -DSQLITE_THREADSAFE=1 + +# Do threads override each others locks by default (1), or do we test (-1) +# +TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1 + +# Any target libraries which libsqlite must be linked against +# +TLIBS = + +# Flags controlling use of the in memory btree implementation +# +# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to +# default to file, 2 to default to memory, and 3 to force temporary +# tables to always be in memory. +# +TCC = $(TCC) -DSQLITE_TEMP_STORE=1 + +# Enable/disable loadable extensions, and other optional features +# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*). +# The same set of OMIT and ENABLE flags should be passed to the +# LEMON parser generator and the mkkeywordhash tool as well. + +# BEGIN standard options +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 +# END standard options + +# BEGIN required Windows option +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100 +# END required Windows option + +TCC = $(TCC) $(OPT_FEATURE_FLAGS) + +# Add in any optional parameters specified on the make commane line +# ie. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". +TCC = $(TCC) $(OPTS) + +# libtool compile/link +LTCOMPILE = $(TCC) -Fo$@ +LTLIB = lib.exe +LTLINK = $(TCC) -Fe$@ + +# If a platform was set, force the linker to target that. +# Note that the vcvars*.bat family of batch files typically +# set this for you. Otherwise, the linker will attempt +# to deduce the binary type based on the object files. +!IF "$(PLATFORM)"!="" +LTLINKOPTS = /MACHINE:$(PLATFORM) +LTLIBOPTS = /MACHINE:$(PLATFORM) +!ENDIF + +# nawk compatible awk. +NAWK = .\gawk.exe + +# You should not have to change anything below this line +############################################################################### + +# Object files for the SQLite library (non-amalgamation). +# +LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ + backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ + callback.lo complete.lo ctime.lo date.lo delete.lo \ + expr.lo fault.lo fkey.lo \ + fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo fts3_porter.lo \ + fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo fts3_write.lo \ + func.lo global.lo hash.lo \ + icu.lo insert.lo journal.lo legacy.lo loadext.lo \ + main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ + memjournal.lo \ + mutex.lo mutex_noop.lo mutex_os2.lo mutex_unix.lo mutex_w32.lo \ + notify.lo opcodes.lo os.lo os_os2.lo os_unix.lo os_win.lo \ + pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ + random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ + table.lo tokenize.lo trigger.lo \ + update.lo util.lo vacuum.lo \ + vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbetrace.lo \ + wal.lo walker.lo where.lo utf.lo vtab.lo + +# Object files for the amalgamation. +# +LIBOBJS1 = sqlite3.lo + +# Determine the real value of LIBOBJ based on the 'configure' script +# +!IF $(USE_AMALGAMATION)==0 +LIBOBJ = $(LIBOBJS0) +!ELSE +LIBOBJ = $(LIBOBJS1) +!ENDIF + +# All of the source code files. +# +SRC = \ + $(TOP)\src\alter.c \ + $(TOP)\src\analyze.c \ + $(TOP)\src\attach.c \ + $(TOP)\src\auth.c \ + $(TOP)\src\backup.c \ + $(TOP)\src\bitvec.c \ + $(TOP)\src\btmutex.c \ + $(TOP)\src\btree.c \ + $(TOP)\src\btree.h \ + $(TOP)\src\btreeInt.h \ + $(TOP)\src\build.c \ + $(TOP)\src\callback.c \ + $(TOP)\src\complete.c \ + $(TOP)\src\ctime.c \ + $(TOP)\src\date.c \ + $(TOP)\src\delete.c \ + $(TOP)\src\expr.c \ + $(TOP)\src\fault.c \ + $(TOP)\src\fkey.c \ + $(TOP)\src\func.c \ + $(TOP)\src\global.c \ + $(TOP)\src\hash.c \ + $(TOP)\src\hash.h \ + $(TOP)\src\hwtime.h \ + $(TOP)\src\insert.c \ + $(TOP)\src\journal.c \ + $(TOP)\src\legacy.c \ + $(TOP)\src\loadext.c \ + $(TOP)\src\main.c \ + $(TOP)\src\malloc.c \ + $(TOP)\src\mem0.c \ + $(TOP)\src\mem1.c \ + $(TOP)\src\mem2.c \ + $(TOP)\src\mem3.c \ + $(TOP)\src\mem5.c \ + $(TOP)\src\memjournal.c \ + $(TOP)\src\mutex.c \ + $(TOP)\src\mutex.h \ + $(TOP)\src\mutex_noop.c \ + $(TOP)\src\mutex_os2.c \ + $(TOP)\src\mutex_unix.c \ + $(TOP)\src\mutex_w32.c \ + $(TOP)\src\notify.c \ + $(TOP)\src\os.c \ + $(TOP)\src\os.h \ + $(TOP)\src\os_common.h \ + $(TOP)\src\os_os2.c \ + $(TOP)\src\os_unix.c \ + $(TOP)\src\os_win.c \ + $(TOP)\src\pager.c \ + $(TOP)\src\pager.h \ + $(TOP)\src\parse.y \ + $(TOP)\src\pcache.c \ + $(TOP)\src\pcache.h \ + $(TOP)\src\pcache1.c \ + $(TOP)\src\pragma.c \ + $(TOP)\src\prepare.c \ + $(TOP)\src\printf.c \ + $(TOP)\src\random.c \ + $(TOP)\src\resolve.c \ + $(TOP)\src\rowset.c \ + $(TOP)\src\select.c \ + $(TOP)\src\status.c \ + $(TOP)\src\shell.c \ + $(TOP)\src\sqlite.h.in \ + $(TOP)\src\sqlite3ext.h \ + $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteLimit.h \ + $(TOP)\src\table.c \ + $(TOP)\src\tclsqlite.c \ + $(TOP)\src\tokenize.c \ + $(TOP)\src\trigger.c \ + $(TOP)\src\utf.c \ + $(TOP)\src\update.c \ + $(TOP)\src\util.c \ + $(TOP)\src\vacuum.c \ + $(TOP)\src\vdbe.c \ + $(TOP)\src\vdbe.h \ + $(TOP)\src\vdbeapi.c \ + $(TOP)\src\vdbeaux.c \ + $(TOP)\src\vdbeblob.c \ + $(TOP)\src\vdbemem.c \ + $(TOP)\src\vdbetrace.c \ + $(TOP)\src\vdbeInt.h \ + $(TOP)\src\vtab.c \ + $(TOP)\src\wal.c \ + $(TOP)\src\wal.h \ + $(TOP)\src\walker.c \ + $(TOP)\src\where.c + +# Source code for extensions +# +SRC = $(SRC) \ + $(TOP)\ext\fts1\fts1.c \ + $(TOP)\ext\fts1\fts1.h \ + $(TOP)\ext\fts1\fts1_hash.c \ + $(TOP)\ext\fts1\fts1_hash.h \ + $(TOP)\ext\fts1\fts1_porter.c \ + $(TOP)\ext\fts1\fts1_tokenizer.h \ + $(TOP)\ext\fts1\fts1_tokenizer1.c +SRC = $(SRC) \ + $(TOP)\ext\fts2\fts2.c \ + $(TOP)\ext\fts2\fts2.h \ + $(TOP)\ext\fts2\fts2_hash.c \ + $(TOP)\ext\fts2\fts2_hash.h \ + $(TOP)\ext\fts2\fts2_icu.c \ + $(TOP)\ext\fts2\fts2_porter.c \ + $(TOP)\ext\fts2\fts2_tokenizer.h \ + $(TOP)\ext\fts2\fts2_tokenizer.c \ + $(TOP)\ext\fts2\fts2_tokenizer1.c +SRC = $(SRC) \ + $(TOP)\ext\fts3\fts3.c \ + $(TOP)\ext\fts3\fts3.h \ + $(TOP)\ext\fts3\fts3Int.h \ + $(TOP)\ext\fts3\fts3_aux.c \ + $(TOP)\ext\fts3\fts3_expr.c \ + $(TOP)\ext\fts3\fts3_hash.c \ + $(TOP)\ext\fts3\fts3_hash.h \ + $(TOP)\ext\fts3\fts3_icu.c \ + $(TOP)\ext\fts3\fts3_porter.c \ + $(TOP)\ext\fts3\fts3_snippet.c \ + $(TOP)\ext\fts3\fts3_tokenizer.h \ + $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_tokenizer1.c \ + $(TOP)\ext\fts3\fts3_write.c +SRC = $(SRC) \ + $(TOP)\ext\icu\sqliteicu.h \ + $(TOP)\ext\icu\icu.c +SRC = $(SRC) \ + $(TOP)\ext\rtree\rtree.h \ + $(TOP)\ext\rtree\rtree.c + + +# Generated source code files +# +SRC = $(SRC) \ + keywordhash.h \ + opcodes.c \ + opcodes.h \ + parse.c \ + parse.h \ + sqlite3.h + +# Source code to the test files. +# +TESTSRC = \ + $(TOP)\src\test1.c \ + $(TOP)\src\test2.c \ + $(TOP)\src\test3.c \ + $(TOP)\src\test4.c \ + $(TOP)\src\test5.c \ + $(TOP)\src\test6.c \ + $(TOP)\src\test7.c \ + $(TOP)\src\test8.c \ + $(TOP)\src\test9.c \ + $(TOP)\src\test_autoext.c \ + $(TOP)\src\test_async.c \ + $(TOP)\src\test_backup.c \ + $(TOP)\src\test_btree.c \ + $(TOP)\src\test_config.c \ + $(TOP)\src\test_demovfs.c \ + $(TOP)\src\test_devsym.c \ + $(TOP)\src\test_func.c \ + $(TOP)\src\test_fuzzer.c \ + $(TOP)\src\test_hexio.c \ + $(TOP)\src\test_init.c \ + $(TOP)\src\test_intarray.c \ + $(TOP)\src\test_journal.c \ + $(TOP)\src\test_malloc.c \ + $(TOP)\src\test_multiplex.c \ + $(TOP)\src\test_mutex.c \ + $(TOP)\src\test_onefile.c \ + $(TOP)\src\test_osinst.c \ + $(TOP)\src\test_pcache.c \ + $(TOP)\src\test_quota.c \ + $(TOP)\src\test_rtree.c \ + $(TOP)\src\test_schema.c \ + $(TOP)\src\test_server.c \ + $(TOP)\src\test_superlock.c \ + $(TOP)\src\test_syscall.c \ + $(TOP)\src\test_stat.c \ + $(TOP)\src\test_tclvar.c \ + $(TOP)\src\test_thread.c \ + $(TOP)\src\test_vfs.c \ + $(TOP)\src\test_wholenumber.c \ + $(TOP)\src\test_wsd.c \ + $(TOP)\ext\fts3\fts3_term.c \ + $(TOP)\ext\fts3\fts3_test.c + +# Source code to the library files needed by the test fixture +# +TESTSRC2 = \ + $(TOP)\src\attach.c \ + $(TOP)\src\backup.c \ + $(TOP)\src\bitvec.c \ + $(TOP)\src\btree.c \ + $(TOP)\src\build.c \ + $(TOP)\src\ctime.c \ + $(TOP)\src\date.c \ + $(TOP)\src\expr.c \ + $(TOP)\src\func.c \ + $(TOP)\src\insert.c \ + $(TOP)\src\wal.c \ + $(TOP)\src\mem5.c \ + $(TOP)\src\os.c \ + $(TOP)\src\os_os2.c \ + $(TOP)\src\os_unix.c \ + $(TOP)\src\os_win.c \ + $(TOP)\src\pager.c \ + $(TOP)\src\pragma.c \ + $(TOP)\src\prepare.c \ + $(TOP)\src\printf.c \ + $(TOP)\src\random.c \ + $(TOP)\src\pcache.c \ + $(TOP)\src\pcache1.c \ + $(TOP)\src\select.c \ + $(TOP)\src\tokenize.c \ + $(TOP)\src\utf.c \ + $(TOP)\src\util.c \ + $(TOP)\src\vdbeapi.c \ + $(TOP)\src\vdbeaux.c \ + $(TOP)\src\vdbe.c \ + $(TOP)\src\vdbemem.c \ + $(TOP)\src\vdbetrace.c \ + $(TOP)\src\where.c \ + parse.c \ + $(TOP)\ext\fts3\fts3.c \ + $(TOP)\ext\fts3\fts3_aux.c \ + $(TOP)\ext\fts3\fts3_expr.c \ + $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_write.c \ + $(TOP)\ext\async\sqlite3async.c + +# Header files used by all library source files. +# +HDR = \ + $(TOP)\src\btree.h \ + $(TOP)\src\btreeInt.h \ + $(TOP)\src\hash.h \ + $(TOP)\src\hwtime.h \ + keywordhash.h \ + $(TOP)\src\mutex.h \ + opcodes.h \ + $(TOP)\src\os.h \ + $(TOP)\src\os_common.h \ + $(TOP)\src\pager.h \ + $(TOP)\src\pcache.h \ + parse.h \ + sqlite3.h \ + $(TOP)\src\sqlite3ext.h \ + $(TOP)\src\sqliteInt.h \ + $(TOP)\src\sqliteLimit.h \ + $(TOP)\src\vdbe.h \ + $(TOP)\src\vdbeInt.h + +# Header files used by extensions +# +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts1\fts1.h \ + $(TOP)\ext\fts1\fts1_hash.h \ + $(TOP)\ext\fts1\fts1_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts2\fts2.h \ + $(TOP)\ext\fts2\fts2_hash.h \ + $(TOP)\ext\fts2\fts2_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\fts3\fts3.h \ + $(TOP)\ext\fts3\fts3Int.h \ + $(TOP)\ext\fts3\fts3_hash.h \ + $(TOP)\ext\fts3\fts3_tokenizer.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\rtree\rtree.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\icu\sqliteicu.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\rtree\sqlite3rtree.h + +# This is the default Makefile target. The objects listed here +# are what get build when you type just "make" with no arguments. +# +all: libsqlite3.lib sqlite3.exe libtclsqlite3.lib + +libsqlite3.lib: $(LIBOBJ) + $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) + +libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib + $(LTLIB) $(LTLIBOPTS) /LIBPATH:$(TCLLIBDIR) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) + +sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h + $(LTLINK) $(READLINE_FLAGS) \ + $(TOP)\src\shell.c \ + /link $(LTLINKOPTS) libsqlite3.lib $(LIBREADLINE) $(TLIBS) + +# This target creates a directory named "tsrc" and fills it with +# copies of all of the C source code and header files needed to +# build on the target system. Some of the C source code and header +# files are automatically generated. This target takes care of +# all that automatic generation. +# +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl + -rmdir /S/Q tsrc + -mkdir tsrc + for %i in ($(SRC)) do copy /Y %i tsrc + del /Q tsrc\sqlite.h.in tsrc\parse.y + $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl vdbe.new + move vdbe.new tsrc\vdbe.c + echo > .target_source + +sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl + $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl + +# Rule to build the amalgamation +# +sqlite3.lo: sqlite3.c + $(LTCOMPILE) -c sqlite3.c + +# Rules to build the LEMON compiler generator +# +lempar.c: $(TOP)\src\lempar.c + copy $(TOP)\src\lempar.c . + +lemon.exe: $(TOP)\tool\lemon.c lempar.c + $(BCC) -Fe$@ $(TOP)\tool\lemon.c + +# Rules to build individual *.lo files from generated *.c files. This +# applies to: +# +# parse.lo +# opcodes.lo +# +parse.lo: parse.c $(HDR) + $(LTCOMPILE) -c parse.c + +opcodes.lo: opcodes.c + $(LTCOMPILE) -c opcodes.c + +# Rules to build individual *.lo files from files in the src directory. +# +alter.lo: $(TOP)\src\alter.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\alter.c + +analyze.lo: $(TOP)\src\analyze.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\analyze.c + +attach.lo: $(TOP)\src\attach.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\attach.c + +auth.lo: $(TOP)\src\auth.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\auth.c + +backup.lo: $(TOP)\src\backup.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\backup.c + +bitvec.lo: $(TOP)\src\bitvec.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\bitvec.c + +btmutex.lo: $(TOP)\src\btmutex.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\btmutex.c + +btree.lo: $(TOP)\src\btree.c $(HDR) $(TOP)\src\pager.h + $(LTCOMPILE) -c $(TOP)\src\btree.c + +build.lo: $(TOP)\src\build.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\build.c + +callback.lo: $(TOP)\src\callback.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\callback.c + +complete.lo: $(TOP)\src\complete.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\complete.c + +ctime.lo: $(TOP)\src\ctime.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\ctime.c + +date.lo: $(TOP)\src\date.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\date.c + +delete.lo: $(TOP)\src\delete.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\delete.c + +expr.lo: $(TOP)\src\expr.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\expr.c + +fault.lo: $(TOP)\src\fault.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\fault.c + +fkey.lo: $(TOP)\src\fkey.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\fkey.c + +func.lo: $(TOP)\src\func.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\func.c + +global.lo: $(TOP)\src\global.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\global.c + +hash.lo: $(TOP)\src\hash.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\hash.c + +insert.lo: $(TOP)\src\insert.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\insert.c + +journal.lo: $(TOP)\src\journal.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\journal.c + +legacy.lo: $(TOP)\src\legacy.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\legacy.c + +loadext.lo: $(TOP)\src\loadext.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\loadext.c + +main.lo: $(TOP)\src\main.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\main.c + +malloc.lo: $(TOP)\src\malloc.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\malloc.c + +mem0.lo: $(TOP)\src\mem0.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem0.c + +mem1.lo: $(TOP)\src\mem1.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem1.c + +mem2.lo: $(TOP)\src\mem2.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem2.c + +mem3.lo: $(TOP)\src\mem3.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem3.c + +mem5.lo: $(TOP)\src\mem5.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mem5.c + +memjournal.lo: $(TOP)\src\memjournal.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\memjournal.c + +mutex.lo: $(TOP)\src\mutex.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex.c + +mutex_noop.lo: $(TOP)\src\mutex_noop.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_noop.c + +mutex_os2.lo: $(TOP)\src\mutex_os2.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_os2.c + +mutex_unix.lo: $(TOP)\src\mutex_unix.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_unix.c + +mutex_w32.lo: $(TOP)\src\mutex_w32.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\mutex_w32.c + +notify.lo: $(TOP)\src\notify.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\notify.c + +pager.lo: $(TOP)\src\pager.c $(HDR) $(TOP)\src\pager.h + $(LTCOMPILE) -c $(TOP)\src\pager.c + +pcache.lo: $(TOP)\src\pcache.c $(HDR) $(TOP)\src\pcache.h + $(LTCOMPILE) -c $(TOP)\src\pcache.c + +pcache1.lo: $(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h + $(LTCOMPILE) -c $(TOP)\src\pcache1.c + +os.lo: $(TOP)\src\os.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os.c + +os_unix.lo: $(TOP)\src\os_unix.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os_unix.c + +os_win.lo: $(TOP)\src\os_win.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os_win.c + +os_os2.lo: $(TOP)\src\os_os2.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\os_os2.c + +pragma.lo: $(TOP)\src\pragma.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\pragma.c + +prepare.lo: $(TOP)\src\prepare.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\prepare.c + +printf.lo: $(TOP)\src\printf.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\printf.c + +random.lo: $(TOP)\src\random.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\random.c + +resolve.lo: $(TOP)\src\resolve.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\resolve.c + +rowset.lo: $(TOP)\src\rowset.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\rowset.c + +select.lo: $(TOP)\src\select.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\select.c + +status.lo: $(TOP)\src\status.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\status.c + +table.lo: $(TOP)\src\table.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\table.c + +tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) + $(LTCOMPILE) -c $(TOP)\src\tokenize.c + +trigger.lo: $(TOP)\src\trigger.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\trigger.c + +update.lo: $(TOP)\src\update.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\update.c + +utf.lo: $(TOP)\src\utf.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\utf.c + +util.lo: $(TOP)\src\util.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\util.c + +vacuum.lo: $(TOP)\src\vacuum.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vacuum.c + +vdbe.lo: $(TOP)\src\vdbe.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbe.c + +vdbeapi.lo: $(TOP)\src\vdbeapi.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbeapi.c + +vdbeaux.lo: $(TOP)\src\vdbeaux.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbeaux.c + +vdbeblob.lo: $(TOP)\src\vdbeblob.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbeblob.c + +vdbemem.lo: $(TOP)\src\vdbemem.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbemem.c + +vdbetrace.lo: $(TOP)\src\vdbetrace.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vdbetrace.c + +vtab.lo: $(TOP)\src\vtab.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\vtab.c + +wal.lo: $(TOP)\src\wal.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\wal.c + +walker.lo: $(TOP)\src\walker.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\walker.c + +where.lo: $(TOP)\src\where.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\where.c + +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) + $(LTCOMPILE) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c + +tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib + $(LTLINK) tclsqlite-shell.lo \ + /link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) libsqlite3.lib $(LIBTCL) + +# Rules to build opcodes.c and opcodes.h +# +opcodes.c: opcodes.h $(TOP)\mkopcodec.awk + $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c + +opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk + type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h + +# Rules to build parse.c and parse.h - the outputs of lemon. +# +parse.h: parse.c + +parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk + del /Q parse.y parse.h parse.h.temp + copy $(TOP)\src\parse.y . + .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y + move parse.h parse.h.temp + $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h + +sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h + +mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c + $(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c + +keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe + .\mkkeywordhash.exe >keywordhash.h + + + +# Rules to build the extension objects. +# +icu.lo: $(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c + +fts2.lo: $(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2.c + +fts2_hash.lo: $(TOP)\ext\fts2\fts2_hash.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_hash.c + +fts2_icu.lo: $(TOP)\ext\fts2\fts2_icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_icu.c + +fts2_porter.lo: $(TOP)\ext\fts2\fts2_porter.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_porter.c + +fts2_tokenizer.lo: $(TOP)\ext\fts2\fts2_tokenizer.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer.c + +fts2_tokenizer1.lo: $(TOP)\ext\fts2\fts2_tokenizer1.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer1.c + +fts3.lo: $(TOP)\ext\fts3\fts3.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3.c + +fts3_aux.lo: $(TOP)\ext\fts3\fts3_aux.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_aux.c + +fts3_expr.lo: $(TOP)\ext\fts3\fts3_expr.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_expr.c + +fts3_hash.lo: $(TOP)\ext\fts3\fts3_hash.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_hash.c + +fts3_icu.lo: $(TOP)\ext\fts3\fts3_icu.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_icu.c + +fts3_snippet.lo: $(TOP)\ext\fts3\fts3_snippet.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_snippet.c + +fts3_porter.lo: $(TOP)\ext\fts3\fts3_porter.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_porter.c + +fts3_tokenizer.lo: $(TOP)\ext\fts3\fts3_tokenizer.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer.c + +fts3_tokenizer1.lo: $(TOP)\ext\fts3\fts3_tokenizer1.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer1.c + +fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_write.c + +rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c + + +# Rules to build the 'testfixture' application. +# +# If using the amalgamation, use sqlite3.c directly to build the test +# fixture. Otherwise link against libsqlite3.lib. (This distinction is +# necessary because the test fixture requires non-API symbols which are +# hidden when the library is built via the amalgamation). +# +TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE + +TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib +TESTFIXTURE_SRC1 = sqlite3.c +!IF $(USE_AMALGAMATION)==0 +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) +!ELSE +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) +!ENDIF + +testfixture.exe: $(TESTFIXTURE_SRC) $(HDR) + $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ + -DBUILD_sqlite -I$(TCLINCDIR) \ + $(TESTFIXTURE_SRC) \ + /link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) + +fulltest: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\all.test + +soaktest: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\all.test -soak=1 + +test: testfixture.exe sqlite3.exe + .\testfixture.exe $(TOP)\test\veryquick.test + +spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl + $(NAWK) -f $(TOP)/tool/tostr.awk \ + $(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h + +sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h + $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ + -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE \ + -DBUILD_sqlite -I$(TCLINCDIR) \ + $(TESTFIXTURE_SRC) \ + /link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) + +clean: + del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib + del /Q sqlite3.h opcodes.c opcodes.h + del /Q lemon.exe lempar.c parse.* + del /Q mkkeywordhash.exe keywordhash.h + -rmdir /Q/S tsrc + del /Q .target_source + del /Q testfixture.exe testfixture.exp test.db + del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def + del /Q sqlite3.c + del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp spaceanal_tcl.h + +# +# Windows section +# +dll: sqlite3.dll + +sqlite3.def: libsqlite3.lib + echo EXPORTS >sqlite3.def + dumpbin /all libsqlite3.lib \ + | $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \ + | sort >>sqlite3.def + +sqlite3.dll: $(LIBOBJ) sqlite3.def + link $(LTLINKOPTS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ) diff --git a/VERSION b/VERSION index 81aca733..9076adfe 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.6.2 +3.7.7.1 diff --git a/config.h.in b/config.h.in index 812c5326..11f22803 100644 --- a/config.h.in +++ b/config.h.in @@ -75,6 +75,9 @@ /* Define to 1 if you have the `usleep' function. */ #undef HAVE_USLEEP +/* Define to 1 if you have the utime() library function. */ +#undef HAVE_UTIME + /* Define to the sub-directory in which libtool stores uninstalled libraries. */ #undef LT_OBJDIR diff --git a/configure b/configure index c2ac0675..b565366d 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.6.2. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.7.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.6.2' -PACKAGE_STRING='sqlite 3.7.6.2' +PACKAGE_VERSION='3.7.7.1' +PACKAGE_STRING='sqlite 3.7.7.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1485,7 +1485,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.6.2 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.7.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1550,7 +1550,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.6.2:";; + short | recursive ) echo "Configuration of sqlite 3.7.7.1:";; esac cat <<\_ACEOF @@ -1666,7 +1666,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.6.2 +sqlite configure 3.7.7.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1680,7 +1680,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.6.2, which was +It was created by sqlite $as_me 3.7.7.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -12145,7 +12145,8 @@ done -for ac_func in usleep fdatasync localtime_r gmtime_r localtime_s + +for ac_func in usleep fdatasync localtime_r gmtime_r localtime_s utime do as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` { $as_echo "$as_me:$LINENO: checking for $ac_func" >&5 @@ -13440,6 +13441,93 @@ fi if test "${use_loadextension}" = "yes" ; then OPT_FEATURE_FLAGS="" + { $as_echo "$as_me:$LINENO: checking for library containing dlopen" >&5 +$as_echo_n "checking for library containing dlopen... " >&6; } +if test "${ac_cv_search_dlopen+set}" = set; then + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat >conftest.$ac_ext <<_ACEOF +/* confdefs.h. */ +_ACEOF +cat confdefs.h >>conftest.$ac_ext +cat >>conftest.$ac_ext <<_ACEOF +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (); +int +main () +{ +return dlopen (); + ; + return 0; +} +_ACEOF +for ac_lib in '' dl; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + rm -f conftest.$ac_objext conftest$ac_exeext +if { (ac_try="$ac_link" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\"" +$as_echo "$ac_try_echo") >&5 + (eval "$ac_link") 2>conftest.er1 + ac_status=$? + grep -v '^ *+' conftest.er1 >conftest.err + rm -f conftest.er1 + cat conftest.err >&5 + $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5 + (exit $ac_status); } && { + test -z "$ac_c_werror_flag" || + test ! -s conftest.err + } && test -s conftest$ac_exeext && { + test "$cross_compiling" = yes || + $as_test_x conftest$ac_exeext + }; then + ac_cv_search_dlopen=$ac_res +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + +fi + +rm -rf conftest.dSYM +rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \ + conftest$ac_exeext + if test "${ac_cv_search_dlopen+set}" = set; then + break +fi +done +if test "${ac_cv_search_dlopen+set}" = set; then + : +else + ac_cv_search_dlopen=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:$LINENO: result: $ac_cv_search_dlopen" >&5 +$as_echo "$ac_cv_search_dlopen" >&6; } +ac_res=$ac_cv_search_dlopen +if test "$ac_res" != no; then + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi @@ -13942,7 +14030,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.6.2, which was +This file was extended by sqlite $as_me 3.7.7.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -13995,7 +14083,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.6.2 +sqlite config.status 3.7.7.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/configure.ac b/configure.ac index 464ed1ef..46afeb75 100644 --- a/configure.ac +++ b/configure.ac @@ -127,7 +127,7 @@ AC_CHECK_HEADERS([sys/types.h stdlib.h stdint.h inttypes.h]) ######### # Figure out whether or not we have these functions # -AC_CHECK_FUNCS([usleep fdatasync localtime_r gmtime_r localtime_s]) +AC_CHECK_FUNCS([usleep fdatasync localtime_r gmtime_r localtime_s utime]) ######### # By default, we use the amalgamation (this may be changed below...) @@ -619,6 +619,7 @@ AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--enable-load-extension], [use_loadextension=$enableval],[use_loadextension=no]) if test "${use_loadextension}" = "yes" ; then OPT_FEATURE_FLAGS="" + AC_SEARCH_LIBS(dlopen, dl) else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 20da0516..67a0ea20 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -292,14 +292,13 @@ ** into a single segment. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif -#include "fts3Int.h" - #include #include #include @@ -420,17 +419,31 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ } /* -** As long as *pp has not reached its end (pEnd), then do the same -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. -** But if we have reached the end of the varint, just set *pp=0 and -** leave *pVal unchanged. +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. */ -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - fts3GetDeltaVarint(pp, pVal); - } +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p = *pp; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; } /* @@ -524,6 +537,8 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); for(i=1; zCols && inColumn; i++){ @@ -629,6 +644,9 @@ static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ sqlite3_step(pStmt); p->nPgsz = sqlite3_column_int(pStmt, 0); rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; } } assert( p->nPgsz>0 || rc!=SQLITE_OK ); @@ -802,6 +820,58 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p = *pp; + int nInt = 0; + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + + +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ +){ + struct Fts3Index *aIndex; + int nIndex = 1; + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = *apFree = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i MATCHINFO */ + { "prefix", 6, 0 }, /* 1 -> PREFIX */ + { "compress", 8, 0 }, /* 2 -> COMPRESS */ + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ + { "order", 5, 0 } /* 4 -> ORDER */ + }; + + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ - zCompress = zVal; - zVal = 0; - }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ - zUncompress = zVal; - zVal = 0; }else{ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + if( iOpt==SizeofArray(aFts4Opt) ){ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + } + } + sqlite3_free(zVal); } - sqlite3_free(zVal); } /* Otherwise, the argument is a column name. */ @@ -925,10 +1046,17 @@ static int fts3InitVtab( } assert( pTokenizer ); + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + if( rc==SQLITE_ERROR ){ + assert( zPrefix ); + *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); + } + if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ - nByte = sizeof(Fts3Table) + /* Fts3Table */ + nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ + nIndex * sizeof(struct Fts3Index) + /* aIndex */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ @@ -943,14 +1071,22 @@ static int fts3InitVtab( p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; - p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + p->bDescIdx = bDescIdx; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->azColumn[nCol]; + zCsr = (char *)&p->aIndex[nIndex]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -961,7 +1097,7 @@ static int fts3InitVtab( /* Fill in the azColumn array */ for(iCol=0; iColnNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -1007,6 +1144,7 @@ fts3_init_out: pTokenizer->pModule->xDestroy(pTokenizer); } }else{ + assert( p->pSegments==0 ); *ppVTab = &p->base; } return rc; @@ -1092,6 +1230,23 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->aConstraintUsage[iCons].argvIndex = 1; pInfo->aConstraintUsage[iCons].omit = 1; } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); return SQLITE_OK; } @@ -1127,6 +1282,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -1138,8 +1294,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ if( pCsr->isRequireSeek ){ - pCsr->isRequireSeek = 0; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; }else{ @@ -1149,7 +1305,7 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ ** table is missing a row that is present in the full-text index. ** The data structures are corrupt. */ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } pCsr->isEof = 1; if( pContext ){ @@ -1209,7 +1365,7 @@ static int fts3ScanInteriorNode( zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); if( zCsr>zEnd ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } while( zCsrzEnd ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; goto finish_scan; } if( nPrefix+nSuffix>nAlloc ){ @@ -1320,7 +1476,7 @@ static int fts3SelectLeaf( int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } @@ -1330,7 +1486,7 @@ static int fts3SelectLeaf( } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); @@ -1706,7 +1862,19 @@ static int fts3PoslistPhraseMerge( } /* -** Merge two position-lists as required by the NEAR operator. +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: +** +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. */ static int fts3PoslistNearMerge( char **pp, /* Output buffer */ @@ -1719,214 +1887,27 @@ static int fts3PoslistNearMerge( char *p1 = *pp1; char *p2 = *pp2; - if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; - *pp1 = p1; - *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; + + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); }else{ - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; - - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; - } - - return res; - } -} - -/* -** Values that may be used as the first parameter to fts3DoclistMerge(). -*/ -#define MERGE_NOT 2 /* D + D -> D */ -#define MERGE_AND 3 /* D + D -> D */ -#define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ -#define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ -#define MERGE_NEAR 8 /* P + P -> D */ -#define MERGE_POS_NEAR 9 /* P + P -> P */ - -/* -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, -** which is guaranteed to be large enough to hold the results. The number -** of bytes written to aBuffer is stored in *pnBuffer before returning. -** -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error -** occurs while allocating a temporary buffer as part of the merge operation, -** SQLITE_NOMEM is returned. -*/ -static int fts3DoclistMerge( - int mergetype, /* One of the MERGE_XXX constants */ - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - char *aBuffer, /* Pre-allocated output buffer */ - int *pnBuffer, /* OUT: Bytes written to aBuffer */ - char *a1, /* Buffer containing first doclist */ - int n1, /* Size of buffer a1 */ - char *a2, /* Buffer containing second doclist */ - int n2, /* Size of buffer a2 */ - int *pnDoc /* OUT: Number of docids in output */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - - char *p = aBuffer; - char *p1 = a1; - char *p2 = a2; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - int nDoc = 0; - - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR - || mergetype==MERGE_AND || mergetype==MERGE_NOT - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR - ); - - if( !aBuffer ){ - *pnBuffer = 0; - return SQLITE_NOMEM; + res = 0; } - /* Read the first docid from each doclist */ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - - switch( mergetype ){ - case MERGE_OR: - case MERGE_POS_OR: - while( p1 || p2 ){ - if( p2 && p1 && i1==i2 ){ - fts3PutDeltaVarint(&p, &iPrev, i1); - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || (p1 && i1=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) + +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; +} + +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *pnRight = p - aOut; +} + + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all @@ -1949,8 +2072,7 @@ struct TermSelect { ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(TermSelect *pTS){ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; @@ -1965,15 +2087,17 @@ static int fts3TermSelectMerge(TermSelect *pTS){ nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ - sqlite3_free(aOut); - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; @@ -2009,9 +2133,7 @@ static int fts3TermSelectCb( if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ + ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ @@ -2020,126 +2142,100 @@ static int fts3TermSelectCb( return SQLITE_NOMEM; } }else{ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aMerge = aDoclist; int nMerge = nDoclist; int iOut; for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; if( pTS->aaOutput[iOut]==0 ){ assert( iOut>0 ); pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; break; - } + }else{ + char *aNew; + int nNew; - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; } - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } } } } return SQLITE_OK; } -static int fts3DeferredTermSelect( - Fts3DeferredToken *pToken, /* Phrase token */ - int isTermPos, /* True to include positions */ - int *pnOut, /* OUT: Size of list */ - char **ppOut /* OUT: Body of list */ +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew ){ - char *aSource; - int nSource; - - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); - if( !aSource ){ - *pnOut = 0; - *ppOut = 0; - }else if( isTermPos ){ - *ppOut = sqlite3_malloc(nSource); - if( !*ppOut ) return SQLITE_NOMEM; - memcpy(*ppOut, aSource, nSource); - *pnOut = nSource; - }else{ - sqlite3_int64 docid; - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); - *ppOut = sqlite3_malloc(*pnOut); - if( !*ppOut ) return SQLITE_NOMEM; - sqlite3Fts3PutVarint(*ppOut, docid); + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; } - + pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } -int sqlite3Fts3SegReaderCursor( +static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; - int iAge = 0; sqlite3_stmt *pStmt = 0; - Fts3SegReader *pPending = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( FTS3_SEGCURSOR_PENDING<0 ); - assert( FTS3_SEGCURSOR_ALL<0 ); - assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); - assert( isPrefix==0 || isScan==0 ); - - - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); - if( iLevel<0 && isScan==0 ){ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); - if( rc==SQLITE_OK && pPending ){ - int nByte = (sizeof(Fts3SegReader *) * 16); - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - pCsr->apSegment[0] = pPending; - pCsr->nSegment = 1; - pPending = 0; - } + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); @@ -2148,18 +2244,6 @@ int sqlite3Fts3SegReaderCursor( int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); - /* If nSegment is a multiple of 16 the array needs to be extended. */ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - pCsr->apSegment = apNew; - } - /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ @@ -2169,53 +2253,117 @@ int sqlite3Fts3SegReaderCursor( if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; } - rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; - pCsr->nSegment++; - iAge++; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; - sqlite3Fts3SegReaderFree(pPending); return rc; } +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelaIndex==0 ); + + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); + + return fts3SegReaderCursor( + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + ); +} + +static int fts3SegReaderCursorAddZero( + Fts3Table *p, + const char *zTerm, + int nTerm, + Fts3MultiSegReader *pCsr +){ + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); +} + + +int sqlite3Fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ - pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; - int nCost = 0; - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr); - - for(i=0; rc==SQLITE_OK && inSegment; i++){ - rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); + int bFound = 0; /* True once an index has been found */ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor( + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; } - pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; return rc; } -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } @@ -2240,7 +2388,7 @@ static int fts3TermSelect( char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ @@ -2266,7 +2414,7 @@ static int fts3TermSelect( } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); + rc = fts3TermSelectMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -2316,660 +2464,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. -*/ -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pExpr ){ - rc = fts3DeferExpression(pCsr, pExpr->pLeft); - if( rc==SQLITE_OK ){ - rc = fts3DeferExpression(pCsr, pExpr->pRight); - } - if( pExpr->eType==FTSQUERY_PHRASE ){ - int iCol = pExpr->pPhrase->iColumn; - int i; - for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; - if( pToken->pDeferred==0 ){ - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); - } - } - } - } - return rc; -} - -/* -** This function removes the position information from a doclist. When -** called, buffer aList (size *pnList bytes) contains a doclist that includes -** position information. This function removes the position information so -** that aList contains only docids, and adjusts *pnList to reflect the new -** (possibly reduced) size of the doclist. -*/ -static void fts3DoclistStripPositions( - char *aList, /* IN/OUT: Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist in bytes */ -){ - if( aList ){ - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Input cursor */ - char *pOut = aList; /* Output cursor */ - - while( piColumn; - int isTermPos = (pPhrase->nToken>1 || isReqPos); - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int isFirst = 1; - - int iPrevTok = 0; - int nDoc = 0; - - /* If this is an xFilter() evaluation, create a segment-reader for each - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo - ** evaluation, only create segment-readers if there are no Fts3DeferredToken - ** objects attached to the phrase-tokens. - */ - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) - ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - if( rc!=SQLITE_OK ) return rc; - } - } - } - - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok; /* Token to find doclist for */ - int iTok = 0; /* The token being queried this iteration */ - char *pList = 0; /* Pointer to token doclist */ - int nList = 0; /* Size of buffer at pList */ - - /* Select a token to process. If this is an xFilter() call, then tokens - ** are processed in order from least to most costly. Otherwise, tokens - ** are processed in the order in which they occur in the phrase. - */ - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ - assert( isReqPos ); - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - if( pTok->bFulltext==0 ) continue; - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - }else{ - int nMinCost = 0x7FFFFFFF; - int jj; - - /* Find the remaining token with the lowest cost. */ - for(jj=0; jjnToken; jj++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; - if( pSegcsr && pSegcsr->nCostnCost; - } - } - pTok = &pPhrase->aToken[iTok]; - - /* This branch is taken if it is determined that loading the doclist - ** for the next token would require more IO than loading all documents - ** currently identified by doclist pOut/nOut. No further doclists will - ** be loaded from the full-text index for this phrase. - */ - if( nMinCost>nDoc && ii>0 ){ - rc = fts3DeferExpression(pCsr, pCsr->pExpr); - break; - } - } - - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); - }else{ - if( pTok->pSegcsr ){ - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); - } - pTok->bFulltext = 1; - } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); - if( rc!=SQLITE_OK ) break; - - if( isFirst ){ - pOut = pList; - nOut = nList; - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ - nDoc = fts3DoclistCountDocids(1, pOut, nOut); - } - isFirst = 0; - iPrevTok = iTok; - }else{ - /* Merge the new term list and the current output. */ - char *aLeft, *aRight; - int nLeft, nRight; - int nDist; - int mt; - - /* If this is the final token of the phrase, and positions were not - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. - ** This drops the position information from the output list. - */ - mt = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; - - assert( iPrevTok!=iTok ); - if( iPrevToknToken ){ - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); - fts3DoclistStripPositions(pOut, &nOut); - } - *paOut = pOut; - *pnOut = nOut; - }else{ - sqlite3_free(pOut); - } - return rc; -} - -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ - - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); - - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } - - *paOut = aOut; - return rc; -} - -/* -** This function is used as part of the processing for the snippet() and -** offsets() functions. -** -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both -** have their respective doclists (including position information) loaded -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from -** each doclist that are not within nNear tokens of a corresponding entry -** in the other doclist. -*/ -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ - int rc; /* Return code */ - - assert( pLeft->eType==FTSQUERY_PHRASE ); - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->isLoaded && pRight->isLoaded ); - - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ - sqlite3_free(pLeft->aDoclist); - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = 0; - pLeft->aDoclist = 0; - rc = SQLITE_OK; - }else{ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - &aOut, &nOut - ); - if( rc!=SQLITE_OK ) return rc; - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = aOut; - pRight->nDoclist = nOut; - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - &aOut, &nOut - ); - sqlite3_free(pLeft->aDoclist); - pLeft->aDoclist = aOut; - pLeft->nDoclist = nOut; - } - return rc; -} - - -/* -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. -** The allocated objects are stored in the Fts3PhraseToken.pArray member -** variables of each token structure. -*/ -static int fts3ExprAllocateSegReaders( - Fts3Cursor *pCsr, /* FTS3 table */ - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - int *pnExpr /* OUT: Number of AND'd expressions */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ - (*pnExpr)++; - pnExpr = 0; - } - - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - } - } - }else{ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); - if( rc==SQLITE_OK ){ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); - } - } - return rc; -} - -/* -** Free the Fts3SegReaderArray objects associated with each token in the -** expression pExpr. In other words, this function frees the resources -** allocated by fts3ExprAllocateSegReaders(). -*/ -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase ){ - int kk; - for(kk=0; kknToken; kk++){ - fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); - pPhrase->aToken[kk].pSegcsr = 0; - } - } - fts3ExprFreeSegReaders(pExpr->pLeft); - fts3ExprFreeSegReaders(pExpr->pRight); - } -} - -/* -** Return the sum of the costs of all tokens in the expression pExpr. This -** function must be called after Fts3SegReaderArrays have been allocated -** for all tokens using fts3ExprAllocateSegReaders(). -*/ -static int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; - if( pSegcsr ) nCost += pSegcsr->nCost; - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; -} - -/* -** The following is a helper function (and type) for fts3EvalExpr(). It -** must be called after Fts3SegReaders have been allocated for every token -** in the expression. See the context it is called from in fts3EvalExpr() -** for further explanation. -*/ -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr); - (*ppExprCost)++; - } -} - -/* -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. -** -** This function is called in two distinct contexts: -** -** * From within the virtual table xFilter() method. In this case, the -** output doclist contains entries for all rows in the table, based on -** data read from the full-text index. -** -** In this case, if the query expression contains one or more tokens that -** are very common, then the returned doclist may contain a superset of -** the documents that actually match the expression. -** -** * From within the virtual table xNext() method. This call is only made -** if the call from within xFilter() found that there were very common -** tokens in the query expression and did return a superset of the -** matching documents. In this case the returned doclist contains only -** entries that correspond to the current row of the table. Instead of -** reading the data for each token from the full-text index, the data is -** already available in-memory in the Fts3PhraseToken.pDeferred structures. -** See fts3EvalDeferred() for how it gets there. -** -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is -** required) Fts3Cursor.doDeferred==1. -** -** If the SQLite invokes the snippet(), offsets() or matchinfo() function -** as part of a SELECT on an FTS3 table, this function is called on each -** individual phrase expression in the query. If there were very common tokens -** found in the xFilter() call, then this function is called once for phrase -** for each row visited, and the returned doclist contains entries for the -** current row only. Otherwise, if there were no very common tokens, then this -** function is called once only for each phrase in the query and the returned -** doclist contains entries for all rows of the table. -** -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a -** result of a snippet(), offsets() or matchinfo() invocation. -*/ -static int fts3EvalExpr( - Fts3Cursor *p, /* Virtual table cursor handle */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut, /* OUT: Size of buffer at *paOut */ - int isReqPos /* Require positions in output buffer */ -){ - int rc = SQLITE_OK; /* Return code */ - - /* Zero the output parameters. */ - *paOut = 0; - *pnOut = 0; - - if( pExpr ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - || pExpr->eType==FTSQUERY_PHRASE - ); - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); - - if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), - paOut, pnOut - ); - fts3ExprFreeSegReaders(pExpr); - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ - int nExpr = 0; /* Size of aExpr[] */ - char *aRet = 0; /* Doclist to return to caller */ - int nRet = 0; /* Length of aRet[] in bytes */ - int nDoc = 0x7FFFFFFF; - - assert( !isReqPos ); - - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); - if( rc==SQLITE_OK ){ - assert( nExpr>1 ); - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); - if( !aExpr ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - int ii; /* Used to iterate through expressions */ - - fts3ExprAssignCosts(pExpr, &aExpr); - aExpr -= nExpr; - for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ - pBest = pCand; - } - } - - if( pBest->nCost>nDoc ){ - rc = fts3DeferExpression(p, p->pExpr); - break; - }else{ - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); - if( rc!=SQLITE_OK ) break; - pBest->pExpr = 0; - if( ii==0 ){ - aRet = aNew; - nRet = nNew; - nDoc = fts3DoclistCountDocids(0, aRet, nRet); - }else{ - fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc - ); - sqlite3_free(aNew); - } - } - } - } - - if( rc==SQLITE_OK ){ - *paOut = aRet; - *pnOut = nRet; - }else{ - assert( *paOut==0 ); - sqlite3_free(aRet); - } - sqlite3_free(aExpr); - fts3ExprFreeSegReaders(pExpr); - - }else{ - char *aLeft; - char *aRight; - int nLeft; - int nRight; - - assert( pExpr->eType==FTSQUERY_NEAR - || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_NOT - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) - ); - - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) - ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: { - Fts3Expr *pLeft; - Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - mergetype = MERGE_POS_NEAR; - } - pLeft = pExpr->pLeft; - while( pLeft->eType==FTSQUERY_NEAR ){ - pLeft=pLeft->pRight; - } - pRight = pExpr->pRight; - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - - rc = fts3NearMerge(mergetype, pExpr->nNear, - pLeft->pPhrase->nToken, aLeft, nLeft, - pRight->pPhrase->nToken, aRight, nRight, - paOut, pnOut - ); - sqlite3_free(aLeft); - break; - } - - case FTSQUERY_OR: { - /* Allocate a buffer for the output. The maximum size is the - ** sum of the sizes of the two input buffers. The +1 term is - ** so that a buffer of zero bytes is never allocated - this can - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. - */ - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aBuffer; - sqlite3_free(aLeft); - break; - } - - default: { - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aLeft; - break; - } - } - } - sqlite3_free(aRight); - } - } - - assert( rc==SQLITE_OK || *paOut==0 ); - return rc; -} - -/* -** This function is called from within xNext() for each row visited by -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() -** was able to determine the exact set of matching rows, this function sets -** *pbRes to true and returns SQLITE_IO immediately. -** -** Otherwise, if evaluating the query expression within xFilter() returned a -** superset of the matching documents instead of an exact set (this happens -** when the query includes very common tokens and it is deemed too expensive to -** load their doclists from disk), this function tests if the current row -** really does match the FTS3 query. -** -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK -** is returned and *pbRes is set to true if the current row matches the -** FTS3 query (and should be included in the results returned to SQLite), or -** false otherwise. -*/ -static int fts3EvalDeferred( - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ - int *pbRes /* OUT: Set to true if row is a match */ -){ - int rc = SQLITE_OK; - if( pCsr->pDeferred==0 ){ - *pbRes = 1; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - if( rc==SQLITE_OK ){ - char *a = 0; - int n = 0; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); - assert( n>=0 ); - *pbRes = (n>0); - sqlite3_free(a); - } - } - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -2982,31 +2476,20 @@ static int fts3EvalDeferred( ** subsequently to determine whether or not an EOF was hit. */ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int res; - int rc = SQLITE_OK; /* Return code */ + int rc; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); }else{ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; - } - sqlite3_reset(pCsr->pStmt); - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); - + }else{ + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -3033,11 +2516,7 @@ static int fts3FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - const char *azSql[] = { - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ - "SELECT %s FROM %Q.'%q_content' AS x ", /* full-scan */ - }; - int rc; /* Return code */ + int rc; char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -3056,6 +2535,13 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)idxNum; + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -3069,8 +2555,8 @@ static int fts3FilterMethod( ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", - zQuery); + static const char *zErr = "malformed MATCH expression: [%s]"; + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); } return rc; } @@ -3078,7 +2564,8 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -3090,20 +2577,24 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; - zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName); - if( !zSql ){ - rc = SQLITE_NOMEM; + if( idxNum==FTS3_FULLSCAN_SEARCH ){ + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); } - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - } - pCsr->eSearch = (i16)idxNum; - + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; + + if( idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( rc!=SQLITE_OK ) return rc; + } + return fts3NextMethod(pCursor); } @@ -3123,16 +2614,7 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->aDoclist ){ - *pRowid = pCsr->iPrevId; - }else{ - /* This branch runs if the query is implemented using a full-table scan - ** (not using the full-text index). In this case grab the rowid from the - ** SELECT statement. - */ - assert( pCsr->isRequireSeek==0 ); - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - } + *pRowid = pCsr->iPrevId; return SQLITE_OK; } @@ -3145,7 +2627,7 @@ static int fts3ColumnMethod( sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -3156,21 +2638,20 @@ static int fts3ColumnMethod( /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); + sqlite3_result_int64(pContext, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -3202,8 +2683,13 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(pVtab); - assert( ((Fts3Table *)pVtab)->nPendingData==0 ); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } @@ -3213,8 +2699,13 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ ** by fts3SyncMethod(). */ static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(pVtab); - assert( ((Fts3Table *)pVtab)->nPendingData==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } @@ -3223,93 +2714,31 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){ ** hash-table. Any changes made to the database are reverted by SQLite. */ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } /* -** Load the doclist associated with expression pExpr to pExpr->aDoclist. -** The loaded doclist contains positions as well as the document ids. -** This is used by the matchinfo(), snippet() and offsets() auxillary -** functions. +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. */ -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); - return rc; -} +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c; -int sqlite3Fts3ExprLoadFtDoclist( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - char **paDoclist, - int *pnDoclist -){ - int rc; - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return rc; -} - -/* -** After ExprLoadDoclist() (see above) has been called, this function is -** used to iterate/search through the position lists that make up the doclist -** stored in pExpr->aDoclist. -*/ -char *sqlite3Fts3FindPositions( - Fts3Expr *pExpr, /* Access this expressions doclist */ - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ - int iCol /* Column of requested pos-list */ -){ - assert( pExpr->isLoaded ); - if( pExpr->aDoclist ){ - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - char *pCsr; - - if( pExpr->pCurrent==0 ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent,&pExpr->iCurrent); - } - pCsr = pExpr->pCurrent; - assert( pCsr ); - - while( pCsriCurrentiCurrent); - } - pExpr->pCurrent = pCsr; - }else{ - if( pExpr->iCurrent==iDocid ){ - int iThis = 0; - if( iCol<0 ){ - /* If iCol is negative, return a pointer to the start of the - ** position-list (instead of a pointer to the start of a list - ** of offsets associated with a specific column). - */ - return pCsr; - } - while( iThispStart && (c=*p--)==0 ); + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; } - - return 0; + if( p>pStart ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; } /* @@ -3542,8 +2971,34 @@ static int fts3RenameMethod( return rc; } +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + return fts3SyncMethod(pVtab); +} +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + static const sqlite3_module fts3Module = { - /* iVersion */ 0, + /* iVersion */ 2, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, @@ -3563,6 +3018,9 @@ static const sqlite3_module fts3Module = { /* xRollback */ fts3RollbackMethod, /* xFindFunction */ fts3FindFunctionMethod, /* xRename */ fts3RenameMethod, + /* xSavepoint */ fts3SavepointMethod, + /* xRelease */ fts3ReleaseMethod, + /* xRollbackTo */ fts3RollbackToMethod, }; /* @@ -3609,6 +3067,11 @@ int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3IcuTokenizerModule(&pIcu); #endif +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif + rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; @@ -3684,4 +3147,1306 @@ int sqlite3_extension_init( } #endif + +/* +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. +*/ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +static void fts3EvalPhraseMergeToken( + Fts3Table *pTab, + Fts3Phrase *p, + int iToken, + char *pList, + int nList +){ + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; +} + +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, + Fts3Phrase *p +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + if( rc==SQLITE_OK ){ + fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; + int rc = SQLITE_OK; + + int nMaxUndeferred = pPhrase->iDoclistToken; + char *aPoslist = 0; + int nPoslist = 0; + int iPrev = -1; + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = aOut - aPoslist; + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + int rc; + Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( pCsr->bDesc==pTab->bDescIdx + && bOptOk==1 + && p->nToken==1 + && pFirst->pSegcsr + && pFirst->pSegcsr->bLookup + ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + rc = sqlite3Fts3MsrIncrStart( + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + p->bIncr = 1; + + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDociddoclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + assert( p->nToken==1 ); + assert( pDL->pNextDocid==0 ); + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( rc==SQLITE_OK && !pDL->pList ){ + *pbEof = 1; + } + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=pEnd ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim2(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } + } + + return rc; +} + +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int bOptOk, + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; + int nOvfl; + int iCol; /* The column the token must match */ +}; + +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3Expr *pExpr, + Fts3TokenAndCost **ppTC, + Fts3Expr ***ppOr, + int *pRc +){ + if( *pRc==SQLITE_OK && pExpr ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3TokenAndCost *aTC, + int nTC +){ + int nDocSize = 0; + int nDocEst = 0; + int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int ii; + + int nOvfl = 0; + int nTerm = 0; + + for(ii=0; iinOvfl) + ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + + return rc; +} + +int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: + ** This call will eventually also be responsible for determining which + ** tokens are 'deferred' until the document text is loaded into memory. + ** + ** Each token in each phrase is dealt with using one of the following + ** three strategies: + ** + ** 1. Entire doclist loaded into memory as part of the + ** fts3EvalStartReaders() call. + ** + ** 2. Doclist loaded into memory incrementally, as part of each + ** sqlite3Fts3EvalNext() call. + ** + ** 3. Token doclist is never loaded. Instead, documents are loaded into + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() + ** call. This is known as a "deferred" token. + */ + + /* If bOptOk is true, check if there are any tokens that should be deferred. + */ + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + nToken = pTC-aTC; + nOr = ppOr-apOr; + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iidoclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +static int fts3EvalNearTrim2( + int nNear, + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear = p->pParent->nNear; + Fts3Phrase *pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + +/* +** This macro is used by the fts3EvalNext() function. The two arguments are +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then +** it returns (i2 - i1). This allows the same code to be used for merging +** doclists in ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) + +static void fts3EvalNext( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + if( pLeft->bDeferred ){ + fts3EvalNext(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + fts3EvalNext(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + } + + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNext(pCsr, pRight, pRc); + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNext(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNext(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalZeroPoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + int bHit = 1; + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else{ + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } + } + } + return bHit; +} + +/* +** Return 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does not match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNext(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + } + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalZeroPoslist(pPhrase); + if( pPhrase->bIncr ){ + assert( pPhrase->nToken==1 ); + assert( pPhrase->aToken[0].pSegcsr ); + sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += sqlite3Fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNext(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalLoadDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + fts3EvalLoadDeferred(pCsr, &rc); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol /* Column to return position list for */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter = pPhrase->doclist.pList; + int iThis; + + assert( iCol>=0 && iColnColumn ); + if( !pIter + || pExpr->bEof + || pExpr->iDocid!=pCsr->iPrevId + || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) + ){ + return 0; + } + + assert( pPhrase->doclist.nList>0 ); + if( *pIter==0x01 ){ + pIter++; + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalZeroPoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + #endif diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index b3f1ab55..16643796 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -11,7 +11,6 @@ ****************************************************************************** ** */ - #ifndef _FTSINT_H #define _FTSINT_H @@ -19,6 +18,16 @@ # define NDEBUG 1 #endif +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#ifdef SQLITE_ENABLE_FTS3 #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" @@ -47,12 +56,35 @@ */ #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. */ #define FTS3_VARINT_MAX 10 +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + /* ** The testcase() macro is only used by the amalgamation. If undefined, ** make it a no-op. @@ -92,22 +124,43 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ typedef unsigned int u32; /* 4-byte unsigned integer */ typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ + /* ** Macro used to suppress compiler warnings for unused parameters. */ #define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; +typedef struct Fts3Doclist Fts3Doclist; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; /* ** A connection to a fulltext index is an instance of the following @@ -128,7 +181,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[24]; + sqlite3_stmt *aStmt[27]; char *zReadExprlist; char *zWriteExprlist; @@ -136,21 +189,43 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* The following hash table is used to buffer pending index updates during + /* TODO: Fix the first paragraph of this comment. + ** + ** The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. */ - int nMaxPendingData; - int nPendingData; - sqlite_int64 iPrevDocid; - Fts3Hash pendingTerms; + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + +#if defined(SQLITE_DEBUG) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribution to the FTS computation; they are used for + ** verifying the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif }; /* @@ -171,8 +246,10 @@ struct Fts3Cursor { char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -203,47 +280,70 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +} doclist; + /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. -** -** The nDocMatch and nMatch variables contain data that may be used by the -** matchinfo() function. They are populated when the full-text index is -** queried for hits on the phrase. If one or more tokens in the phrase -** are deferred, the nDocMatch and nMatch variables are populated based -** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ - int bFulltext; /* True if full-text index was used */ - Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ }; struct Fts3Phrase { - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, -** containing the results of the NEAR or phrase query in FTS3 doclist -** format. As usual, the initial "Length" field found in doclists stored -** on disk is omitted from this buffer. +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. ** -** Variable pCurrent always points to the start of a docid field within -** aDoclist. Since the doclist is usually scanned in docid order, this can -** be used to accelerate seeking to the required docid within the doclist. +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ @@ -253,12 +353,13 @@ struct Fts3Expr { Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ - sqlite3_int64 iCurrent; - char *pCurrent; + u32 *aMI; }; /* @@ -286,12 +387,12 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); +int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); @@ -300,17 +401,18 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); -int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); -void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -327,7 +429,7 @@ struct Fts3SegFilter { int flags; }; -struct Fts3SegReaderCursor { +struct Fts3MultiSegReader { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ @@ -336,8 +438,12 @@ struct Fts3SegReaderCursor { char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ - /* Cost of running this iterator. Used by fts3.c only. */ - int nCost; + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ @@ -352,11 +458,9 @@ int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); -int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); @@ -380,9 +484,34 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +int sqlite3Fts3InitTerm(sqlite3 *db); #endif /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); +int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +); + +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); + +int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + +#endif /* SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 6108689a..ada85d79 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -11,10 +11,9 @@ ****************************************************************************** ** */ - +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include @@ -28,7 +27,7 @@ struct Fts3auxTable { struct Fts3auxCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ @@ -96,6 +95,7 @@ static int fts3auxConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -342,6 +342,7 @@ static int fts3auxFilterMethod( int isScan; UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); assert( idxStr==0 ); assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 @@ -375,7 +376,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -459,7 +460,10 @@ int sqlite3Fts3InitAux(sqlite3 *db){ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ - 0 /* xRename */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; int rc; /* Return code */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 43f6d84a..7eb2962d 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -15,6 +15,7 @@ ** syntax is relatively simple, the whole tokenizer/parser system is ** hand-coded. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* @@ -77,16 +78,24 @@ int sqlite3_fts3_enable_parentheses = 0; */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 -#include "fts3Int.h" #include #include +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; @@ -172,7 +181,7 @@ static int getNextToken( iEnd++; } if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pRet->pPhrase->isNot = 1; + pParse->isNot = 1; } } nConsumed = iEnd; @@ -224,36 +233,55 @@ static int getNextString( char *zTemp = 0; int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ - const char *zToken; - int nToken, iBegin, iEnd, iPos; - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + const char *zByte; + int nByte, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); - if( !p || !zTemp ){ - goto no_mem; - } - if( ii==0 ){ - memset(p, 0, nByte); - p->pPhrase = (Fts3Phrase *)&p[1]; - } - p->pPhrase = (Fts3Phrase *)&p[1]; - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); - p->pPhrase->nToken = ii+1; - p->pPhrase->aToken[ii].n = nToken; - memcpy(&zTemp[nTemp], zToken, nToken); - nTemp += nToken; - if( iEndpPhrase->aToken[ii].isPrefix = 1; - }else{ - p->pPhrase->aToken[ii].isPrefix = 0; - } + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndpPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); - p = fts3ReallocOrFree(p, nByte + nTemp); - if( !p ){ - goto no_mem; - } - if( zTemp ){ - zNew = &(((char *)p)[nByte]); - memcpy(zNew, zTemp, nTemp); - }else{ - memset(p, 0, nByte+nTemp); - } - p->pPhrase = (Fts3Phrase *)&p[1]; - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = &zNew[nNew]; - nNew += p->pPhrase->aToken[jj].n; - } - sqlite3_free(zTemp); + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } rc = SQLITE_OK; } @@ -341,6 +365,8 @@ static int getNextNode( const char *zInput = z; int nInput = n; + pParse->isNot = 0; + /* Skip over any whitespace before checking for a keyword, an open or ** close bracket, or a quoted string. */ @@ -559,7 +585,7 @@ static int fts3ExprParse( int isPhrase; if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); @@ -577,7 +603,6 @@ static int fts3ExprParse( p = pPrev; }else{ int eType = p->eType; - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); /* The isRequirePhrase variable is set to true if a phrase or @@ -740,9 +765,11 @@ int sqlite3Fts3ExprParse( */ void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - sqlite3_free(p->aDoclist); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); sqlite3_free(p); } } @@ -800,7 +827,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; zBuf = sqlite3_mprintf( - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 98be5296..b7c3e8b5 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -23,6 +23,7 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include diff --git a/ext/fts3/fts3_icu.c b/ext/fts3/fts3_icu.c index 85390d3b..a10a55d6 100644 --- a/ext/fts3/fts3_icu.c +++ b/ext/fts3/fts3_icu.c @@ -10,10 +10,8 @@ ** ************************************************************************* ** This file implements a tokenizer for fts3 based on the ICU library. -** -** $Id: fts3_icu.c,v 1.3 2008/09/01 18:34:20 danielk1977 Exp $ */ - +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #ifdef SQLITE_ENABLE_ICU diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 27f9cf39..148c5700 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -22,9 +22,8 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 6b745350..b569eb13 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -11,9 +11,9 @@ ****************************************************************************** */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include @@ -176,51 +176,6 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } -/* -** The argument to this function is always a phrase node. Its doclist -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes -** to the left of this one in the query tree have already been loaded. -** -** If this phrase node is part of a series of phrase nodes joined by -** NEAR operators (and is not the left-most of said series), then elements are -** removed from the phrases doclist consistent with the NEAR restriction. If -** required, elements may be removed from the doclists of phrases to the -** left of this one that are part of the same series of NEAR operator -** connected phrases. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ExprNearTrim(Fts3Expr *pExpr){ - int rc = SQLITE_OK; - Fts3Expr *pParent = pExpr->pParent; - - assert( pExpr->eType==FTSQUERY_PHRASE ); - while( rc==SQLITE_OK - && pParent - && pParent->eType==FTSQUERY_NEAR - && pParent->pRight==pExpr - ){ - /* This expression (pExpr) is the right-hand-side of a NEAR operator. - ** Find the expression to the left of the same operator. - */ - int nNear = pParent->nNear; - Fts3Expr *pLeft = pParent->pLeft; - - if( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pRight; - } - - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); - - pExpr = pLeft; - pParent = pExpr->pParent; - } - - return rc; -} - /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also @@ -228,20 +183,13 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){ */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; UNUSED_PARAMETER(iPhrase); p->nPhrase++; - p->nToken += pExpr->pPhrase->nToken; - - if( pExpr->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pExpr->isLoaded = 1; - if( rc==SQLITE_OK ){ - rc = fts3ExprNearTrim(pExpr); - } - } + p->nToken += pPhrase->nToken; return rc; } @@ -415,7 +363,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -772,26 +720,6 @@ static int fts3ColumnlistCount(char **ppCollist){ return nEntry; } -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ - char *pCsr = *pp; - while( *pCsr ){ - int nHit; - sqlite3_int64 iCol = 0; - if( *pCsr==0x01 ){ - pCsr++; - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); - } - nHit = fts3ColumnlistCount(&pCsr); - assert( nHit>0 ); - if( isGlobal ){ - aOut[iCol*3+1]++; - } - aOut[iCol*3] += nHit; - } - pCsr++; - *pp = pCsr; -} - /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -825,48 +753,9 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - Fts3Cursor *pCsr = p->pCursor; - char *pIter; - char *pEnd; - char *pFree = 0; - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - - assert( pExpr->isLoaded ); - assert( pExpr->eType==FTSQUERY_PHRASE ); - - if( pCsr->pDeferred ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - for(ii=0; iinToken; ii++){ - if( pPhrase->aToken[ii].bFulltext ) break; - } - if( iinToken ){ - int nFree = 0; - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); - if( rc!=SQLITE_OK ) return rc; - pIter = pFree; - pEnd = &pFree[nFree]; - }else{ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - return SQLITE_OK; - } - }else{ - pIter = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - } - - /* Fill in the global hit count matrix row for this phrase. */ - while( pIterpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* @@ -883,14 +772,13 @@ static int fts3ExprLocalHitsCb( int iStart = iPhrase * p->nCol * 3; int i; - for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - - if( pExpr->aDoclist ){ + for(i=0; inCol; i++){ char *pCsr; - - pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); if( pCsr ){ - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } } @@ -960,7 +848,7 @@ static int fts3MatchinfoSelectDoctotal( a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return SQLITE_CORRUPT; + if( nDoc==0 ) return SQLITE_CORRUPT_VTAB; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; @@ -976,9 +864,8 @@ static int fts3MatchinfoSelectDoctotal( typedef struct LcsIterator LcsIterator; struct LcsIterator { Fts3Expr *pExpr; /* Pointer to phrase expression */ - char *pRead; /* Cursor used to iterate through aDoclist */ int iPosOffset; /* Tokens count up to end of this phrase */ - int iCol; /* Current column number */ + char *pRead; /* Cursor used to iterate through aDoclist */ int iPos; /* Current position */ }; @@ -1009,17 +896,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ int rc = 0; pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 ){ - pIter->iCol = LCS_ITERATOR_FINISHED; + if( iRead==0 || iRead==1 ){ + pRead = 0; rc = 1; }else{ - if( iRead==1 ){ - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - pIter->iCol = (int)iRead; - pIter->iPos = pIter->iPosOffset; - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - rc = 1; - } pIter->iPos += (int)(iRead-2); } @@ -1051,42 +931,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1); - if( pIter->pRead ){ - pIter->iPos = pIter->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - }else{ - pIter->iCol = LCS_ITERATOR_FINISHED; - } } for(iCol=0; iColnCol; iCol++){ int nLcs = 0; /* LCS value for this column */ int nLive = 0; /* Number of iterators in aIter not at EOF */ - /* Loop through the iterators in aIter[]. Set nLive to the number of - ** iterators that point to a position-list corresponding to column iCol. - */ for(i=0; inPhrase; i++){ - assert( aIter[i].iCol>=iCol ); - if( aIter[i].iCol==iCol ) nLive++; + LcsIterator *pIt = &aIter[i]; + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } } - /* The following loop runs until all iterators in aIter[] have finished - ** iterating through positions in column iCol. Exactly one of the - ** iterators is advanced each time the body of the loop is run. - */ while( nLive>0 ){ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ int nThisLcs = 0; /* LCS for the current iterator positions */ for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; - if( iCol!=pIter->iCol ){ + if( pIter->pRead==0 ){ /* This iterator is already at EOF for this column. */ nThisLcs = 0; }else{ @@ -1152,7 +1024,7 @@ static int fts3MatchinfoValues( case FTS3_MATCHINFO_NDOC: if( bGlobal ){ - sqlite3_int64 nDoc; + sqlite3_int64 nDoc = 0; rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); pInfo->aMatchinfo[0] = (u32)nDoc; } @@ -1408,6 +1280,7 @@ struct TermOffset { }; struct TermOffsetCtx { + Fts3Cursor *pCsr; int iCol; /* Column of table to populate aTerm for */ int iTerm; sqlite3_int64 iDocid; @@ -1425,7 +1298,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); @@ -1478,6 +1351,7 @@ void sqlite3Fts3Offsets( goto offsets_out; } sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; /* Loop through the table columns, appending offset information to ** string-buffer res for each column. @@ -1553,7 +1427,7 @@ void sqlite3Fts3Offsets( ); rc = fts3StringAppend(&res, aBuffer, -1); }else if( rc==SQLITE_DONE ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } } diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c new file mode 100644 index 00000000..d3eb690b --- /dev/null +++ b/ext/fts3/fts3_term.c @@ -0,0 +1,369 @@ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is not part of the production FTS code. It is only used for +** testing. It contains a virtual table implementation that provides direct +** access to the full-text index of an FTS table. +*/ + +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_TEST + +#include +#include +#include + +typedef struct Fts3termTable Fts3termTable; +typedef struct Fts3termCursor Fts3termCursor; + +struct Fts3termTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + int iIndex; /* Index for Fts3Table.aIndex[] */ + Fts3Table *pFts3Tab; +}; + +struct Fts3termCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + + int isEof; /* True if cursor is at EOF */ + char *pNext; + + sqlite3_int64 iRowid; /* Current 'rowid' value */ + sqlite3_int64 iDocid; /* Current 'docid' value */ + int iCol; /* Current 'col' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, docid, col, pos)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3termConnectMethod( + sqlite3 *db, /* Database connection */ + void *pCtx, /* Non-zero for an fts4prefix table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3termTable *p; /* Virtual table object to return */ + int iIndex = 0; + + if( argc==5 ){ + iIndex = atoi(argv[4]); + argc--; + } + + /* The user should specify a single argument - the name of an fts3 table. */ + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "wrong number of arguments to fts4term constructor" + ); + return SQLITE_ERROR; + } + + zDb = argv[1]; + nDb = strlen(zDb); + zFts3 = argv[3]; + nFts3 = strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3termTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = iIndex+1; + p->iIndex = iIndex; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3termTable *p = (Fts3termTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3termBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + UNUSED_PARAMETER(pVTab); + + /* This vtab naturally does "ORDER BY term, docid, col, pos". */ + if( pInfo->nOrderBy ){ + int i; + for(i=0; inOrderBy; i++){ + if( pInfo->aOrderBy[i].iColumn!=i || pInfo->aOrderBy[i].desc ) break; + } + if( i==pInfo->nOrderBy ){ + pInfo->orderByConsumed = 1; + } + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3termOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3termCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3termCursor *)sqlite3_malloc(sizeof(Fts3termCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3termCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3termCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + int rc; + sqlite3_int64 v; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + /* Advance to the next term in the full-text index. */ + if( pCsr->csr.aDoclist==0 + || pCsr->pNext>=&pCsr->csr.aDoclist[pCsr->csr.nDoclist-1] + ){ + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc!=SQLITE_ROW ){ + pCsr->isEof = 1; + return rc; + } + + pCsr->iCol = 0; + pCsr->iPos = 0; + pCsr->iDocid = 0; + pCsr->pNext = pCsr->csr.aDoclist; + + /* Read docid */ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &pCsr->iDocid); + } + + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + if( v==0 ){ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iDocid += v; + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iCol = 0; + pCsr->iPos = 0; + } + + if( v==1 ){ + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + pCsr->iCol += v; + pCsr->iPos = 0; + pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v); + } + + pCsr->iPos += (v - 2); + + return SQLITE_OK; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3termFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + Fts3termTable *p = (Fts3termTable *)pCursor->pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int rc; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(apVal); + + assert( idxStr==0 && idxNum==0 ); + + /* In case this cursor is being reused, close and zero it. */ + testcase(pCsr->filter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + if( rc==SQLITE_OK ){ + rc = fts3termNextMethod(pCursor); + } + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3termEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3termColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3termCursor *p = (Fts3termCursor *)pCursor; + + assert( iCol>=0 && iCol<=3 ); + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(pCtx, p->iDocid); + break; + case 2: + sqlite3_result_int64(pCtx, p->iCol); + break; + default: + sqlite3_result_int64(pCtx, p->iPos); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3termRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3term module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTerm(sqlite3 *db){ + static const sqlite3_module fts3term_module = { + 0, /* iVersion */ + fts3termConnectMethod, /* xCreate */ + fts3termConnectMethod, /* xConnect */ + fts3termBestIndexMethod, /* xBestIndex */ + fts3termDisconnectMethod, /* xDisconnect */ + fts3termDisconnectMethod, /* xDestroy */ + fts3termOpenMethod, /* xOpen */ + fts3termCloseMethod, /* xClose */ + fts3termFilterMethod, /* xFilter */ + fts3termNextMethod, /* xNext */ + fts3termEofMethod, /* xEof */ + fts3termColumnMethod, /* xColumn */ + fts3termRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0 /* xRename */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c new file mode 100644 index 00000000..c5f72d84 --- /dev/null +++ b/ext/fts3/fts3_test.c @@ -0,0 +1,321 @@ +/* +** 2011 Jun 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is not part of the production FTS code. It is only used for +** testing. It contains a Tcl command that can be used to test if a document +** matches an FTS NEAR expression. +*/ + +#include +#include +#include + +/* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ +#include "fts3Int.h" + +#define NM_MAX_TOKEN 12 + +typedef struct NearPhrase NearPhrase; +typedef struct NearDocument NearDocument; +typedef struct NearToken NearToken; + +struct NearDocument { + int nToken; /* Length of token in bytes */ + NearToken *aToken; /* Token array */ +}; + +struct NearToken { + int n; /* Length of token in bytes */ + const char *z; /* Pointer to token string */ +}; + +struct NearPhrase { + int nNear; /* Preceding NEAR value */ + int nToken; /* Number of tokens in this phrase */ + NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */ +}; + +static int nm_phrase_match( + NearPhrase *p, + NearToken *aToken +){ + int ii; + + for(ii=0; iinToken; ii++){ + NearToken *pToken = &p->aToken[ii]; + if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){ + if( aToken[ii].n<(pToken->n-1) ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0; + }else{ + if( aToken[ii].n!=pToken->n ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0; + } + } + + return 1; +} + +static int nm_near_chain( + int iDir, /* Direction to iterate through aPhrase[] */ + NearDocument *pDoc, /* Document to match against */ + int iPos, /* Position at which iPhrase was found */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase found */ +){ + int iStart; + int iStop; + int ii; + int nNear; + int iPhrase2; + NearPhrase *p; + NearPhrase *pPrev; + + assert( iDir==1 || iDir==-1 ); + + if( iDir==1 ){ + if( (iPhrase+1)==nPhrase ) return 1; + nNear = aPhrase[iPhrase+1].nNear; + }else{ + if( iPhrase==0 ) return 1; + nNear = aPhrase[iPhrase].nNear; + } + pPrev = &aPhrase[iPhrase]; + iPhrase2 = iPhrase+iDir; + p = &aPhrase[iPhrase2]; + + iStart = iPos - nNear - p->nToken; + iStop = iPos + nNear + pPrev->nToken; + + if( iStart<0 ) iStart = 0; + if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; + + for(ii=iStart; ii<=iStop; ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1; + } + } + + return 0; +} + +static int nm_match_count( + NearDocument *pDoc, /* Document to match against */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase to count matches for */ +){ + int nOcc = 0; + int ii; + NearPhrase *p = &aPhrase[iPhrase]; + + for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + /* Test forward NEAR chain (i>iPhrase) */ + if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; + + /* Test reverse NEAR chain (iNM_MAX_TOKEN ){ + Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + rc = TCL_ERROR; + goto near_match_out; + } + for(jj=0; jjz = Tcl_GetStringFromObj(apToken[jj], &pT->n); + } + aPhrase[ii].nToken = nToken; + } + for(ii=1; ii0)); + + near_match_out: + ckfree((char *)aPhrase); + ckfree((char *)doc.aToken); + return rc; +} + +/* +** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD? +** +** Normally, FTS uses hard-coded values to determine the minimum doclist +** size eligible for incremental loading, and the size of the chunks loaded +** when a doclist is incrementally loaded. This command allows the built-in +** values to be overridden for testing purposes. +** +** If present, the first argument is the chunksize in bytes to load doclists +** in. The second argument is the minimum doclist size in bytes to use +** incremental loading with. +** +** Whether or not the arguments are present, this command returns a list of +** two integers - the initial chunksize and threshold when the command is +** invoked. This can be used to restore the default behaviour after running +** tests. For example: +** +** # Override incr-load settings for testing: +** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] +** +** .... run tests .... +** +** # Restore initial incr-load settings: +** eval fts3_configure_incr_load $cfg +*/ +static int fts3_configure_incr_load_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + extern int test_fts3_node_chunksize; + extern int test_fts3_node_chunk_threshold; + Tcl_Obj *pRet; + + if( objc!=1 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?"); + return TCL_ERROR; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize)); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold)); + + if( objc==3 ){ + int iArg1; + int iArg2; + if( Tcl_GetIntFromObj(interp, objv[1], &iArg1) + || Tcl_GetIntFromObj(interp, objv[2], &iArg2) + ){ + Tcl_DecrRefCount(pRet); + return TCL_ERROR; + } + test_fts3_node_chunksize = iArg1; + test_fts3_node_chunk_threshold = iArg2; + } + + Tcl_SetObjResult(interp, pRet); + Tcl_DecrRefCount(pRet); +#endif + return TCL_OK; +} + +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + Tcl_CreateObjCommand(interp, + "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 + ); + return TCL_OK; +} diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index aa289334..c4ad6a5c 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -23,14 +23,14 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif - #include "fts3Int.h" + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + #include #include @@ -156,7 +156,7 @@ int sqlite3Fts3InitTokenizer( ){ int rc; char *z = (char *)zArg; - int n; + int n = 0; char *zCopy; char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c index 432c35d1..d11a4997 100644 --- a/ext/fts3/fts3_tokenizer1.c +++ b/ext/fts3/fts3_tokenizer1.c @@ -22,9 +22,8 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - #include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include #include diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 1e718743..36f2249e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -17,9 +17,9 @@ ** code in fts3.c. */ +#include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#include "fts3Int.h" #include #include #include @@ -36,14 +36,40 @@ */ #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; @@ -74,7 +100,6 @@ struct Fts3DeferredToken { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -93,6 +118,9 @@ struct Fts3SegReader { char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -106,8 +134,11 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; @@ -145,6 +176,14 @@ struct SegmentWriter { ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ @@ -175,10 +214,10 @@ struct SegmentNode { #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 @@ -187,6 +226,11 @@ struct SegmentNode { #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 + +#define SQL_DELETE_SEGDIR_RANGE 26 + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -222,10 +266,11 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -235,6 +280,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -291,7 +341,7 @@ static int fts3SelectDocsize( rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; + if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT_VTAB; pStmt = 0; }else{ rc = SQLITE_OK; @@ -390,14 +440,32 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ ** 3: end_block ** 4: root */ -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ +int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ int rc; sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + if( iLevel<0 ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + } }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + } } *ppStmt = pStmt; return rc; @@ -512,6 +580,47 @@ static int fts3PendingListAppend( return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -542,6 +651,14 @@ static int fts3PendingTermsAdd( assert( pTokenizer && pModule ); + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; @@ -552,8 +669,7 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ - PendingList *pList; - + int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -564,22 +680,19 @@ static int fts3PendingTermsAdd( break; } - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; - } - } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); } } @@ -609,14 +722,19 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ } /* -** Discard the contents of the pending-terms hash table. +** Discard the contents of the pending-terms hash tables. */ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); } - fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } @@ -632,11 +750,9 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); - if( zText ){ - int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); + if( rc!=SQLITE_OK ){ + return rc; } aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } @@ -741,14 +857,14 @@ static int fts3DeleteAll(Fts3Table *p){ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ - sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ + sqlite3_value *pRowid, /* The docid to be deleted */ u32 *aSz /* Sizes of deleted document written here */ ){ int rc; sqlite3_stmt *pSelect; if( *pRC ) return; - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; @@ -774,7 +890,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int); +static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -791,7 +907,12 @@ static int fts3SegmentMerge(Fts3Table *, int); ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ @@ -799,7 +920,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iLevel); + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -813,7 +934,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iLevel); + rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -854,7 +975,8 @@ int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob /* OUT: Size of blob data */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ ){ int rc; /* Return code */ @@ -875,11 +997,16 @@ int sqlite3Fts3ReadBlock( if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; if( paBlob ){ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); memset(&aByte[nByte], 0, FTS3_NODE_PADDING); if( rc!=SQLITE_OK ){ @@ -889,7 +1016,6 @@ int sqlite3Fts3ReadBlock( } *paBlob = aByte; } - *pnBlob = nByte; } return rc; @@ -903,13 +1029,55 @@ void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -921,7 +1089,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -941,6 +1108,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; } pReader->aNode = 0; @@ -952,21 +1121,31 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } pNext = pReader->aNode; } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } if( nPrefix+nSuffix>pReader->nTermAlloc ){ @@ -978,6 +1157,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; @@ -990,9 +1173,9 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** of these statements is untrue, then the data structure is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || pReader->aDoclist[pReader->nDoclist-1] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } return SQLITE_OK; } @@ -1001,12 +1184,26 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int n; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; } /* @@ -1019,128 +1216,125 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ -static void fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ + int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( *p | c ) c = *p++ & 0x80; - p++; + assert( p ); - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } }else{ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } + } } + + return SQLITE_OK; } -/* -** This function is called to estimate the amount of data that will be -** loaded from the disk If SegReaderIterate() is called on this seg-reader, -** in units of average document size. -** -** This can be used as follows: If the caller has a small doclist that -** contains references to N documents, and is considering merging it with -** a large doclist (size X "average documents"), it may opt not to load -** the large doclist if X>N. -*/ -int sqlite3Fts3SegReaderCost( - Fts3Cursor *pCsr, /* FTS3 cursor handle */ - Fts3SegReader *pReader, /* Segment-reader handle */ - int *pnCost /* IN/OUT: Number of bytes read */ + +int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl ){ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Return code */ - int nCost = 0; /* Cost in bytes to return */ - int pgsz = p->nPgsz; /* Database page size */ + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; - /* If this seg-reader is reading the pending-terms table, or if all data - ** for the segment is stored on the root page of the b-tree, then the cost - ** is zero. In this case all required data is already in main memory. - */ - if( p->bHasStat - && !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - int nBlob = 0; - sqlite3_int64 iBlock; + assert( p->bHasStat ); + assert( pgsz>0 ); - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - int nOvfl = (nBlob + 34)/pgsz; - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } } } } - - *pnCost += nCost; + *pnOvfl = nOvfl; return rc; } @@ -1153,6 +1347,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); } } sqlite3_free(pReader); @@ -1229,24 +1424,42 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ /* ** This function is used to allocate an Fts3SegReader that iterates through ** a subset of the terms stored in the Fts3Table.pendingTerms array. +** +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): +** +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite +** +** Whereas if isPrefixIter is zero, the terms visited are: +** +** firebird mysql sqlite */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isPrefix, /* True for a term-prefix query */ + int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; - if( isPrefix ){ + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ @@ -1263,6 +1476,7 @@ int sqlite3Fts3SegReaderPending( } aElem = aElem2; } + aElem[nElem++] = pE; } } @@ -1276,7 +1490,9 @@ int sqlite3Fts3SegReaderPending( } }else{ - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. */ + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -1296,7 +1512,7 @@ int sqlite3Fts3SegReaderPending( } } - if( isPrefix ){ + if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -1360,6 +1576,18 @@ static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ assert( pLhs->aNode && pRhs->aNode ); return rc; } +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} /* ** Compare the term that the Fts3SegReader object passed as the first argument @@ -1888,16 +2116,16 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ ** The first value in the apVal[] array is assumed to contain an integer. ** This function tests if there exist any documents with docid values that ** are different from that integer. i.e. if deleting the document with docid -** apVal[0] would mean the FTS3 table were empty. +** pRowid would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for -** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ -static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pisEmpty = sqlite3_column_int(pStmt, 0); @@ -1908,21 +2136,30 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ } /* -** Set *pnSegment to the total number of segments in the database. Set -** *pnMax to the largest segment level in the database (segment levels -** are stored in the 'level' column of the %_segdir table). +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; + assert( iIndex>=0 && iIndexnIndex ); - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - *pnMax = sqlite3_column_int(pStmt, 1); + *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -1943,6 +2180,7 @@ static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ @@ -1965,18 +2203,23 @@ static int fts3DeleteSegdir( return rc; } + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - sqlite3Fts3PendingTermsClear(p); - }else{ - assert( iLevel>=0 ); - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iLevel); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } return rc; @@ -2025,15 +2268,106 @@ static void fts3ColumnFilter( *pnList = nList; } -int sqlite3Fts3SegReaderStart( +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList +){ + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +int sqlite3Fts3MsrIncrNext( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr, /* Cursor object */ - Fts3SegFilter *pFilter /* Restrictions on range of iteration */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + } + + if( nList>0 ){ + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + *paPoslist = pMsr->aBuffer; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + }else{ + *paPoslist = pList; + } + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ ){ int i; - - /* Initialize the cursor object */ - pCsr->pFilter = pFilter; + int nSeg = pCsr->nSegment; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of @@ -2041,24 +2375,105 @@ int sqlite3Fts3SegReaderStart( ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - for(i=0; inSegment; i++){ - int nTerm = pFilter->nTerm; - const char *zTerm = pFilter->zTerm; + for(i=0; pCsr->bRestart==0 && inSegment; i++){ Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } - fts3SegReaderSort( - pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); return SQLITE_OK; } +int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + + int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; @@ -2071,6 +2486,9 @@ int sqlite3Fts3SegReaderStep( Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( pCsr->nSegment==0 ) return SQLITE_OK; @@ -2082,7 +2500,7 @@ int sqlite3Fts3SegReaderStep( ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ - rc = fts3SegReaderNext(p, apSegment[i]); + rc = fts3SegReaderNext(p, apSegment[i], 0); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); @@ -2121,10 +2539,18 @@ int sqlite3Fts3SegReaderStep( } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ - pCsr->aDoclist = apSegment[0]->aDoclist; + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ pCsr->nDoclist = apSegment[0]->nDoclist; - rc = SQLITE_ROW; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ @@ -2134,22 +2560,22 @@ int sqlite3Fts3SegReaderStep( ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } @@ -2158,7 +2584,19 @@ int sqlite3Fts3SegReaderStep( } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; @@ -2168,9 +2606,7 @@ int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint( - &pCsr->aBuffer[nDoclist], iDocid-iPrev - ); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); @@ -2179,7 +2615,7 @@ int sqlite3Fts3SegReaderStep( } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; @@ -2193,8 +2629,9 @@ int sqlite3Fts3SegReaderStep( return rc; } + void sqlite3Fts3SegReaderFinish( - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ if( pCsr ){ int i; @@ -2221,43 +2658,56 @@ void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level to create new segment at */ + int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically - ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. */ - int nDummy; /* TODO: Remove this */ + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); + bIgnoreEmpty = 1; + + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ - /* This call is to merge all segments at level iLevel. Find the next + /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - iNewLevel = iLevel+1; - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=0 ); + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ @@ -2269,8 +2719,10 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); - if( rc!=SQLITE_OK ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); + if( rc!=SQLITE_OK ) goto finished; + } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: @@ -2281,10 +2733,17 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ /* -** Flush the contents of pendingTerms to a level 0 segment. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + int rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + return rc; } /* @@ -2435,6 +2894,23 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int i; + int bSeenDone = 0; + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -2451,12 +2927,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - }else{ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -2469,44 +2940,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } - sqlite3Fts3SegmentsClose(p); return rc; } -/* -** Return the deferred doclist associated with deferred token pDeferred. -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already -** been called to allocate and populate the doclist. -*/ -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ - if( pDeferred->pList ){ - *pnByte = pDeferred->pList->nData; - return pDeferred->pList->aData; - } - *pnByte = 0; - return 0; -} - -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - if( pExpr->isLoaded ){ - sqlite3_free(pExpr->aDoclist); - pExpr->isLoaded = 0; - pExpr->aDoclist = 0; - pExpr->nDoclist = 0; - pExpr->pCurrent = 0; - pExpr->iCurrent = 0; - } - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -2514,12 +2950,9 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* @@ -2531,7 +2964,7 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; @@ -2596,6 +3029,33 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ return rc; } +int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ @@ -2621,6 +3081,40 @@ int sqlite3Fts3DeferToken( return SQLITE_OK; } +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnDoc, + u32 *aSzDel +){ + int isEmpty = 0; + int rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p); + *pnDoc = *pnDoc - 1; + }else{ + sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); + rc = fts3PendingTermsDocid(p, iRemove); + fts3DeleteTerms(&rc, p, pRowid, aSzDel); + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + + return rc; +} /* ** This function does the work for the xUpdate method of FTS3 virtual @@ -2636,49 +3130,97 @@ int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; assert( p->pSegments==0 ); + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); - /* If this is a DELETE or UPDATE operation, remove the old record. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty = 0; - rc = fts3IsEmpty(p, apVal, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. - */ - rc = fts3DeleteAll(p); + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); }else{ - isRemove = 1; - iRemove = sqlite3_value_int64(apVal[0]); - rc = fts3PendingTermsDocid(p, iRemove); - fts3DeleteTerms(&rc, p, apVal, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); - } - nChng--; + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; } } - }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - sqlite3_free(aSzIns); - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + isRemove = 1; + iRemove = sqlite3_value_int64(apVal[0]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ - rc = fts3InsertData(p, apVal, pRowid); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB; + } if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ rc = fts3PendingTermsDocid(p, *pRowid); } @@ -2695,6 +3237,7 @@ int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; @@ -2709,12 +3252,10 @@ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - sqlite3Fts3PendingTermsClear(p); - } + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index ebf430a9..2375069e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -517,17 +517,17 @@ nodeAcquire( if( pNode && iNode==1 ){ pRtree->iDepth = readInt16(pNode->zData); if( pRtree->iDepth>RTREE_MAX_DEPTH ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } /* If no error has occurred so far, check if the "number of entries" ** field on the node is too large. If so, set the return code to - ** SQLITE_CORRUPT. + ** SQLITE_CORRUPT_VTAB. */ if( pNode && rc==SQLITE_OK ){ if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } } @@ -535,7 +535,7 @@ nodeAcquire( if( pNode!=0 ){ nodeHashInsert(pRtree, pNode); }else{ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_VTAB; } *ppNode = pNode; }else{ @@ -1062,7 +1062,7 @@ static int nodeRowidIndex( return SQLITE_OK; } } - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } /* @@ -1421,7 +1421,7 @@ static float cellArea(Rtree *pRtree, RtreeCell *p){ float area = 1.0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + area = (float)(area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); } return area; } @@ -1434,7 +1434,7 @@ static float cellMargin(Rtree *pRtree, RtreeCell *p){ float margin = 0.0; int ii; for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + margin += (float)(DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); } return margin; } @@ -1519,7 +1519,7 @@ static float cellOverlap( o = 0.0; break; }else{ - o = o * (x2-x1); + o = o * (float)(x2-x1); } } overlap += o; @@ -1538,12 +1538,12 @@ static float cellOverlapEnlargement( int nCell, int iExclude ){ - float before; - float after; + double before; + double after; before = cellOverlap(pRtree, p, aCell, nCell, iExclude); cellUnion(pRtree, p, pInsert); after = cellOverlap(pRtree, p, aCell, nCell, iExclude); - return after-before; + return (float)(after-before); } #endif @@ -1565,11 +1565,11 @@ static int ChooseLeaf( for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; - sqlite3_int64 iBest; + sqlite3_int64 iBest = 0; - float fMinGrowth; - float fMinArea; - float fMinOverlap; + float fMinGrowth = 0.0; + float fMinArea = 0.0; + float fMinOverlap = 0.0; int nCell = NCELL(pNode); RtreeCell cell; @@ -1657,7 +1657,7 @@ static int AdjustTree( int iCell; if( nodeParentIndex(pRtree, p, &iCell) ){ - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_VTAB; } nodeGetCell(pRtree, pParent, iCell, &cell); @@ -1999,9 +1999,9 @@ static int splitNodeStartree( int *aSpare; int ii; - int iBestDim; - int iBestSplit; - float fBestMargin; + int iBestDim = 0; + int iBestSplit = 0; + float fBestMargin = 0.0; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -2023,9 +2023,9 @@ static int splitNodeStartree( for(ii=0; iinDim; ii++){ float margin = 0.0; - float fBestOverlap; - float fBestArea; - int iBestLeft; + float fBestOverlap = 0.0; + float fBestArea = 0.0; + int iBestLeft = 0; int nLeft; for( @@ -2329,7 +2329,7 @@ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ } rc = sqlite3_reset(pRtree->pReadParent); if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; pChild = pChild->pParent; } return rc; @@ -2340,7 +2340,7 @@ static int deleteCell(Rtree *, RtreeNode *, int, int); static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ int rc; int rc2; - RtreeNode *pParent; + RtreeNode *pParent = 0; int iCell; assert( pNode->nRef==1 ); @@ -2488,19 +2488,19 @@ static int Reinsert( } aOrder[ii] = ii; for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2+1]); } } for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = aCenterCoord[iDim]/((float)nCell*2.0); + aCenterCoord[iDim] = (float)(aCenterCoord[iDim]/((float)nCell*2.0)); } for(ii=0; iinDim; iDim++){ - float coord = DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2]); + float coord = (float)(DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); } } @@ -2599,10 +2599,10 @@ static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ /* Find a node to store this cell in. pNode->iNode currently contains ** the height of the sub-tree headed by the cell. */ - rc = ChooseLeaf(pRtree, &cell, pNode->iNode, &pInsert); + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); if( rc==SQLITE_OK ){ int rc2; - rc = rtreeInsertCell(pRtree, pInsert, &cell, pNode->iNode); + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); rc2 = nodeRelease(pRtree, pInsert); if( rc==SQLITE_OK ){ rc = rc2; @@ -2625,6 +2625,90 @@ static int newRowid(Rtree *pRtree, i64 *piRowid){ return rc; } +/* +** Remove the entry with rowid=iDelete from the r-tree structure. +*/ +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot; /* Root node of rtree structure */ + + + /* Obtain a reference to the root node to initialise Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf); + } + + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } + + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } + } + + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + sqlite3_free(pLeaf); + } + + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} + /* ** The xUpdate method for rtree module virtual tables. */ @@ -2636,103 +2720,25 @@ static int rtreeUpdate( ){ Rtree *pRtree = (Rtree *)pVtab; int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ rtreeReference(pRtree); - assert(nData>=1); - /* If azData[0] is not an SQL NULL value, it is the rowid of a - ** record to delete from the r-tree table. The following block does - ** just that. + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. */ - if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ - i64 iDelete; /* The rowid to delete */ - RtreeNode *pLeaf; /* Leaf node containing record iDelete */ - int iCell; /* Index of iDelete cell in pLeaf */ - RtreeNode *pRoot; - - /* Obtain a reference to the root node to initialise Rtree.iDepth */ - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - - /* Obtain a reference to the leaf node that contains the entry - ** about to be deleted. - */ - if( rc==SQLITE_OK ){ - iDelete = sqlite3_value_int64(azData[0]); - rc = findLeafNode(pRtree, iDelete, &pLeaf); - } - - /* Delete the cell in question from the leaf node. */ - if( rc==SQLITE_OK ){ - int rc2; - rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); - if( rc==SQLITE_OK ){ - rc = deleteCell(pRtree, pLeaf, iCell, 0); - } - rc2 = nodeRelease(pRtree, pLeaf); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - - /* Delete the corresponding entry in the _rowid table. */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); - sqlite3_step(pRtree->pDeleteRowid); - rc = sqlite3_reset(pRtree->pDeleteRowid); - } - - /* Check if the root node now has exactly one child. If so, remove - ** it, schedule the contents of the child for reinsertion and - ** reduce the tree height by one. - ** - ** This is equivalent to copying the contents of the child into - ** the root node (the operation that Gutman's paper says to perform - ** in this scenario). - */ - if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ - int rc2; - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - rc2 = nodeRelease(pRtree, pChild); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } - } - - /* Re-insert the contents of any underfull nodes removed from the tree. */ - for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ - if( rc==SQLITE_OK ){ - rc = reinsertNodeContent(pRtree, pLeaf); - } - pRtree->pDeleted = pLeaf->pNext; - sqlite3_free(pLeaf); - } - - /* Release the reference to the root node. */ - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRoot); - }else{ - nodeRelease(pRtree, pRoot); - } - } - - /* If the azData[] array contains more than one element, elements - ** (azData[2]..azData[argc-1]) contain a new record to insert into - ** the r-tree structure. - */ - if( rc==SQLITE_OK && nData>1 ){ - /* Insert a new record into the r-tree */ - RtreeCell cell; + if( nData>1 ){ int ii; - RtreeNode *pLeaf; /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */ assert( nData==(pRtree->nDim*2 + 3) ); @@ -2756,18 +2762,49 @@ static int rtreeUpdate( } } - /* Figure out the rowid of the new row. */ - if( sqlite3_value_type(azData[2])==SQLITE_NULL ){ - rc = newRowid(pRtree, &cell.iRowid); - }else{ + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ cell.iRowid = sqlite3_value_int64(azData[2]); - sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); - if( SQLITE_ROW==sqlite3_step(pRtree->pReadRowid) ){ - sqlite3_reset(pRtree->pReadRowid); - rc = SQLITE_CONSTRAINT; - goto constraint; + if( sqlite3_value_type(azData[0])==SQLITE_NULL + || sqlite3_value_int64(azData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = SQLITE_CONSTRAINT; + goto constraint; + } + } } - rc = sqlite3_reset(pRtree->pReadRowid); + bHaveRowid = 1; + } + } + + /* If azData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + } + + /* If the azData[] array contains more than one element, elements + ** (azData[2]..azData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = newRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; @@ -2812,7 +2849,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ } static sqlite3_module rtreeModule = { - 0, /* iVersion */ + 0, /* iVersion */ rtreeCreate, /* xCreate - create a table */ rtreeConnect, /* xConnect - connect to an existing table */ rtreeBestIndex, /* xBestIndex - Determine search strategy */ @@ -2831,7 +2868,10 @@ static sqlite3_module rtreeModule = { 0, /* xCommit - commit transaction */ 0, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ - rtreeRename /* xRename - rename the table */ + rtreeRename, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; static int rtreeSqlInit( @@ -2951,7 +2991,7 @@ static int getNodeSize( int rc; char *zSql; if( isCreate ){ - int iPageSize; + int iPageSize = 0; zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); rc = getIntFromStmt(db, zSql, &iPageSize); if( rc==SQLITE_OK ){ @@ -3008,6 +3048,8 @@ static int rtreeInit( return SQLITE_ERROR; } + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index fe5fa0ae..583b0285 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -31,6 +31,8 @@ source $testdir/tester.tcl # rtree-7.*: Test renaming an r-tree table. # rtree-8.*: Test constrained scans of r-tree data. # +# rtree-12.*: Test that on-conflict clauses are supported. +# ifcapable !rtree { finish_test @@ -416,4 +418,83 @@ do_test rtree-11.2 { } } {2} +#------------------------------------------------------------------------- +# Test on-conflict clause handling. +# +db_delete_and_reopen +do_execsql_test 12.0 { + CREATE VIRTUAL TABLE t1 USING rtree_i32(idx, x1, x2, y1, y2); + INSERT INTO t1 VALUES(1, 1, 2, 3, 4); + INSERT INTO t1 VALUES(2, 2, 3, 4, 5); + INSERT INTO t1 VALUES(3, 3, 4, 5, 6); + + CREATE TABLE source(idx, x1, x2, y1, y2); + INSERT INTO source VALUES(5, 8, 8, 8, 8); + INSERT INTO source VALUES(2, 7, 7, 7, 7); + +} +db_save_and_close +foreach {tn sql_template testdata} { + 1 "INSERT %CONF% INTO t1 VALUES(2, 7, 7, 7, 7)" { + ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 0 0 {1 1 2 3 4 2 7 7 7 7 3 3 4 5 6 4 4 5 6 7} + } + + 2 "INSERT %CONF% INTO t1 SELECT * FROM source" { + ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8} + FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8} + REPLACE 1 0 {1 1 2 3 4 2 7 7 7 7 3 3 4 5 6 4 4 5 6 7 5 8 8 8 8} + } + + 3 "UPDATE %CONF% t1 SET idx = 2 WHERE idx = 4" { + ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 1 0 {1 1 2 3 4 2 4 5 6 7 3 3 4 5 6} + } + + 3 "UPDATE %CONF% t1 SET idx = ((idx+1)%5)+1 WHERE idx > 2" { + ROLLBACK 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 1 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 1 0 {1 1 2 3 4 2 2 3 4 5 4 4 5 6 7 5 3 4 5 6} + FAIL 1 1 {1 1 2 3 4 2 2 3 4 5 4 4 5 6 7 5 3 4 5 6} + REPLACE 1 0 {1 4 5 6 7 2 2 3 4 5 5 3 4 5 6} + } + + 4 "INSERT %CONF% INTO t1 VALUES(2, 7, 6, 7, 7)" { + ROLLBACK 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6} + ABORT 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + IGNORE 0 0 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + FAIL 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + REPLACE 0 1 {1 1 2 3 4 2 2 3 4 5 3 3 4 5 6 4 4 5 6 7} + } + +} { + foreach {mode uses error data} $testdata { + db_restore_and_reopen + + set sql [string map [list %CONF% "OR $mode"] $sql_template] + set testname "12.$tn.[string tolower $mode]" + + execsql { + BEGIN; + INSERT INTO t1 VALUES(4, 4, 5, 6, 7); + } + + set res(0) {0 {}} + set res(1) {1 {constraint failed}} + do_catchsql_test $testname.1 $sql $res($error) + do_test $testname.2 [list sql_uses_stmt db $sql] $uses + do_execsql_test $testname.3 { SELECT * FROM t1 ORDER BY idx } $data + + do_test $testname.4 { rtree_check db t1 } 0 + db close + } +} finish_test diff --git a/main.mk b/main.mk index e1df7a77..33b2e6ec 100644 --- a/main.mk +++ b/main.mk @@ -54,8 +54,8 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ - fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ - func.o global.o hash.o \ + fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ + fts3_write.o func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ @@ -220,6 +220,8 @@ SRC += \ # Source code to the test files. # TESTSRC = \ + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/src/test1.c \ $(TOP)/src/test2.c \ $(TOP)/src/test3.c \ @@ -548,13 +550,8 @@ threadtest: threadtest3$(EXE) sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \ $(TOP)/tool/spaceanal.tcl - sed \ - -e '/^#/d' \ - -e 's,\\,\\\\,g' \ - -e 's,",\\",g' \ - -e 's,^,",' \ - -e 's,$$,\\n",' \ - $(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h + $(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl \ + >spaceanal_tcl.h $(TCCX) $(TCL_FLAGS) -DTCLSH=2 $(TESTFIXTURE_FLAGS) \ -DSQLITE_TEST=1 -DSQLITE_PRIVATE="" \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ diff --git a/manifest b/manifest index eaac4d64..b2251f24 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,12 @@ -C Version\s3.7.6.2 -D 2011-04-17T17:25:17.173 +C Release\scandidate\s3.7.7.1. +D 2011-06-28T17:39:05.200 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 7a4d9524721d40ef9ee26f93f9bd6a51dba106f2 +F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 +F Makefile.msc 11082f65b452b908d93013292c17850378c39284 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 7fbd89ecdebc63a53a5cedadb822b74f5ae42a24 +F VERSION 5d587aa6adf86fd587768fe736359bd9f5e43890 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@ -20,10 +21,10 @@ F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/src_logo.gif 9341ef09f0e53cd44c0c9b6fc3c16f7f3d6c2ad9 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 -F config.h.in 868fdb48c028421a203470e15c69ada15b9ba673 +F config.h.in 405a958bdb3af382a809dccb08a44694923ddd61 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure eb34ce3c85e7658b2eab3439f0e5b5c10b166b13 x -F configure.ac 87a3c71bbe9c925381c154413eea7f3cdc397244 +F configure b8e6049719c5f649670388236e2253183243b873 x +F configure.ac 298a759c086e72c013da459c2aec02a104f4224f F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html f0f682f50210928c07e562621c3b7e8ab912a538 F doc/pager-invariants.txt 870107036470d7c419e93768676fae2f8749cf9e @@ -61,29 +62,31 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5653c5654ac9b65bf3646af7e1d695c7e9b991a0 +F ext/fts3/fts3.c ca776037493d0081da70a6afc0df80f5ce347cb7 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 945926ea4b6a686c3e9834640a252d9870b7191e -F ext/fts3/fts3_aux.c 9e931f55eed8498dafe7bc1160f10cbb1a652fdf -F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 -F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c +F ext/fts3/fts3Int.h fa493ccbad78a2c99ad1c984f651c0c202e68536 +F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 +F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b +F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec -F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 -F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c e857c6a89d81d3b89df59f3b44b35c68d8ed5c62 -F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d +F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa +F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c +F ext/fts3/fts3_snippet.c 58b2ba2b934c1e2a2f6ac857d7f3c7e1a14b4532 +F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef +F ext/fts3/fts3_test.c c0aae3219df989d3e827d07846097adf8cb09945 +F ext/fts3/fts3_tokenizer.c 6089986ebcfc19d4b7aabd2b92773368efa4354f F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 -F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 813495ed106eb9461044e3c0374f4db69b37eb09 +F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 +F ext/fts3/fts3_write.c 194829c8fd024a448fc899e5ff02a8ed06595529 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c f5fa951eba03c41d292958064604a033021acdee +F ext/rtree/rtree.c b431c54d1ed05f04f2987e8a4fbb931acb40c312 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test dbd4250ac0ad367a262eb9676f7e3080b0368206 +F ext/rtree/rtree1.test 28e1b8da4da98093ce3210187434dd760a8d89d8 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test 0061e6f464fd3dc6a79f82454c5a1c3dadbe42af @@ -101,7 +104,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk bd4e376deea4704b2bd9c77a4e6f0fa3de25c495 +F main.mk d799fb600383cee11fed58876a7cccb749dd2614 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -114,37 +117,37 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad -F src/alter.c 280f5c04b11b492703a342222b3de0a999445280 +F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5 F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a -F src/attach.c 7f97ca76ef2453440170929531a9c778267c0830 +F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 6a9164af8a2ef4612ee30b253635a9bd8e5e1b1b -F src/btree.h 11753dd46597a20702bca8746cb4caa4486a82b5 +F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f +F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c f09c46c66a1e7668c6ee25c9a2518aaa6842044c +F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 -F src/date.c 1548fdac51377e4e7833251de878b4058c148e1b -F src/delete.c 7a24fcc9a31664d145acb97ce56b6d9f249a25e4 -F src/expr.c e3cf0957c6b8faaaf7386a3bc69e53c0dc9705be +F src/date.c d3c11de76392ea62637bfac0f4655889fc2f5a85 +F src/delete.c 4925f9121525fc871f5d8d13c1f7dcc91abb38bb +F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c a43ba8a005fb5efd1deeee06853e3a6120d46a91 -F src/func.c 3a8cb2fb2de3e3aed7f39106daf4878d9d17fcce -F src/global.c 02335177cf6946fe5525c6f0755cf181140debf3 +F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 +F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 +F src/global.c c70a46f28680f8d7c097dbc0430ccf3b932e90b0 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c acfb89fe4a73d703e425e167bfcc72985f4299ae +F src/insert.c 3eea5a53d2644116fb865afaa4699fabe62b441c F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c a8571665d43ff18f89a49d47a281605ce5ea825e -F src/malloc.c 788f2ed928786dfe305b6783d551d6b1a9080976 +F src/main.c fa654e1802e14a5f6c09c37971c3b69c1001d9c8 +F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf @@ -162,32 +165,32 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 4389231d079f8e323d9e24db6658bfd31f6d27a2 -F src/os_win.c 24d72407a90551969744cf9bcbb1b4c72c5fa845 -F src/pager.c 055239dcdfe12b3f5d97f6f01f85da01e2d6d912 +F src/os_unix.c 07acbb3e074e52b48a4248c06f66c9a91db1a0ce +F src/os_win.c eafcd6b91cf204a7ef29ac1ef2a1b7132e132e58 +F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 -F src/pcache.c 09d38c44ab275db581f7a2f6ff8b9bc7f8c0faaa +F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 -F src/pcache1.c d548e31beafa792d1994b663a29a5303569efc4e -F src/pragma.c 49c90ab27a4339d4b5bc0b03c08cbcf20ed8d454 +F src/pcache1.c 912bd5687d6df344698d8e69560f347b6e21c18a +F src/pragma.c ebcd20f1e654f5cb3aeef864ed69c4697719fbaa F src/prepare.c e64261559a3187698a3e7e6c8b001a4f4f98dab4 F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 649a6f10f7eb7b52a5a28847773cb9968a828ae8 -F src/shell.c 72e7e176bf46d5c6518d15ac4ad6847c4bb5df79 -F src/sqlite.h.in 4d28db70c37a1b17942820308eb59f211140da43 +F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff +F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 +F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h ac8f3f5846275c634f6649969304a9e97f6f9854 +F src/sqliteInt.h 7b72f7c6929757c0678d94524dbc7e7e3d6dc398 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 501c9a200fd998a268be475be5858febc90b725b -F src/test1.c 9ca440e80e16e53920904a0a5ac7feffb9b2c9a1 +F src/tclsqlite.c 5db825be61708b1a2b3f8f6e185e9b753829acef +F src/test1.c 8eef932f1024f1076a8adb8a6f5d9c5267c9e969 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 -F src/test3.c 056093cfef69ff4227a6bdb9108564dc7f45e4bc +F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 F src/test5.c e1a19845625144caf038031234a12185e40d315c F src/test6.c c7256cc21d2409486d094277d5b017e8eced44ba @@ -198,8 +201,8 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 -F src/test_config.c d536042f27226b4639f0f87d4795fd37428a9ddf -F src/test_demovfs.c 31050680fa6925b4f677cfd4fa965b5f19195e50 +F src/test_config.c 791a9acbf9a695cf9c12ff6bbe874dfde9a8c5f3 +F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5 F src/test_fuzzer.c f884f6f32e8513d34248d6e1ac8a32047fead254 @@ -207,16 +210,16 @@ F src/test_hexio.c c4773049603151704a6ab25ac5e936b5109caf5a F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 -F src/test_journal.c 785edd54f963aefb3c1628124170a56697c68c70 +F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e -F src/test_malloc.c fd6188b1501c0010fb4241ddc9f0d5ac402c688d -F src/test_multiplex.c fdabd793ee7a9642c5a8a470def2347144c46d05 +F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 +F src/test_multiplex.c a7457a1ac46964b7f897d75ecfd447410ec067e6 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec F src/test_osinst.c 62b0b8ef21ce754cc94e17bb42377ed8795dba32 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 -F src/test_quota.c b5576f17d701af461effd7ca1e71f0d100071192 +F src/test_quota.c cc4f67e12558a252ea4a11720be268348f4b1595 F src/test_rtree.c 30c981837445a4e187ee850a49c4760d9642f7c3 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -224,34 +227,35 @@ F src/test_stat.c f682704b5d1ba8e1d4e7e882a6d7922e2dcf066c F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 162c4ec0137a549c009bb9ecab550527743cfc5d F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa -F src/test_thread.c 361ae0a0f1cbf5a28ad0388a258b104017a370c0 -F src/test_vfs.c 2ed8853c1e51ac6f9ea091f7ce4e0d618bba8b86 -F src/test_vfstrace.c 2265c9895f350c8d3c39b079998fbe7481505cc1 +F src/test_thread.c fe9a7803fc1d69cccb60f016f28c1cedf2d9fcfa +F src/test_vfs.c 956cb3f5cbd2a0d09129540e615bb0fb761c083d +F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 604607d6813e9551cf5189d899e0a25c12681080 -F src/trigger.c 144cc18bb701f3286484aae4292a9531f09278c8 -F src/update.c 81911be16ece3c3e7716aa18565b4814ec41f8b9 -F src/utf.c d83650c3ea08f7407bd9d0839d9885241c209c60 -F src/util.c 465fe10aabf0ca7d7826a156dab919b0b65c525a +F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 +F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf +F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 +F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 +F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 05deeec6659f2579674a5e6510b3ada2a442f8d5 -F src/vdbe.h 8a675fefdf7119441fe817c800a9a52440c2e797 -F src/vdbeInt.h fe8f58d305e629fff02f61f655aca1d299f1f6ae -F src/vdbeapi.c e0e2672e0a96ae3f8575c8ecd02912a3e8a554a1 -F src/vdbeaux.c 9ae5074b19bdff2d8806a278533956fb281510d5 -F src/vdbeblob.c c3ccb7c8732858c680f442932e66ad06bb036562 +F src/vdbe.c 50b5b4ef29d928a4f60905f820ee1ab2f8ced29f +F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 +F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 +F src/vdbeapi.c 2f908c65337b16c6249ab378d477e74492cf759b +F src/vdbeaux.c 4d100407e3c72e163854aff8903d19d5ecdf46c0 +F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 -F src/vtab.c b0abc931f95af94c9ffdf9f747eb191cda953123 -F src/wal.c 7334009b396285b658a95a3b6bc6d2b016a1f794 -F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840 +F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 +F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 +F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 +F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 51756962d522e474338e9b2ebb26e7364d4aa125 -F test/alter.test 4e47fb9ea59348b88fce4e8bb49de530128b104c +F test/alter.test a3f570072b53d7c0fe463bab3f5affa8e113c487 F test/alter2.test 75f731508f1bf27ba09a6075c66cd02216ba464b F test/alter3.test 8677e48d95536f7a6ed86a1a774744dadcc22b07 F test/alter4.test 1e5dd6b951e9f65ca66422edff02e56df82dd403 @@ -311,7 +315,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test bea67403a5e37a4b33230ee4723e315a2ffb31e7 F test/capi3d.test cd36571f014f34bdc4421967f6453cbb597d5d16 F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde -F test/cast.test 166951664a0b0a2e0f8fb5997a152490c6363932 +F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test db2b29d557544347d28e25b8406f5d5ecc3d1bc3 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 @@ -365,37 +369,38 @@ F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test b40fc61bc4f1ad2a3c84590bd1d711507263d921 -F test/e_delete.test 55d868b647acc091c261a10b9b0cb0ab660a6acb +F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 +F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab F test/e_droptrigger.test ddd4b28ed8a3d81bd5153fa0ab7559529a2ca03a F test/e_dropview.test b347bab30fc8de67b131594b3cd6f3d3bdaa753d -F test/e_expr.test 9e8b9790803df4de23c2d68d566959934a6179d4 +F test/e_expr.test 71b55f90c9336ecec5a99641679036931260c754 F test/e_fkey.test 38039b840ab19331000b0f0eb1d82baa7208a67a F test/e_fts3.test 75bb0aee26384ef586165e21018a17f7cd843469 -F test/e_insert.test 7390c2da39f16a134dc9a439144768c727757d2c +F test/e_insert.test 76d4bb5da9b28014d515d91ffe29a79a1e99f2bc F test/e_reindex.test a064f0878b8f848fbca38f1f61f82f15a3000c64 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test bf385ae3aa0f014c4933ae66fd3e1302138493eb +F test/e_select.test 7ac53674e822d4d77bbb4a9a4aaefa5fdc9e493f F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 -F test/e_update.test 963d6876064e65f318d1c93aaed36a02b9b389bf +F test/e_update.test b926341a65955d69a6375c9eb4fd82e7089bc83a +F test/e_uri.test 6f35b491f80dac005c8144f38b2dfb4d96483596 F test/e_vacuum.test 6c09c2af7f2f140518f371c5342100118f779dcf F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041 -F test/enc4.test 4b575ef09e0eff896e73bd24076f96c2aa6a42de +F test/enc4.test b145fa25feb56ea1d51a5bc43ca268c0cf691a67 F test/eqp.test f14fadd76da53405e9885e2431cacf7191d83cdb F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test 53e1841b422e554cecf0160f937c473d6d0e3062 F test/exclusive2.test 343d55130c12c67b8bf10407acec043a6c26c86b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 5e2d64b4eb5a9d08876599bdae2e1213d2d12e2a -F test/expr.test 19e8ac40313e2282a47b586d11c4892040990d3a +F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d F test/fallocate.test 43dc34b8c24be6baffadc3b4401ee15710ce83c6 F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e F test/filefmt.test f178cfc29501a14565954c961b226e61877dd32c F test/fkey1.test 01c7de578e11747e720c2d9aeef27f239853c4da F test/fkey2.test 080969fe219b3b082b0e097ac18c6af2e5b0631f -F test/fkey3.test 42f88d6048d8dc079e2a8cf7baad1cc1483a7620 +F test/fkey3.test 5ec899d12b13bcf1e9ef40eff7fb692fdb91392e F test/fkey4.test c6c8f9f9be885f95c85c7bceb26f243ad906fd49 F test/fkey_malloc.test a5ede29bd2f6e56dea78c3d43fb86dd696c068c8 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb @@ -450,30 +455,34 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 -F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa -F test/fts3aux1.test 719c35cbbcc04dde8e5a54a6f69851a0af9ed1f2 +F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 +F test/fts3auto.test f1cb0a55130897013ca5850dbee2945c2908a45a +F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c -F test/fts3corrupt.test 7890cc202406858386ddf390a879dcf80bc10abf +F test/fts3conf.test 8e65ea56f88ced6cdd2252bdddb1a8327ae5af7e +F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f -F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c +F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 +F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba +F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 +F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad +F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd -F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f +F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 +F test/fts4aa.test 148d9eb54901af23b5d402b1f388f43e559e1728 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 7ba2ca5a1e9bca900ba2c230cf04bd67184bc1bc @@ -483,7 +492,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 -F test/hook.test f04c3412463f8ec117c1c704c74ca0f627ce733a +F test/hook.test f2277c309e4ee8067d95d6b9b315568e9d5329b2 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -505,7 +514,7 @@ F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test aef273dd1cee84cc92407469e6bd1b3cdcb76908 F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 -F test/insert4.test c1469999a58e86a85b74df645a820f4cc7a8273b +F test/insert4.test b3e02648a5fc3075c29e13c369b5127bf859b5a2 F test/insert5.test 1f93cbe9742110119133d7e8e3ccfe6d7c249766 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1 @@ -543,11 +552,11 @@ F test/lock4.test c82268c031d39345d05efa672f80b025481b3ae5 F test/lock5.test b2abb5e711bc59b0eae00f6c97a36ec9f458fada F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64 -F test/lock_common.tcl d279887a0ab16cdb6d935c1203e64113c5a000e9 +F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 9d7bbfcc1b52c88ba7b2ba6554068ecf9939f252 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test e56c9c3358da2c18385aea15a42dc970913986c2 +F test/malloc.test 76017be66cec4375a4b4ea5c71245e27a9fe2d0b F test/malloc3.test 4128b1e6ffa506103b278ad97af89174f310c7ca F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test 4d16d1bb26d2deddd7c4f480deec341f9b2d0e22 @@ -585,35 +594,35 @@ F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test a88f3e2c16e567e72be7296195c59fbdd6a8d3d4 +F test/multiplex.test 555080c87abfc72ba68e2f3df01d4a9a7a4fdf58 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 -F test/nan.test a44e04df1486fcfb02d32468cbcd3c8e1e433723 +F test/nan.test dc212a22b36109fd1ae37154292444ef249c5ec2 F test/notify1.test 8433bc74bd952fb8a6e3f8d7a4c2b28dfd69e310 F test/notify2.test 195a467e021f74197be2c4fb02d6dee644b8d8db F test/notify3.test d60923e186e0900f4812a845fcdfd8eea096e33a F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test af02ed0a9cbc0d2a61b8f35171d4d117e588e4ec -F test/oserror.test 498d8337e9d15543eb7b004fef8594bf204ff43c -F test/pager1.test d8672fd0af5f4f9b99b06283d00f01547809bebe +F test/oserror.test 3fe52e0bd2891a9bf7cdeb639554992453d46301 +F test/pager1.test 228a831060dab96bc91b03ba2a85cedefd1ab38a F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 9de4d3e0c59970b4c6cb8dac511fa242f335d8a7 +F test/pagerfault.test 4194b8ea2a5da7958cd155556605ff554e1b065a F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6 +F test/permutations.test 1e8892ebf1bd6e9e8036f4841c72a91bf72da74a F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 -F test/quota.test ddafe133653093eb9a99ccd6264884ae43f9c9b8 +F test/quota.test 48c3a5a98687d67ef06fc16d2e603284756bbec3 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a @@ -677,7 +686,7 @@ F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796 F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 5d7a4954b0059c903f82c7b67867bc5451a7c082 -F test/sync.test ded6b39d8d8ca3c0c5518516c6371b3316d3e3a3 +F test/sync.test 2bd73b585089c99e76b3e1796c42059b7d89d872 F test/syscall.test 707c95e4ab7863e13f1293c6b0c76bead30249b3 F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f F test/table.test 04ba066432430657712d167ebf28080fe878d305 @@ -686,7 +695,7 @@ F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl 6fa3d2f581b479a3a088b1b5b0d145e548ebe662 +F test/tester.tcl 7d3d17914ca5397a82f801db733b81d4a82f50c3 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -700,6 +709,7 @@ F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 +F test/tkt-2d1a5c67d.test 73574c758502bf23260c17f97fcd9316dfb5a060 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 5741cd48de500347a437ba1be58c8335e83c5a5e F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac @@ -715,10 +725,12 @@ F test/tkt-78e04e52ea.test ab52f0c1e2de6e46c910f4cc16b086bba05952b7 F test/tkt-80ba201079.test a09684db1a0bd55b8838f606adccee456a51ddbf F test/tkt-80e031a00f.test 9a154173461a4dbe2de49cda73963e04842d52f7 F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c +F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test be5ea61cb04dfdc047d19b5c5a9e75fa3da67a7f F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test e6b62b2b2785c04d0d698d6a603507e384165049 +F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-cbd054fa6b.test f14f97ea43662e6f70c9e63287081e8be5d9d589 F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7 F test/tkt-d82e3f3721.test 731359dfdcdb36fea0559cd33fec39dd0ceae8e6 @@ -798,13 +810,13 @@ F test/tkt3793.test 754b73f0e6a9349c70dc57e522cf3247272ecd5d F test/tkt3810.test 90fa0635dfa7da9680c8cd3513350a49b3a8ae12 F test/tkt3824.test 150aa00bb6220672e5f0eb14dc8eaa36750425f0 F test/tkt3832.test 2300d10d57562b89875b72148338ac3e14f8847d -F test/tkt3838.test f956f0719b5f805b12dd1dbf19f19d298bacebc3 +F test/tkt3838.test d8490365a1c473d214f7878007e543410cbb715f F test/tkt3841.test 4659845bc53f809a5932c61c6ce8c5bb9d6b947f F test/tkt3871.test 43ecbc8d90dc83908e2a454aef345acc9d160c6f F test/tkt3879.test 2ad5bef2c87e9991ce941e054c31abe26ef7fb90 F test/tkt3911.test 74cd324f3ba653040cc6d94cc4857b290d12d633 F test/tkt3918.test e6cdf6bfcfe9ba939d86a4238a9dc55d6eec5d42 -F test/tkt3922.test 022ace32c049e3964f68492c12eb803e8e4856d8 +F test/tkt3922.test f26be40ab4fe6c00795629bd2006d96e270d9b1a F test/tkt3929.test 75a862e45bcb39e9a7944c89b92afa531304afca F test/tkt3935.test e15261fedb9e30a4305a311da614a5d8e693c767 F test/tkt3992.test f3e7d548ac26f763b47bc0f750da3d03c81071da @@ -812,7 +824,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test 0ce11265c83333d8f5beeca19e71ed93a88d386c +F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 @@ -827,16 +839,17 @@ F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 F test/triggerA.test eaf11a29db2a11967d2d4b49d37f92bce598194e F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe -F test/triggerC.test 8a691ff6dd47df2e57395bbec4b62101fac0f363 +F test/triggerC.test 02c690febf608ae20b9af86184a9867f79855b1d F test/triggerD.test c6add3817351451e419f6ff9e9a259b02b6e2de7 F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 -F test/types3.test a0f66bf12f80fad89493535474f7a6d16fa58150 +F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 083c7fff74695bcc27a71d75699deba3595bc9c2 F test/unixexcl.test 9d80a54d86d2261f660758928959368ffc36151e -F test/unordered.test c479d3027f9c4db05b44b83010735c6708abcc91 +F test/unordered.test e81169ce2a8f31b2c6b66af691887e1376ab3ced F test/update.test 8bc86fd7ef1a00014f76dc6a6a7c974df4aef172 +F test/uri.test 53de9a2549cbda9c343223236918ef502f6a9051 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test 29b60e8cc9e573b39676df6c4a75fe9e02d04a09 F test/vacuum2.test 91a84c9b08adfc4472097d2e8deb0150214e0e76 @@ -863,11 +876,12 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test 5617ad308bfdb8a8885220d8a261a6096a8d7e57 -F test/wal2.test e561a8c6fdd1c2cd1876f3e39757934e7b7361f8 +F test/wal2.test aa0fb2314b3235be4503c06873e41ebfc0757782 F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 -F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b +F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088 F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 +F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0 @@ -878,6 +892,7 @@ F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 +F test/walro.test 2d5d69e2e99da19ce6faab340330234fc4ca0720 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 F test/walthread.test a25a393c068a2b42b44333fa3fdaae9072f1617c @@ -895,28 +910,30 @@ F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 +F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 -F tool/lemon.c dfd81a51b6e27e469ba21d01a75ddf092d429027 +F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +F tool/lemon.c d51c68d405ff7f9bad99268ca3c20a198eb983ed F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl 623e26cc8c83322e4151d3ad85ac69d41221bae8 -F tool/mksqlite3h.tcl d76c226a5e8e1f3b5f6593bcabe5e98b3b1ec9ff +F tool/mksqlite3c.tcl 1fa0ed9cfdc768bf5de7e65fda8d97a46dd2a7e6 +F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 -F tool/omittest.tcl b1dd290c1596e0f31fd335160a74ec5dfea3df4a +F tool/omittest.tcl 8086c014cbae90f1f2b564d59d05a5e4ac1783c9 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/shell1.test 7a389c6aaad05621be39501d6f8db410da464dcd +F tool/shell1.test 20dfe7099cf2afe37aecd69afb7678d14f7a0abf F tool/shell2.test 5dc76b8005b465f420fed8241621da7513060ff3 F tool/shell3.test 4fad469e8003938426355afdf34155f08c587836 F tool/shell4.test 35f9c3d452b4e76d5013c63e1fd07478a62f14ce F tool/shell5.test 62bfaf9267296da1b91e4b1c03e44e7b393f6a94 -F tool/showdb.c 471c0f8fa472e71bb7654500096a5bdb4ea1fb2a +F tool/showdb.c 43e913d954684c2f5007dcab46d1a1308852a0ad F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 F tool/showwal.c f09e5a80a293919290ec85a6a37c85a5ddcf37d9 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -928,8 +945,11 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c +F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 +F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c429edf30accac12d5deef10e59cd31146036f11 -R 050fbfdb8675b7eaafb0b1677e5953e7 -U drh -Z 289c52539a2a59a211a989409cbc5426 +F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d +P de8ad5f8176d5573c02ec39b14cca324cfef5cb9 +R 627b3b91e5169f5233cd18fa7efa5202 +U dan +Z 94b0ac9079eaca2f6ca6db06cfa18fd2 diff --git a/manifest.uuid b/manifest.uuid index e5411089..735e9cae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -154ddbc17120be2915eb03edc52af1225eb7cb5e +af0d91adf497f5f36ec3813f04235a6e195a605f diff --git a/src/alter.c b/src/alter.c index aa3fa929..fb6d89de 100644 --- a/src/alter.c +++ b/src/alter.c @@ -358,14 +358,14 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ /* Reload the table, index and permanent trigger schemas. */ zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); if( !zWhere ) return; - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); #ifndef SQLITE_OMIT_TRIGGER /* Now, if the table is not stored in the temp database, reload any temp ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. */ if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); } #endif } diff --git a/src/attach.c b/src/attach.c index bda1c874..18f8823b 100644 --- a/src/attach.c +++ b/src/attach.c @@ -70,8 +70,12 @@ static void attachFunc( sqlite3 *db = sqlite3_context_db_handle(context); const char *zName; const char *zFile; + char *zPath = 0; + char *zErr = 0; + unsigned int flags; Db *aNew; char *zErrDyn = 0; + sqlite3_vfs *pVfs; UNUSED_PARAMETER(NotUsed); @@ -124,8 +128,18 @@ static void attachFunc( ** it to obtain the database schema. At this point the schema may ** or may not be initialised. */ - rc = sqlite3BtreeOpen(zFile, db, &aNew->pBt, 0, - db->openFlags | SQLITE_OPEN_MAIN_DB); + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); + sqlite3_free( zPath ); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; diff --git a/src/btree.c b/src/btree.c index 103a1f32..3d7162db 100644 --- a/src/btree.c +++ b/src/btree.c @@ -788,6 +788,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ *pRC = SQLITE_CORRUPT_BKPT; goto ptrmap_exit; } + assert( offset <= (int)pBt->usableSize-5 ); pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ @@ -827,6 +828,11 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); + return SQLITE_CORRUPT_BKPT; + } + assert( offset <= (int)pBt->usableSize-5 ); assert( pEType!=0 ); *pEType = pPtrmap[offset]; if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); @@ -851,6 +857,8 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ */ #define findCell(P,I) \ ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) + /* ** This a more complex version of findCell() that works for @@ -1688,13 +1696,13 @@ static int btreeInvokeBusyHandler(void *pArg){ ** to problems with locking. */ int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use for this b-tree */ const char *zFilename, /* Name of the file containing the BTree database */ sqlite3 *db, /* Associated database handle */ Btree **ppBtree, /* Pointer to new Btree object written here */ int flags, /* Options */ int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ ){ - sqlite3_vfs *pVfs; /* The VFS to use for this btree */ BtShared *pBt = 0; /* Shared part of btree structure */ Btree *p; /* Handle to return */ sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ @@ -1716,6 +1724,7 @@ int sqlite3BtreeOpen( #endif assert( db!=0 ); + assert( pVfs!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ @@ -1734,7 +1743,6 @@ int sqlite3BtreeOpen( if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; } - pVfs = db->pVfs; p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ return SQLITE_NOMEM; @@ -4445,7 +4453,7 @@ int sqlite3BtreeMovetoUnpacked( } assert( pCur->apPage[0]->intKey || pIdxKey ); for(;;){ - int lwr, upr; + int lwr, upr, idx; Pgno chldPg; MemPage *pPage = pCur->apPage[pCur->iPage]; int c; @@ -4461,14 +4469,14 @@ int sqlite3BtreeMovetoUnpacked( lwr = 0; upr = pPage->nCell-1; if( biasRight ){ - pCur->aiIdx[pCur->iPage] = (u16)upr; + pCur->aiIdx[pCur->iPage] = (u16)(idx = upr); }else{ - pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2); } for(;;){ - int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */ u8 *pCell; /* Pointer to current cell in pPage */ + assert( idx==pCur->aiIdx[pCur->iPage] ); pCur->info.nSize = 0; pCell = findCell(pPage, idx) + pPage->childPtrSize; if( pPage->intKey ){ @@ -4551,7 +4559,7 @@ int sqlite3BtreeMovetoUnpacked( if( lwr>upr ){ break; } - pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); + pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); } assert( lwr==upr+1 ); assert( pPage->isInit ); @@ -5384,10 +5392,10 @@ static int fillInCell( ** "sz" must be the number of bytes in the cell. */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ - int i; /* Loop counter */ u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ + u8 *endPtr; /* End of loop */ int rc; /* The return code */ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ @@ -5412,9 +5420,11 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ *pRC = rc; return; } - for(i=idx+1; inCell; i++, ptr+=2){ - ptr[0] = ptr[2]; - ptr[1] = ptr[3]; + endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptrnCell--; put2byte(&data[hdr+3], pPage->nCell); @@ -5454,6 +5464,7 @@ static void insertCell( int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ u8 *ptr; /* Used for moving information around in data[] */ + u8 *endPtr; /* End of the loop */ int nSkip = (iChild ? 4 : 0); @@ -5504,9 +5515,12 @@ static void insertCell( if( iChild ){ put4byte(&data[idx], iChild); } - for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){ - ptr[0] = ptr[-2]; - ptr[1] = ptr[-1]; + ptr = &data[end]; + endPtr = &data[ins]; + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ + while( ptr>endPtr ){ + *(u16*)ptr = *(u16*)&ptr[-2]; + ptr -= 2; } put2byte(&data[ins], idx); put2byte(&data[pPage->hdrOffset+3], pPage->nCell); @@ -5551,10 +5565,11 @@ static void assemblePage( pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; for(i=nCell-1; i>=0; i--){ + u16 sz = aSize[i]; pCellptr -= 2; - cellbody -= aSize[i]; + cellbody -= sz; put2byte(pCellptr, cellbody); - memcpy(&data[cellbody], apCell[i], aSize[i]); + memcpy(&data[cellbody], apCell[i], sz); } put2byte(&data[hdr+3], nCell); put2byte(&data[hdr+5], cellbody); @@ -6008,12 +6023,24 @@ static int balance_nonroot( memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); limit = pOld->nCell+pOld->nOverflow; - for(j=0; jnOverflow>0 ){ + for(j=0; jaData; + u16 maskPage = pOld->maskPage; + u16 cellOffset = pOld->cellOffset; + for(j=0; jpAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse->nVar, pParse->nMem, - pParse->nTab, pParse->nMaxArg, pParse->explain, - pParse->isMultiWrite && pParse->mayAbort); + sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; pParse->colNamesSet = 0; }else{ @@ -1621,8 +1619,8 @@ void sqlite3EndTable( #endif /* Reparse everything to update our internal data structures */ - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); } @@ -2819,9 +2817,8 @@ Index *sqlite3CreateIndex( if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), - P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } @@ -3443,7 +3440,7 @@ int sqlite3OpenTempDatabase(Parse *pParse){ SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TEMP_DB; - rc = sqlite3BtreeOpen(0, db, &pBt, 0, flags); + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pBt, 0, flags); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " "file for storing temporary tables"); diff --git a/src/date.c b/src/date.c index b81049aa..c0744e8c 100644 --- a/src/date.c +++ b/src/date.c @@ -50,22 +50,6 @@ #ifndef SQLITE_OMIT_DATETIME_FUNCS -/* -** On recent Windows platforms, the localtime_s() function is available -** as part of the "Secure CRT". It is essentially equivalent to -** localtime_r() available under most POSIX platforms, except that the -** order of the parameters is reversed. -** -** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. -** -** If the user has not indicated to use localtime_r() or localtime_s() -** already, check for an MSVC build environment that provides -** localtime_s(). -*/ -#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ - defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) -#define HAVE_LOCALTIME_S 1 -#endif /* ** A structure for holding a single date and time. @@ -411,15 +395,83 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !defined(HAVE_LOCALTIME_R) && !defined(HAVE_LOCALTIME_S) && \ + defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#define HAVE_LOCALTIME_S 1 +#endif + #ifndef SQLITE_OMIT_LOCALTIME /* -** Compute the difference (in milliseconds) -** between localtime and UTC (a.k.a. GMT) -** for the time value p where p is in UTC. +** The following routine implements the rough equivalent of localtime_r() +** using whatever operating-system specific localtime facility that +** is available. This routine returns 0 on success and +** non-zero on any kind of error. +** +** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this +** routine will always fail. */ -static sqlite3_int64 localtimeOffset(DateTime *p){ +static int osLocaltime(time_t *t, struct tm *pTm){ + int rc; +#if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ + && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) + struct tm *pX; + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex_enter(mutex); + pX = localtime(t); +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#endif + if( pX ) *pTm = *pX; + sqlite3_mutex_leave(mutex); + rc = pX==0; +#else +#ifndef SQLITE_OMIT_BUILTIN_TEST + if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#endif +#if defined(HAVE_LOCALTIME_R) && HAVE_LOCALTIME_R + rc = localtime_r(t, pTm)==0; +#else + rc = localtime_s(pTm, t); +#endif /* HAVE_LOCALTIME_R */ +#endif /* HAVE_LOCALTIME_R || HAVE_LOCALTIME_S */ + return rc; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** Compute the difference (in milliseconds) between localtime and UTC +** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, +** return this value and set *pRc to SQLITE_OK. +** +** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value +** is undefined in this case. +*/ +static sqlite3_int64 localtimeOffset( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx, /* Write error here if one occurs */ + int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +){ DateTime x, y; time_t t; + struct tm sLocal; + + /* Initialize the contents of sLocal to avoid a compiler warning. */ + memset(&sLocal, 0, sizeof(sLocal)); + x = *p; computeYMD_HMS(&x); if( x.Y<1971 || x.Y>=2038 ){ @@ -437,47 +489,23 @@ static sqlite3_int64 localtimeOffset(DateTime *p){ x.validJD = 0; computeJD(&x); t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); -#ifdef HAVE_LOCALTIME_R - { - struct tm sLocal; - localtime_r(&t, &sLocal); - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; + if( osLocaltime(&t, &sLocal) ){ + sqlite3_result_error(pCtx, "local time unavailable", -1); + *pRc = SQLITE_ERROR; + return 0; } -#elif defined(HAVE_LOCALTIME_S) && HAVE_LOCALTIME_S - { - struct tm sLocal; - localtime_s(&sLocal, &t); - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; - } -#else - { - struct tm *pTm; - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - pTm = localtime(&t); - y.Y = pTm->tm_year + 1900; - y.M = pTm->tm_mon + 1; - y.D = pTm->tm_mday; - y.h = pTm->tm_hour; - y.m = pTm->tm_min; - y.s = pTm->tm_sec; - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - } -#endif + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; y.validYMD = 1; y.validHMS = 1; y.validJD = 0; y.validTZ = 0; computeJD(&y); + *pRc = SQLITE_OK; return y.iJD - x.iJD; } #endif /* SQLITE_OMIT_LOCALTIME */ @@ -501,9 +529,12 @@ static sqlite3_int64 localtimeOffset(DateTime *p){ ** localtime ** utc ** -** Return 0 on success and 1 if there is any kind of error. +** Return 0 on success and 1 if there is any kind of error. If the error +** is in a system call (i.e. localtime()), then an error message is written +** to context pCtx. If the error is an unrecognized modifier, no error is +** written to pCtx. */ -static int parseModifier(const char *zMod, DateTime *p){ +static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ int rc = 1; int n; double r; @@ -523,9 +554,8 @@ static int parseModifier(const char *zMod, DateTime *p){ */ if( strcmp(z, "localtime")==0 ){ computeJD(p); - p->iJD += localtimeOffset(p); + p->iJD += localtimeOffset(p, pCtx, &rc); clearYMD_HMS_TZ(p); - rc = 0; } break; } @@ -546,11 +576,12 @@ static int parseModifier(const char *zMod, DateTime *p){ else if( strcmp(z, "utc")==0 ){ sqlite3_int64 c1; computeJD(p); - c1 = localtimeOffset(p); - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p); - rc = 0; + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } } #endif break; @@ -731,9 +762,8 @@ static int isDate( } } for(i=1; ipSelect || (pParse->db->flags & SQLITE_IdxRealAsInt)!=0 ){ + zAff = 0; + }else{ + zAff = sqlite3IndexAffinityStr(v, pIdx); + } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); - sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), P4_TRANSIENT); + sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); } sqlite3ReleaseTempRange(pParse, regBase, nCol+1); return regBase; diff --git a/src/expr.c b/src/expr.c index c0e9ba6f..be2f4d7c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -555,53 +555,53 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); - }else if( z[0]=='?' ){ - /* Wildcard of the form "?nnn". Convert "nnn" to an integer and - ** use it as the variable number */ - i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); - pExpr->iColumn = (ynVar)i; - testcase( i==0 ); - testcase( i==1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); - testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ - sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", - db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; - } }else{ - /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable - ** number as the prior appearance of the same name, or if the name - ** has never appeared before, reuse the same variable number - */ - int i; - u32 n; - n = sqlite3Strlen30(z); - for(i=0; inVarExpr; i++){ - Expr *pE = pParse->apVarExpr[i]; - assert( pE!=0 ); - if( memcmp(pE->u.zToken, z, n)==0 && pE->u.zToken[n]==0 ){ - pExpr->iColumn = pE->iColumn; - break; + ynVar x = 0; + u32 n = sqlite3Strlen30(z); + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + pExpr->iColumn = x = (ynVar)i; + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + x = 0; } + if( i>pParse->nVar ){ + pParse->nVar = (int)i; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + ynVar i; + for(i=0; inzVar; i++){ + if( pParse->azVar[i] && memcmp(pParse->azVar[i],z,n+1)==0 ){ + pExpr->iColumn = x = (ynVar)i+1; + break; + } + } + if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); } - if( i>=pParse->nVarExpr ){ - pExpr->iColumn = (ynVar)(++pParse->nVar); - if( pParse->nVarExpr>=pParse->nVarExprAlloc-1 ){ - pParse->nVarExprAlloc += pParse->nVarExprAlloc + 10; - pParse->apVarExpr = - sqlite3DbReallocOrFree( - db, - pParse->apVarExpr, - pParse->nVarExprAlloc*sizeof(pParse->apVarExpr[0]) - ); + if( x>0 ){ + if( x>pParse->nzVar ){ + char **a; + a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); + if( a==0 ) return; /* Error reported through db->mallocFailed */ + pParse->azVar = a; + memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); + pParse->nzVar = x; } - if( !db->mallocFailed ){ - assert( pParse->apVarExpr!=0 ); - pParse->apVarExpr[pParse->nVarExpr++] = pExpr; + if( z[0]!='?' || pParse->azVar[x-1]==0 ){ + sqlite3DbFree(db, pParse->azVar[x-1]); + pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); } } } @@ -2345,7 +2345,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); if( pExpr->u.zToken[1]!=0 ){ - sqlite3VdbeChangeP4(v, -1, pExpr->u.zToken, P4_TRANSIENT); + assert( pExpr->u.zToken[0]=='?' + || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); + sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); } break; } diff --git a/src/fkey.c b/src/fkey.c index 34fdfda5..dda8a50d 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -386,13 +386,25 @@ static void fkLookupParent( /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not - ** increment the constraint-counter. */ + ** increment the constraint-counter. + ** + ** If any of the parent-key values are NULL, then the row cannot match + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any + ** of the parent-key values are NULL (at this point it is known that + ** none of the child key values are). + */ if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); } sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); } diff --git a/src/func.c b/src/func.c index 6a4f7c09..16de6bbb 100644 --- a/src/func.c +++ b/src/func.c @@ -506,10 +506,10 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A,C) (*(A++)) -# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A,C) (*(A++)) +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] #else -# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } +# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -552,9 +552,9 @@ static int patternCompare( const u8 *zPattern, /* The glob pattern */ const u8 *zString, /* The string to compare against the glob */ const struct compareInfo *pInfo, /* Information about how to do the compare */ - const int esc /* The escape character */ + u32 esc /* The escape character */ ){ - int c, c2; + u32 c, c2; int invert; int seen; u8 matchOne = pInfo->matchOne; @@ -608,7 +608,7 @@ static int patternCompare( return 0; } }else if( c==matchSet ){ - int prior_c = 0; + u32 prior_c = 0; assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; @@ -684,7 +684,7 @@ static void likeFunc( sqlite3_value **argv ){ const unsigned char *zA, *zB; - int escape = 0; + u32 escape = 0; int nPat; sqlite3 *db = sqlite3_context_db_handle(context); @@ -774,6 +774,21 @@ static void sourceidFunc( sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); } +/* +** Implementation of the sqlite_log() function. This is a wrapper around +** sqlite3_log(). The return value is NULL. The function exists purely for +** its side-effects. +*/ +static void errlogFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(context); + sqlite3_log(sqlite3_value_int(argv[0]), "%s", sqlite3_value_text(argv[1])); +} + /* ** Implementation of the sqlite_compileoption_used() function. ** The result is an integer that identifies if the compiler option @@ -1541,6 +1556,7 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(nullif, 2, 0, 1, nullifFunc ), FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), diff --git a/src/global.c b/src/global.c index 0c890684..69daa5e9 100644 --- a/src/global.c +++ b/src/global.c @@ -129,7 +129,9 @@ const unsigned char sqlite3CtypeMap[256] = { }; #endif - +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 0 +#endif /* ** The following singleton contains the global configuration for @@ -139,6 +141,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ 1, /* bCoreMutex */ SQLITE_THREADSAFE==1, /* bFullMutex */ + SQLITE_USE_URI, /* bOpenUri */ 0x7ffffffe, /* mxStrlen */ 100, /* szLookaside */ 500, /* nLookaside */ @@ -166,6 +169,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nRefInitMutex */ 0, /* xLog */ 0, /* pLogArg */ + 0, /* bLocaltimeFault */ }; diff --git a/src/insert.c b/src/insert.c index 588a84f3..452939db 100644 --- a/src/insert.c +++ b/src/insert.c @@ -969,6 +969,7 @@ void sqlite3Insert( const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); }else #endif @@ -1734,6 +1735,18 @@ static int xferOptimization( return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + /* Disallow the transfer optimization if the destination table constains + ** any foreign key constraints. This is more restrictive than necessary. + ** But the main beneficiary of the transfer optimization is the VACUUM + ** command, and the VACUUM command disables foreign key constraints. So + ** the extra complication to make this rule less restrictive is probably + ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] + */ + if( (pParse->db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + return 0; + } +#endif /* If we get this far, it means either: ** diff --git a/src/main.c b/src/main.c index 4aaa6189..b2366304 100644 --- a/src/main.c +++ b/src/main.c @@ -426,6 +426,11 @@ int sqlite3_config(int op, ...){ break; } + case SQLITE_CONFIG_URI: { + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + default: { rc = SQLITE_ERROR; break; @@ -1785,6 +1790,236 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ return oldLimit; /* IMP: R-53341-35419 */ } +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. It is the +** responsibility of the caller to eventually call sqlite3_free() to release +** this buffer. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) + && nUri>=5 && memcmp(zUri, "file:", 5)==0 + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + int nByte = nUri+2; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + zFile[iOut++] = '\0'; + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( mode>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc(nUri+2); + if( !zFile ) return SQLITE_NOMEM; + memcpy(zFile, zUri, nUri); + zFile[nUri] = '\0'; + zFile[nUri+1] = '\0'; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + + /* ** This routine does the work of opening a database on behalf of ** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" @@ -1793,12 +2028,14 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ static int openDatabase( const char *zFilename, /* Database filename UTF-8 encoded */ sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned flags, /* Operational flags */ + unsigned int flags, /* Operational flags */ const char *zVfs /* Name of the VFS to use */ ){ - sqlite3 *db; - int rc; - int isThreadsafe; + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT @@ -1822,7 +2059,7 @@ static int openDatabase( testcase( (1<<(flags&7))==0x02 ); /* READONLY */ testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE; + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; @@ -1903,13 +2140,6 @@ static int openDatabase( sqlite3HashInit(&db->aModule); #endif - db->pVfs = sqlite3_vfs_find(zVfs); - if( !db->pVfs ){ - rc = SQLITE_ERROR; - sqlite3Error(db, rc, "no such vfs: %s", zVfs); - goto opendb_out; - } - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 ** and UTF-16, so add a version for each to avoid any unnecessary ** conversions. The only error that can occur here is a malloc() failure. @@ -1932,9 +2162,18 @@ static int openDatabase( createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, nocaseCollatingFunc, 0); - /* Open the backend database driver */ + /* Parse the filename/URI argument. */ db->openFlags = flags; - rc = sqlite3BtreeOpen(zFilename, db, &db->aDb[0].pBt, 0, + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ @@ -2027,6 +2266,7 @@ static int openDatabase( sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: + sqlite3_free(zOpen); if( db ){ assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); sqlite3_mutex_leave(db->mutex); @@ -2058,7 +2298,7 @@ int sqlite3_open_v2( int flags, /* Flags */ const char *zVfs /* Name of VFS module to use */ ){ - return openDatabase(filename, ppDb, flags, zVfs); + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); } #ifndef SQLITE_OMIT_UTF16 @@ -2663,8 +2903,41 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, configure the wrappers so that all + ** subsequent calls to localtime() and variants fail. If onoff is zero, + ** undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; } + +/* +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +*/ +const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} diff --git a/src/malloc.c b/src/malloc.c index 50fdf524..3e38d1df 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -266,7 +266,7 @@ static int mallocWithAlarm(int n, void **pp){ sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmCallback!=0 ){ int nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - if( nUsed+nFull >= mem0.alarmThreshold ){ + if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); }else{ @@ -507,7 +507,7 @@ void sqlite3DbFree(sqlite3 *db, void *p){ ** Change the size of an existing memory allocation */ void *sqlite3Realloc(void *pOld, int nBytes){ - int nOld, nNew; + int nOld, nNew, nDiff; void *pNew; if( pOld==0 ){ return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ @@ -530,9 +530,10 @@ void *sqlite3Realloc(void *pOld, int nBytes){ }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); - if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nNew-nOld >= - mem0.alarmThreshold ){ - sqlite3MallocAlarm(nNew-nOld); + nDiff = nNew - nOld; + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); } assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); diff --git a/src/os_unix.c b/src/os_unix.c index 2626ab4d..682e74c9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -138,6 +138,10 @@ # include #endif +#ifdef HAVE_UTIME +# include +#endif + /* ** Allowed values of unixFile.fsFlags */ @@ -281,6 +285,18 @@ struct unixFile { #define threadid 0 #endif +/* +** Different Unix systems declare open() in different ways. Same use +** open(const char*,int,mode_t). Others use open(const char*,int,...). +** The difference is important when using a pointer to the function. +** +** The safest way to deal with the problem is to always use this wrapper +** which always has the same well-defined interface. +*/ +static int posixOpen(const char *zFile, int flags, int mode){ + return open(zFile, flags, mode); +} + /* ** Many system calls are accessed through pointer-to-functions so that ** they may be overridden at runtime to facilitate fault injection during @@ -292,8 +308,8 @@ static struct unix_syscall { sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { - { "open", (sqlite3_syscall_ptr)open, 0 }, -#define osOpen ((int(*)(const char*,int,...))aSyscall[0].pCurrent) + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, +#define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) { "close", (sqlite3_syscall_ptr)close, 0 }, #define osClose ((int(*)(int))aSyscall[1].pCurrent) @@ -330,7 +346,7 @@ static struct unix_syscall { { "read", (sqlite3_syscall_ptr)read, 0 }, #define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) -#if defined(USE_PREAD) || defined(SQLITE_ENABLE_LOCKING_STYLE) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pread", (sqlite3_syscall_ptr)pread, 0 }, #else { "pread", (sqlite3_syscall_ptr)0, 0 }, @@ -347,7 +363,7 @@ static struct unix_syscall { { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) -#if defined(USE_PREAD) || defined(SQLITE_ENABLE_LOCKING_STYLE) +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, @@ -933,7 +949,7 @@ struct unixInodeInfo { UnixUnusedFd *pUnused; /* Unused file descriptors to close */ unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ unixInodeInfo *pPrev; /* .... doubly linked */ -#if defined(SQLITE_ENABLE_LOCKING_STYLE) +#if SQLITE_ENABLE_LOCKING_STYLE unsigned long long sharedByte; /* for AFP simulated shared lock */ #endif #if OS_VXWORKS @@ -1927,8 +1943,10 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { */ if( pFile->eFileLock > NO_LOCK ){ pFile->eFileLock = eFileLock; -#if !OS_VXWORKS /* Always update the timestamp on the old file */ +#ifdef HAVE_UTIME + utime(zLockFile, NULL); +#else utimes(zLockFile, NULL); #endif return SQLITE_OK; @@ -3090,7 +3108,7 @@ static int unixWrite( SimulateDiskfullError(( wrote=0, amt=1 )); if( amt>0 ){ - if( wrote<0 ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; }else{ @@ -3525,7 +3543,8 @@ struct unixShmNode { char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ - int nRegion; /* Size of array apRegion */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ @@ -3757,6 +3776,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ (u32)sStat.st_ino, (u32)sStat.st_dev); #else sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); #endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; @@ -3771,8 +3791,17 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + } + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + goto shm_open_err; + } } /* Check to see if another process is holding the dead-man switch. @@ -3911,11 +3940,12 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, + pMem = mmap(0, szRegion, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion ); if( pMem==MAP_FAILED ){ - rc = SQLITE_IOERR; + rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ @@ -3937,6 +3967,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } @@ -4790,6 +4821,11 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** corresponding database file and sets *pMode to this value. Whenever ** possible, WAL and journal files are created using the same permissions ** as the associated database file. +** +** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the +** original filename is unavailable. But 8_3_NAMES is only used for +** FAT filesystems and permissions do not matter there, so just use +** the default permissions. */ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ @@ -4797,6 +4833,7 @@ static int findCreateFileMode( mode_t *pMode /* OUT: Permissions to open file with */ ){ int rc = SQLITE_OK; /* Return Code */ + *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ @@ -4808,15 +4845,15 @@ static int findCreateFileMode( ** ** "-journal" ** "-wal" - ** "-journal-NNNN" - ** "-wal-NNNN" + ** "-journalNN" + ** "-walNN" ** - ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are + ** where NN is a 4 digit decimal number. The NN naming schemes are ** used by the test_multiplex.c module. */ nDb = sqlite3Strlen30(zPath) - 1; - while( nDb>0 && zPath[nDb]!='l' ) nDb--; - nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7); + while( nDb>0 && zPath[nDb]!='-' ) nDb--; + if( nDb==0 ) return SQLITE_OK; memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; @@ -4827,8 +4864,6 @@ static int findCreateFileMode( } }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; - }else{ - *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; } return rc; } diff --git a/src/os_win.c b/src/os_win.c index c8768339..bd0f2f21 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -118,6 +118,7 @@ struct winFile { #endif }; + /* ** Forward prototypes. */ @@ -285,7 +286,7 @@ char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ ** Convert UTF-8 to multibyte character string. Space to hold the ** returned string is obtained from malloc(). */ -static char *utf8ToMbcs(const char *zFilename){ +char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ char *zFilenameMbcs; WCHAR *zTmpWide; @@ -298,6 +299,109 @@ static char *utf8ToMbcs(const char *zFilename){ return zFilenameMbcs; } + +/* +** The return value of getLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). +*/ +static int getLastErrorMsg(int nBuf, char *zBuf){ + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + DWORD error = GetLastError(); + DWORD dwLen = 0; + char *zOut = 0; + + if( isNT() ){ + WCHAR *zTempWide = NULL; + dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + error, + 0, + (LPWSTR) &zTempWide, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + zOut = unicodeToUtf8(zTempWide); + /* free the system buffer allocated by FormatMessage */ + LocalFree(zTempWide); + } +/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. +** Since the ASCII version of these Windows API do not exist for WINCE, +** it's important to not reference them for WINCE builds. +*/ +#if SQLITE_OS_WINCE==0 + }else{ + char *zTemp = NULL; + dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + error, + 0, + (LPSTR) &zTemp, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + /* free the system buffer allocated by FormatMessage */ + LocalFree(zTemp); + } +#endif + } + if( 0 == dwLen ){ + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); + }else{ + /* copy a maximum of nBuf chars to output buffer */ + sqlite3_snprintf(nBuf, zBuf, "%s", zOut); + /* free the UTF8 buffer */ + free(zOut); + } + return 0; +} + +/* +** +** This function - winLogErrorAtLine() - is only ever called via the macro +** winLogError(). +** +** This routine is invoked after an error occurs in an OS function. +** It logs a message using sqlite3_log() containing the current value of +** error code and, if possible, the human-readable equivalent from +** FormatMessage. +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed and the the associated file-system path, if any. +*/ +#define winLogError(a,b,c) winLogErrorAtLine(a,b,c,__LINE__) +static int winLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char zMsg[500]; /* Human readable error text */ + int i; /* Loop counter */ + DWORD iErrno = GetLastError(); /* Error code */ + + zMsg[0] = 0; + getLastErrorMsg(sizeof(zMsg), zMsg); + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} + zMsg[i] = 0; + sqlite3_log(errcode, + "os_win.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zMsg + ); + + return errcode; +} + #if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. @@ -375,6 +479,7 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = CreateMutexW(NULL, FALSE, zName); if (!pFile->hMutex){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_ERROR, "winceCreateLock1", zFilename); free(zName); return FALSE; } @@ -406,6 +511,7 @@ static BOOL winceCreateLock(const char *zFilename, winFile *pFile){ /* If mapping failed, close the shared memory handle and erase it */ if (!pFile->shared){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_ERROR, "winceCreateLock2", zFilename); CloseHandle(pFile->hShared); pFile->hShared = NULL; } @@ -651,6 +757,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_IOERR_SEEK, "seekWinFile", pFile->zPath); return 1; } @@ -696,7 +803,8 @@ static int winClose(sqlite3_file *id){ #endif OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); OpenCounter(-1); - return rc ? SQLITE_OK : SQLITE_IOERR; + return rc ? SQLITE_OK + : winLogError(SQLITE_IOERR_CLOSE, "winClose", pFile->zPath); } /* @@ -722,7 +830,7 @@ static int winRead( } if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ pFile->lastErrno = GetLastError(); - return SQLITE_IOERR_READ; + return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); } if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ @@ -770,10 +878,11 @@ static int winWrite( } if( rc ){ - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ + if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) + || ( pFile->lastErrno==ERROR_DISK_FULL )){ return SQLITE_FULL; } - return SQLITE_IOERR_WRITE; + return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); } return SQLITE_OK; } @@ -801,10 +910,10 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ - rc = SQLITE_IOERR_TRUNCATE; + rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate1", pFile->zPath); }else if( 0==SetEndOfFile(pFile->h) ){ pFile->lastErrno = GetLastError(); - rc = SQLITE_IOERR_TRUNCATE; + rc = winLogError(SQLITE_IOERR_TRUNCATE, "winTruncate2", pFile->zPath); } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -826,6 +935,7 @@ int sqlite3_fullsync_count = 0; static int winSync(sqlite3_file *id, int flags){ #if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || defined(SQLITE_DEBUG) winFile *pFile = (winFile*)id; + BOOL rc; #else UNUSED_PARAMETER(id); #endif @@ -838,20 +948,19 @@ static int winSync(sqlite3_file *id, int flags){ OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); -#ifndef SQLITE_TEST - UNUSED_PARAMETER(flags); -#else - if( flags & SQLITE_SYNC_FULL ){ - sqlite3_fullsync_count++; - } - sqlite3_sync_count++; -#endif - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); - SimulateIOError( return SQLITE_IOERR; ); + +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( (flags&0x0F)==SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; + } + sqlite3_sync_count++; +#endif /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a ** no-op @@ -859,11 +968,13 @@ static int winSync(sqlite3_file *id, int flags){ #ifdef SQLITE_NO_SYNC return SQLITE_OK; #else - if( FlushFileBuffers(pFile->h) ){ + rc = FlushFileBuffers(pFile->h); + SimulateIOError( rc=FALSE ); + if( rc ){ return SQLITE_OK; }else{ pFile->lastErrno = GetLastError(); - return SQLITE_IOERR; + return winLogError(SQLITE_IOERR_FSYNC, "winSync", pFile->zPath); } #endif } @@ -884,7 +995,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ && ((error = GetLastError()) != NO_ERROR) ) { pFile->lastErrno = error; - return SQLITE_IOERR_FSTAT; + return winLogError(SQLITE_IOERR_FSTAT, "winFileSize", pFile->zPath); } *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; return SQLITE_OK; @@ -923,6 +1034,7 @@ static int getReadLock(winFile *pFile){ } if( res == 0 ){ pFile->lastErrno = GetLastError(); + /* No need to log a failure to lock */ } return res; } @@ -941,8 +1053,9 @@ static int unlockReadLock(winFile *pFile){ res = UnlockFile(pFile->h, SHARED_FIRST + pFile->sharedLockByte, 0, 1, 0); #endif } - if( res == 0 ){ + if( res==0 && GetLastError()!=ERROR_NOT_LOCKED ){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_IOERR_UNLOCK, "unlockReadLock", pFile->zPath); } return res; } @@ -1143,7 +1256,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ if( locktype==SHARED_LOCK && !getReadLock(pFile) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ - rc = SQLITE_IOERR_UNLOCK; + rc = winLogError(SQLITE_IOERR_UNLOCK, "winUnlock", pFile->zPath); } } if( type>=RESERVED_LOCK ){ @@ -1458,6 +1571,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ memset(pNew, 0, sizeof(*pNew)); pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); /* Look to see if there is an existing winShmNode that can be used. ** If no matching winShmNode currently exists, create a new one. @@ -1500,7 +1614,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMOPEN; + rc = winLogError(SQLITE_IOERR_SHMOPEN, "winOpenShm", pDbFd->zPath); } } if( rc==SQLITE_OK ){ @@ -1759,7 +1873,7 @@ static int winShmMap( */ rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMSIZE; + rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap1", pDbFd->zPath); goto shmpage_out; } @@ -1773,7 +1887,7 @@ static int winShmMap( if( !isWrite ) goto shmpage_out; rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte); if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR_SHMSIZE; + rc = winLogError(SQLITE_IOERR_SHMSIZE, "winShmMap2", pDbFd->zPath); goto shmpage_out; } } @@ -1810,7 +1924,7 @@ static int winShmMap( } if( !pMap ){ pShmNode->lastErrno = GetLastError(); - rc = SQLITE_IOERR; + rc = winLogError(SQLITE_IOERR_SHMMAP, "winShmMap3", pDbFd->zPath); if( hMap ) CloseHandle(hMap); goto shmpage_out; } @@ -1892,7 +2006,7 @@ static void *convertUtf8Filename(const char *zFilename){ */ #if SQLITE_OS_WINCE==0 }else{ - zConverted = utf8ToMbcs(zFilename); + zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); #endif } /* caller will handle out of memory */ @@ -1972,68 +2086,6 @@ static int getTempname(int nBuf, char *zBuf){ return SQLITE_OK; } -/* -** The return value of getLastErrorMsg -** is zero if the error message fits in the buffer, or non-zero -** otherwise (if the message was truncated). -*/ -static int getLastErrorMsg(int nBuf, char *zBuf){ - /* FormatMessage returns 0 on failure. Otherwise it - ** returns the number of TCHARs written to the output - ** buffer, excluding the terminating null char. - */ - DWORD error = GetLastError(); - DWORD dwLen = 0; - char *zOut = 0; - - if( isNT() ){ - WCHAR *zTempWide = NULL; - dwLen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPWSTR) &zTempWide, - 0, - 0); - if( dwLen > 0 ){ - /* allocate a buffer and convert to UTF8 */ - zOut = unicodeToUtf8(zTempWide); - /* free the system buffer allocated by FormatMessage */ - LocalFree(zTempWide); - } -/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. -** Since the ASCII version of these Windows API do not exist for WINCE, -** it's important to not reference them for WINCE builds. -*/ -#if SQLITE_OS_WINCE==0 - }else{ - char *zTemp = NULL; - dwLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - 0, - (LPSTR) &zTemp, - 0, - 0); - if( dwLen > 0 ){ - /* allocate a buffer and convert to UTF8 */ - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); - /* free the system buffer allocated by FormatMessage */ - LocalFree(zTemp); - } -#endif - } - if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", error, error); - }else{ - /* copy a maximum of nBuf chars to output buffer */ - sqlite3_snprintf(nBuf, zBuf, "%s", zOut); - /* free the UTF8 buffer */ - free(zOut); - } - return 0; -} - /* ** Open a file. */ @@ -2205,6 +2257,7 @@ static int winOpen( if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = GetLastError(); + winLogError(SQLITE_CANTOPEN, "winOpen", zUtf8Name); free(zConverted); if( isReadWrite ){ return winOpen(pVfs, zName, id, @@ -2308,7 +2361,8 @@ static int winDelete( "ok" : "failed" )); return ( (rc == INVALID_FILE_ATTRIBUTES) - && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : SQLITE_IOERR_DELETE; + && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : + winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); } /* @@ -2348,6 +2402,7 @@ static int winAccess( } }else{ if( GetLastError()!=ERROR_FILE_NOT_FOUND ){ + winLogError(SQLITE_IOERR_ACCESS, "winAccess", zFilename); free(zConverted); return SQLITE_IOERR_ACCESS; }else{ @@ -2412,6 +2467,13 @@ static int winFullPathname( void *zConverted; char *zOut; + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && sqlite3Isalpha(zRelative[1]) && zRelative[2]==':' ){ + zRelative++; + } + /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the diff --git a/src/pager.c b/src/pager.c index 94f647dc..7ff9a9a0 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4299,6 +4299,8 @@ int sqlite3PagerOpen( int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ + int nUri = 0; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). This @@ -4329,6 +4331,7 @@ int sqlite3PagerOpen( ** leave both nPathname and zPathname set to 0. */ if( zFilename && zFilename[0] ){ + const char *z; nPathname = pVfs->mxPathname+1; zPathname = sqlite3Malloc(nPathname*2); if( zPathname==0 ){ @@ -4337,6 +4340,12 @@ int sqlite3PagerOpen( zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ + z += sqlite3Strlen30(z)+1; + z += sqlite3Strlen30(z)+1; + } + nUri = &z[1] - zUri; if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by ** the database being opened will be more than pVfs->mxPathname @@ -4369,7 +4378,7 @@ int sqlite3PagerOpen( ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ journalFileSize * 2 + /* The two journal files */ - nPathname + 1 + /* zFilename */ + nPathname + 1 + nUri + /* zFilename */ nPathname + 8 + 1 /* zJournal */ #ifndef SQLITE_OMIT_WAL + nPathname + 4 + 1 /* zWal */ @@ -4391,14 +4400,17 @@ int sqlite3PagerOpen( /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ if( zPathname ){ assert( nPathname>0 ); - pPager->zJournal = (char*)(pPtr += nPathname + 1); + pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); memcpy(pPager->zFilename, zPathname, nPathname); + memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); memcpy(pPager->zJournal, zPathname, nPathname); memcpy(&pPager->zJournal[nPathname], "-journal", 8); + sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); #ifndef SQLITE_OMIT_WAL pPager->zWal = &pPager->zJournal[nPathname+8+1]; memcpy(pPager->zWal, zPathname, nPathname); memcpy(&pPager->zWal[nPathname], "-wal", 4); + sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); #endif sqlite3_free(zPathname); } @@ -5735,11 +5747,21 @@ int sqlite3PagerCommitPhaseOne( }else{ if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); - if( pList ){ + PgHdr *pPageOne = 0; + if( pList==0 ){ + /* Must have at least one page for the WAL commit flag. + ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ + rc = sqlite3PagerGet(pPager, 1, &pPageOne); + pList = pPageOne; + pList->pDirty = 0; + } + assert( rc==SQLITE_OK ); + if( ALWAYS(pList) ){ rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, (pPager->fullSync ? pPager->syncFlags : 0) ); } + sqlite3PagerUnref(pPageOne); if( rc==SQLITE_OK ){ sqlite3PcacheCleanAll(pPager->pPCache); } @@ -6597,6 +6619,7 @@ int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ if( iLimit>=-1 ){ pPager->journalSizeLimit = iLimit; + sqlite3WalLimit(pPager->pWal, iLimit); } return pPager->journalSizeLimit; } @@ -6688,7 +6711,8 @@ static int pagerOpenWal(Pager *pPager){ */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal + pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, &pPager->pWal ); } diff --git a/src/pcache.c b/src/pcache.c index 242f3071..f37511e9 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -253,6 +253,13 @@ int sqlite3PcacheFetch( } if( pPg ){ int rc; +#ifdef SQLITE_LOG_CACHE_SPILL + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + pCache->nMax); +#endif rc = pCache->xStress(pCache->pStress, pPg); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ return rc; diff --git a/src/pcache1.c b/src/pcache1.c index ad443954..e47265a2 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -574,7 +574,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int bPurgeable){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ - pGroup = &pcache1_g.grp; + pGroup = &pcache1.grp; } pCache->pGroup = pGroup; pCache->szPage = szPage; diff --git a/src/pragma.c b/src/pragma.c index 75ab26d4..cd2aab22 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -13,10 +13,6 @@ */ #include "sqliteInt.h" -/* Ignore this whole file if pragmas are disabled -*/ -#if !defined(SQLITE_OMIT_PRAGMA) - /* ** Interpret the given string as a safety level. Return 0 for OFF, ** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or @@ -49,10 +45,16 @@ static u8 getSafetyLevel(const char *z){ /* ** Interpret the given string as a boolean value. */ -static u8 getBoolean(const char *z){ +u8 sqlite3GetBoolean(const char *z){ return getSafetyLevel(z)&1; } +/* The sqlite3GetBoolean() function is used by other modules but the +** remainder of this file is specific to PRAGMA processing. So omit +** the rest of the file if PRAGMAs are omitted from the build. +*/ +#if !defined(SQLITE_OMIT_PRAGMA) + /* ** Interpret the given string as a locking mode value. */ @@ -219,7 +221,7 @@ static int flagPragma(Parse *pParse, const char *zLeft, const char *zRight){ mask &= ~(SQLITE_ForeignKeys); } - if( getBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight) ){ db->flags |= mask; }else{ db->flags &= ~mask; @@ -433,7 +435,7 @@ void sqlite3Pragma( int b = -1; assert( pBt!=0 ); if( zRight ){ - b = getBoolean(zRight); + b = sqlite3GetBoolean(zRight); } if( pId2->n==0 && b>=0 ){ int ii; @@ -1033,7 +1035,7 @@ void sqlite3Pragma( #ifndef NDEBUG if( sqlite3StrICmp(zLeft, "parser_trace")==0 ){ if( zRight ){ - if( getBoolean(zRight) ){ + if( sqlite3GetBoolean(zRight) ){ sqlite3ParserTrace(stderr, "parser: "); }else{ sqlite3ParserTrace(0, 0); @@ -1047,7 +1049,7 @@ void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft, "case_sensitive_like")==0 ){ if( zRight ){ - sqlite3RegisterLikeFunctions(db, getBoolean(zRight)); + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight)); } }else diff --git a/src/select.c b/src/select.c index 3a4a8816..8dcfb84b 100644 --- a/src/select.c +++ b/src/select.c @@ -4239,11 +4239,13 @@ int sqlite3Select( ** and pKeyInfo to the KeyInfo structure required to navigate the ** index. ** + ** (2011-04-15) Do not do a full scan of an unordered index. + ** ** In practice the KeyInfo structure will not be used. It is only ** passed to keep OP_OpenRead happy. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( !pBest || pIdx->nColumnnColumn ){ + if( pIdx->bUnordered==0 && (!pBest || pIdx->nColumnnColumn) ){ pBest = pIdx; } } diff --git a/src/shell.c b/src/shell.c index aab70b29..a54c922e 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2302,6 +2302,11 @@ static int do_meta_command(char *zLine, struct callback_data *p){ enableTimer = booleanValue(azArg[1]); }else + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ + printf("SQLite %s %s\n", + sqlite3_libversion(), sqlite3_sourceid()); + }else + if( c=='w' && strncmp(azArg[0], "width", n)==0 && nArg>1 ){ int j; assert( nArg<=ArraySize(azArg) ); @@ -2649,6 +2654,7 @@ static void main_init(struct callback_data *data) { data->mode = MODE_List; memcpy(data->separator,"|", 2); data->showHeader = 0; + sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); @@ -2663,6 +2669,11 @@ int main(int argc, char **argv){ int i; int rc = 0; + if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ + fprintf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); + exit(1); + } Argv0 = argv[0]; main_init(&data); stdin_is_interactive = isatty(0); @@ -2830,7 +2841,7 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ - printf("%s\n", sqlite3_libversion()); + printf("%s %s\n", sqlite3_libversion(), sqlite3_sourceid()); return 0; }else if( strcmp(z,"-interactive")==0 ){ stdin_is_interactive = 1; @@ -2875,10 +2886,10 @@ int main(int argc, char **argv){ char *zHistory = 0; int nHistory; printf( - "SQLite version %s\n" + "SQLite version %s %.19s\n" "Enter \".help\" for instructions\n" "Enter SQL statements terminated with a \";\"\n", - sqlite3_libversion() + sqlite3_libversion(), sqlite3_sourceid() ); zHome = find_home_dir(); if( zHome ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ec7e502d..2e6ca2b7 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -305,7 +305,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row ** coming out of the evaluated SQL statements. ^The 4th argument to -** to sqlite3_exec() is relayed through to the 1st argument of each +** sqlite3_exec() is relayed through to the 1st argument of each ** callback invocation. ^If the callback pointer to sqlite3_exec() ** is NULL, then no callback is ever invoked and result rows are ** ignored. @@ -370,7 +370,8 @@ int sqlite3_exec( ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes] +** See also: [SQLITE_IOERR_READ | extended result codes], +** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -447,17 +448,21 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) #define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) #define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations ** ** These bit values are intended for use in the ** 3rd parameter to the [sqlite3_open_v2()] interface and -** in the 4th parameter to the xOpen method of the -** [sqlite3_vfs] object. +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ @@ -465,6 +470,7 @@ int sqlite3_exec( #define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ #define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -575,17 +581,18 @@ struct sqlite3_file { /* ** CAPI3REF: OS Interface File Virtual Methods Object ** -** Every file opened by the [sqlite3_vfs] xOpen method populates an +** Every file opened by the [sqlite3_vfs.xOpen] method populates an ** [sqlite3_file] object (or, more commonly, a subclass of the ** [sqlite3_file] object) with a pointer to an instance of this object. ** This object defines the methods used to perform various operations ** against the open file represented by the [sqlite3_file] object. ** -** If the xOpen method sets the sqlite3_file.pMethods element +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element ** to a non-NULL pointer, then the sqlite3_io_methods.xClose method -** may be invoked even if the xOpen reported that it failed. The -** only way to prevent a call to xClose following a failed xOpen -** is for the xOpen to set the sqlite3_file.pMethods element to NULL. +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. ** ** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or ** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). @@ -757,7 +764,8 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ** An instance of the sqlite3_vfs object defines the interface between ** the SQLite core and the underlying operating system. The "vfs" -** in the name of the object stands for "virtual file system". +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. ** ** The value of the iVersion field is initially 1 but may be larger in ** future versions of SQLite. Additional fields may be appended to this @@ -786,6 +794,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** +** [[sqlite3_vfs.xOpen]] ** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained ** from xFullPathname() with an optional suffix added. @@ -863,6 +872,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** +** [[sqlite3_vfs.xAccess]] ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] @@ -887,7 +897,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** method returns a Julian Day Number for the current date and time as ** a floating point value. ** ^The xCurrentTimeInt64() method returns, as an integer, the Julian -** Day Number multipled by 86400000 (the number of milliseconds in +** Day Number multiplied by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current ** date and time if that method is available (if iVersion is 2 or @@ -1109,9 +1119,9 @@ int sqlite3_os_end(void); ** implementation of an application-defined [sqlite3_os_init()]. ** ** The first argument to sqlite3_config() is an integer -** [SQLITE_CONFIG_SINGLETHREAD | configuration option] that determines +** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments -** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option] +** vary depending on the [configuration option] ** in the first argument. ** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. @@ -1221,6 +1231,7 @@ struct sqlite3_mem_methods { /* ** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} ** ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. @@ -1233,7 +1244,7 @@ struct sqlite3_mem_methods { ** is invoked. ** **
-**
SQLITE_CONFIG_SINGLETHREAD
+** [[SQLITE_CONFIG_SINGLETHREAD]]
SQLITE_CONFIG_SINGLETHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Single-thread. In other words, it disables ** all mutexing and puts SQLite into a mode where it can only be used @@ -1244,7 +1255,7 @@ struct sqlite3_mem_methods { ** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD ** configuration option.
** -**
SQLITE_CONFIG_MULTITHREAD
+** [[SQLITE_CONFIG_MULTITHREAD]]
SQLITE_CONFIG_MULTITHREAD
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Multi-thread. In other words, it disables ** mutexing on [database connection] and [prepared statement] objects. @@ -1258,7 +1269,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_MULTITHREAD configuration option.
** -**
SQLITE_CONFIG_SERIALIZED
+** [[SQLITE_CONFIG_SERIALIZED]]
SQLITE_CONFIG_SERIALIZED
**
There are no arguments to this option. ^This option sets the ** [threading mode] to Serialized. In other words, this option enables ** all mutexes including the recursive @@ -1274,7 +1285,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] will return [SQLITE_ERROR] if called with the ** SQLITE_CONFIG_SERIALIZED configuration option.
** -**
SQLITE_CONFIG_MALLOC
+** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The argument specifies ** alternative low-level memory allocation routines to be used in place of @@ -1282,7 +1293,7 @@ struct sqlite3_mem_methods { ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
** -**
SQLITE_CONFIG_GETMALLOC
+** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ @@ -1290,7 +1301,7 @@ struct sqlite3_mem_methods { ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
** -**
SQLITE_CONFIG_MEMSTATUS
+** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
**
^This option takes single argument of type int, interpreted as a ** boolean, which enables or disables the collection of memory allocation ** statistics. ^(When memory allocation statistics are disabled, the @@ -1306,7 +1317,7 @@ struct sqlite3_mem_methods { ** allocation statistics are disabled by default. **
** -**
SQLITE_CONFIG_SCRATCH
+** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
**
^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be @@ -1322,9 +1333,9 @@ struct sqlite3_mem_methods { ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
** -**
SQLITE_CONFIG_PAGECACHE
+** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
**
^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implemenation. +** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. ** There are three arguments to this option: A pointer to 8-byte aligned @@ -1343,7 +1354,7 @@ struct sqlite3_mem_methods { ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
** -**
SQLITE_CONFIG_HEAP
+** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
**
^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. @@ -1360,7 +1371,7 @@ struct sqlite3_mem_methods { ** The minimum allocation size is capped at 2^12. Reasonable values ** for the minimum allocation size are 2^5 through 2^8.
** -**
SQLITE_CONFIG_MUTEX
+** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The argument specifies ** alternative low-level mutex routines to be used in place @@ -1372,7 +1383,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_GETMUTEX
+** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] @@ -1385,7 +1396,7 @@ struct sqlite3_mem_methods { ** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will ** return [SQLITE_ERROR].
** -**
SQLITE_CONFIG_LOOKASIDE
+** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
**
^(This option takes two arguments that determine the default ** memory allocation for the lookaside memory allocator on each ** [database connection]. The first argument is the @@ -1395,18 +1406,18 @@ struct sqlite3_mem_methods { ** verb to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
** -**
SQLITE_CONFIG_PCACHE
+** [[SQLITE_CONFIG_PCACHE]]
SQLITE_CONFIG_PCACHE
**
^(This option takes a single argument which is a pointer to ** an [sqlite3_pcache_methods] object. This object specifies the interface ** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
** -**
SQLITE_CONFIG_GETPCACHE
+** [[SQLITE_CONFIG_GETPCACHE]]
SQLITE_CONFIG_GETPCACHE
**
^(This option takes a single argument which is a pointer to an ** [sqlite3_pcache_methods] object. SQLite copies of the current ** page cache implementation into that object.)^
** -**
SQLITE_CONFIG_LOG
+** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
**
^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is @@ -1424,6 +1435,18 @@ struct sqlite3_mem_methods { ** In a multi-threaded application, the application-defined logger ** function must be threadsafe.
** +** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI +**
This option takes a single argument of type int. If non-zero, then +** URI handling is globally enabled. If the parameter is zero, then URI handling +** is globally disabled. If URI handling is globally enabled, all filenames +** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined. **
*/ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1442,6 +1465,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ /* ** CAPI3REF: Database Connection Configuration Options @@ -1527,13 +1551,17 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); ** ** ^This routine returns the [rowid] of the most recent ** successful [INSERT] into the database from the [database connection] -** in the first argument. ^If no successful [INSERT]s +** in the first argument. ^As of SQLite version 3.7.7, this routines +** records the last insert rowid of both ordinary tables and [virtual tables]. +** ^If no successful [INSERT]s ** have ever occurred on that database connection, zero is returned. ** -** ^(If an [INSERT] occurs within a trigger, then the [rowid] of the inserted -** row is returned by this routine as long as the trigger is running. -** But once the trigger terminates, the value returned by this routine -** reverts to the last value inserted before the trigger fired.)^ +** ^(If an [INSERT] occurs within a trigger or within a [virtual table] +** method, then this routine will return the [rowid] of the inserted +** row as long as the trigger or virtual table method is running. +** But once the trigger or virtual table method ends, the value returned +** by this routine reverts to what it was before the trigger or virtual +** table method began.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2196,6 +2224,9 @@ int sqlite3_set_authorizer( ** to signal SQLite whether or not the action is permitted. See the ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. +** +** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] +** from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -2318,7 +2349,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection ** -** ^These routines open an SQLite database file whose name is given by the +** ^These routines open an SQLite database file as specified by the ** filename argument. ^The filename argument is interpreted as UTF-8 for ** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte ** order for sqlite3_open16(). ^(A [database connection] handle is usually @@ -2345,7 +2376,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2() can take one of ** the following three values, optionally combined with the ** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE], -** and/or [SQLITE_OPEN_PRIVATECACHE] flags:)^ +** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^ ** **
** ^(
[SQLITE_OPEN_READONLY]
@@ -2364,9 +2395,8 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); **
** ** If the 3rd parameter to sqlite3_open_v2() is not one of the -** combinations shown above or one of the combinations shown above combined -** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, +** combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection @@ -2381,6 +2411,11 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not ** participate in [shared cache mode] even if it is enabled. ** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** ** ^If the filename is ":memory:", then a private, temporary in-memory database ** is created for the connection. ^This in-memory database will vanish when ** the database connection is closed. Future versions of SQLite might @@ -2393,10 +2428,111 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** on-disk database will be created. ^This private database will be ** automatically deleted as soon as the database connection is closed. ** -** ^The fourth parameter to sqlite3_open_v2() is the name of the -** [sqlite3_vfs] object that defines the operating system interface that -** the new database connection should use. ^If the fourth parameter is -** a NULL pointer then the default [sqlite3_vfs] object is used. +** [[URI filenames in sqlite3_open()]]

URI Filenames

+** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the fourth argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** As of SQLite version 3.7.7, URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^On windows, the first component of an absolute path +** is a drive specification (e.g. "C:"). +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite interprets the following three query parameters: +** +**
    +**
  • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
  • mode: ^(The mode parameter may be set to either "ro", "rw" or +** "rwc". Attempting to set it to any other value is an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_prepare_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If sqlite3_open_v2() is +** used, it is an error to specify a value for the mode parameter that is +** less restrictive than that specified by the flags passed as the third +** parameter. +** +**
  • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behaviour requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +**
+** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

URI filename examples

+** +** +**
URI filenames Results +**
file:data.db +** Open the file "data.db" in the current directory. +**
file:/home/fred/data.db
+** file:///home/fred/data.db
+** file://localhost/home/fred/data.db
+** Open the database file "/home/fred/data.db". +**
file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
+** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
file:/home/fred/data.db?vfs=unix-nolock +** Open file "/home/fred/data.db". Use the special VFS "unix-nolock". +**
file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +**
+** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. ** ** Note to Windows users: The encoding used for the filename argument ** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever @@ -2419,6 +2555,26 @@ int sqlite3_open_v2( const char *zVfs /* Name of VFS module to use */ ); +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. +** +** If the zFilename argument to this function is not a pointer that SQLite +** passed into the xOpen VFS method, then the behavior of this routine +** is undefined and probably undesirable. +*/ +const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); + + /* ** CAPI3REF: Error Codes And Messages ** @@ -2534,43 +2690,45 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** Additional information is available at [limits | Limits in SQLite]. ** **
-** ^(
SQLITE_LIMIT_LENGTH
+** [[SQLITE_LIMIT_LENGTH]] ^(
SQLITE_LIMIT_LENGTH
**
The maximum size of any string or BLOB or table row, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_SQL_LENGTH
+** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
SQLITE_LIMIT_SQL_LENGTH
**
The maximum length of an SQL statement, in bytes.
)^ ** -** ^(
SQLITE_LIMIT_COLUMN
+** [[SQLITE_LIMIT_COLUMN]] ^(
SQLITE_LIMIT_COLUMN
**
The maximum number of columns in a table definition or in the ** result set of a [SELECT] or the maximum number of columns in an index ** or in an ORDER BY or GROUP BY clause.
)^ ** -** ^(
SQLITE_LIMIT_EXPR_DEPTH
+** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
SQLITE_LIMIT_EXPR_DEPTH
**
The maximum depth of the parse tree on any expression.
)^ ** -** ^(
SQLITE_LIMIT_COMPOUND_SELECT
+** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
SQLITE_LIMIT_COMPOUND_SELECT
**
The maximum number of terms in a compound SELECT statement.
)^ ** -** ^(
SQLITE_LIMIT_VDBE_OP
+** [[SQLITE_LIMIT_VDBE_OP]] ^(
SQLITE_LIMIT_VDBE_OP
**
The maximum number of instructions in a virtual machine program ** used to implement an SQL statement. This limit is not currently ** enforced, though that might be added in some future release of ** SQLite.
)^ ** -** ^(
SQLITE_LIMIT_FUNCTION_ARG
+** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
SQLITE_LIMIT_FUNCTION_ARG
**
The maximum number of arguments on a function.
)^ ** -** ^(
SQLITE_LIMIT_ATTACHED
+** [[SQLITE_LIMIT_ATTACHED]] ^(
SQLITE_LIMIT_ATTACHED
**
The maximum number of [ATTACH | attached databases].)^
** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] ** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH
**
The maximum length of the pattern argument to the [LIKE] or ** [GLOB] operators.
)^ ** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] ** ^(
SQLITE_LIMIT_VARIABLE_NUMBER
**
The maximum index number of any [parameter] in an SQL statement.)^ ** -** ^(
SQLITE_LIMIT_TRIGGER_DEPTH
+** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
SQLITE_LIMIT_TRIGGER_DEPTH
**
The maximum depth of recursion for triggers.
)^ **
*/ @@ -3099,7 +3257,7 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** ^[SQLITE_BUSY] means that the database engine was unable to acquire the ** database locks it needs to do its job. ^If the statement is a [COMMIT] ** or occurs outside of an explicit transaction, then you can retry the -** statement. If the statement is not a [COMMIT] and occurs within a +** statement. If the statement is not a [COMMIT] and occurs within an ** explicit transaction then you should rollback the transaction before ** continuing. ** @@ -3378,7 +3536,7 @@ sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the most recent evaluation of the statement encountered no errors or +** ^If the most recent evaluation of the statement encountered no errors ** or if the statement is never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or @@ -4605,6 +4763,11 @@ struct sqlite3_module { void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg); int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); }; /* @@ -5287,7 +5450,7 @@ struct sqlite3_mutex_methods { ** ** ^If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since -** clearly the mutex cannot be held if it does not exist. But the +** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is @@ -5410,7 +5573,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 #define SQLITE_TESTCTRL_SCRATCHMALLOC 18 -#define SQLITE_TESTCTRL_LAST 18 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 19 +#define SQLITE_TESTCTRL_LAST 19 /* ** CAPI3REF: SQLite Runtime Status @@ -5419,7 +5583,7 @@ int sqlite3_test_control(int op, ...); ** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes -** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ +** are of the form [status parameters | SQLITE_STATUS_...].)^ ** ^The current value of the parameter is returned into *pCurrent. ** ^The highest recorded value is returned in *pHighwater. ^If the ** resetFlag is true, then the highest record value is reset after @@ -5446,12 +5610,13 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); /* ** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} ** ** These integer constants designate various run-time status parameters ** that can be returned by [sqlite3_status()]. ** **
-** ^(
SQLITE_STATUS_MEMORY_USED
+** [[SQLITE_STATUS_MEMORY_USED]] ^(
SQLITE_STATUS_MEMORY_USED
**
This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application @@ -5461,23 +5626,24 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** this parameter. The amount returned is the sum of the allocation ** sizes as reported by the xSize method in [sqlite3_mem_methods].
)^ ** -** ^(
SQLITE_STATUS_MALLOC_SIZE
+** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
SQLITE_STATUS_MALLOC_SIZE
**
This parameter records the largest memory allocation request ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_MALLOC_COUNT
+** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
SQLITE_STATUS_MALLOC_COUNT
**
This parameter records the number of separate memory allocations ** currently checked out.
)^ ** -** ^(
SQLITE_STATUS_PAGECACHE_USED
+** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
SQLITE_STATUS_PAGECACHE_USED
**
This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
)^ ** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] ** ^(
SQLITE_STATUS_PAGECACHE_OVERFLOW
**
This parameter returns the number of bytes of page cache ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] @@ -5487,13 +5653,13 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.
)^ ** -** ^(
SQLITE_STATUS_PAGECACHE_SIZE
+** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
SQLITE_STATUS_PAGECACHE_SIZE
**
This parameter records the largest memory allocation request ** handed to [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_USED
+** [[SQLITE_STATUS_SCRATCH_USED]] ^(
SQLITE_STATUS_SCRATCH_USED
**
This parameter returns the number of allocations used out of the ** [scratch memory allocator] configured using ** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not @@ -5501,7 +5667,7 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** outstanding at time, this parameter also reports the number of threads ** using scratch memory at the same time.
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
+** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
SQLITE_STATUS_SCRATCH_OVERFLOW
**
This parameter returns the number of bytes of scratch memory ** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values @@ -5511,13 +5677,13 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** slots were available. **
)^ ** -** ^(
SQLITE_STATUS_SCRATCH_SIZE
+** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
SQLITE_STATUS_SCRATCH_SIZE
**
This parameter records the largest memory allocation request ** handed to [scratch memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
)^ ** -** ^(
SQLITE_STATUS_PARSER_STACK
+** [[SQLITE_STATUS_PARSER_STACK]] ^(
SQLITE_STATUS_PARSER_STACK
**
This parameter records the deepest parser stack. It is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
)^ **
@@ -5542,9 +5708,9 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that +** [SQLITE_DBSTATUS options], that ** determines the parameter to interrogate. The set of -** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely +** [SQLITE_DBSTATUS options] is likely ** to grow in future releases of SQLite. ** ** ^The current value of the requested parameter is written into *pCur @@ -5561,6 +5727,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); /* ** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} ** ** These constants are the available integer "verbs" that can be passed as ** the second argument to the [sqlite3_db_status()] interface. @@ -5572,15 +5739,16 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** if a discontinued or unsupported verb is invoked. ** **
-** ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
+** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
SQLITE_DBSTATUS_LOOKASIDE_USED
**
This parameter returns the number of lookaside memory slots currently ** checked out.
)^ ** -** ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
+** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
**
This parameter returns the number malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; ** the current value is always zero.)^ ** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of @@ -5588,6 +5756,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside @@ -5595,12 +5764,12 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** Only the high-water value is meaningful; ** the current value is always zero.)^ ** -** ^(
SQLITE_DBSTATUS_CACHE_USED
+** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
SQLITE_DBSTATUS_CACHE_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** -** ^(
SQLITE_DBSTATUS_SCHEMA_USED
+** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
SQLITE_DBSTATUS_SCHEMA_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ @@ -5609,7 +5778,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** -** ^(
SQLITE_DBSTATUS_STMT_USED
+** [[SQLITE_DBSTATUS_STMT_USED]] ^(
SQLITE_DBSTATUS_STMT_USED
**
This parameter returns the approximate number of of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ @@ -5631,7 +5800,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** CAPI3REF: Prepared Statement Status ** ** ^(Each prepared statement maintains various -** [SQLITE_STMTSTATUS_SORT | counters] that measure the number +** [SQLITE_STMTSTATUS counters] that measure the number ** of times it has performed specific operations.)^ These counters can ** be used to monitor the performance characteristics of the prepared ** statements. For example, if the number of table steps greatly exceeds @@ -5642,7 +5811,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** ^(This interface is used to retrieve and reset counter values from ** a [prepared statement]. The first argument is the prepared statement ** object to be interrogated. The second argument -** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter] +** is an integer code for a specific [SQLITE_STMTSTATUS counter] ** to be interrogated.)^ ** ^The current value of the requested counter is returned. ** ^If the resetFlg is true, then the counter is reset to zero after this @@ -5654,24 +5823,25 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} ** ** These preprocessor macros define integer codes that name counter ** values associated with the [sqlite3_stmt_status()] interface. ** The meanings of the various counters are as follows: ** **
-**
SQLITE_STMTSTATUS_FULLSCAN_STEP
+** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
SQLITE_STMTSTATUS_FULLSCAN_STEP
**
^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter ** may indicate opportunities for performance improvement through ** careful use of indices.
** -**
SQLITE_STMTSTATUS_SORT
+** [[SQLITE_STMTSTATUS_SORT]]
SQLITE_STMTSTATUS_SORT
**
^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to ** improvement performance through careful use of indices.
** -**
SQLITE_STMTSTATUS_AUTOINDEX
+** [[SQLITE_STMTSTATUS_AUTOINDEX]]
SQLITE_STMTSTATUS_AUTOINDEX
**
^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to @@ -5722,6 +5892,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** +** [[the xInit() page cache method]] ** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() @@ -5732,6 +5903,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** built-in default page cache is used instead of the application defined ** page cache.)^ ** +** [[the xShutdown() page cache method]] ** ^The xShutdown() method is called by [sqlite3_shutdown()]. ** It can be used to clean up ** any outstanding resources before process shutdown, if required. @@ -5746,6 +5918,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** +** [[the xCreate() page cache methods]] ** ^SQLite invokes the xCreate() method to construct a new cache instance. ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The @@ -5770,6 +5943,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** +** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using @@ -5777,14 +5951,16 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** +** [[the xPagecount() page cache methods]] ** The xPagecount() method must return the number of pages currently ** stored in the cache, both pinned and unpinned. ** +** [[the xFetch() page cache methods]] ** The xFetch() method locates a page in the cache and returns a pointer to ** the page, or a NULL pointer. ** A "page", in this context, means a buffer of szPage bytes aligned at an ** 8-byte boundary. The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** minimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** ** If the requested page is already in the page cache, then the page cache @@ -5808,6 +5984,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** attempt to unpin one or more cache pages by spilling the content of ** pinned pages to disk and synching the operating system disk cache. ** +** [[the xUnpin() page cache method]] ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. @@ -5820,6 +5997,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** call to xUnpin() unpins the page regardless of the number of prior calls ** to xFetch(). ** +** [[the xRekey() page cache methods]] ** The xRekey() method is used to change the key value associated with the ** page passed as the second argument. If the cache ** previously contains an entry associated with newKey, it must be @@ -5832,6 +6010,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** +** [[the xDestroy() page cache method]] ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] @@ -5894,7 +6073,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** There should be exactly one call to sqlite3_backup_finish() for each ** successful call to sqlite3_backup_init(). ** -** sqlite3_backup_init() +** [[sqlite3_backup_init()]] sqlite3_backup_init() ** ** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the ** [database connection] associated with the destination database @@ -5921,7 +6100,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** sqlite3_backup_finish() functions to perform the specified backup ** operation. ** -** sqlite3_backup_step() +** [[sqlite3_backup_step()]] sqlite3_backup_step() ** ** ^Function sqlite3_backup_step(B,N) will copy up to N pages between ** the source and destination databases specified by [sqlite3_backup] object B. @@ -5978,7 +6157,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** by the backup operation, then the backup database is automatically ** updated at the same time. ** -** sqlite3_backup_finish() +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() ** ** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the ** application wishes to abandon the backup operation, the application @@ -6001,7 +6180,8 @@ typedef struct sqlite3_backup sqlite3_backup; ** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** -** sqlite3_backup_remaining(), sqlite3_backup_pagecount() +** [[sqlite3_backup__remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed @@ -6387,6 +6567,93 @@ int sqlite3_wal_checkpoint_v2( #define SQLITE_CHECKPOINT_FULL 1 #define SQLITE_CHECKPOINT_RESTART 2 +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** At present, there is only one option that may be configured using +** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options +** may be added in the future. +*/ +int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
+**
SQLITE_VTAB_CONSTRAINT_SUPPORT +**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
+*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Conflict resolution modes +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + + /* ** Undo the hack that converts floating point types to integer for diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ea0925e4..09f64b79 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -632,6 +632,7 @@ typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; +typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WherePlan WherePlan; typedef struct WhereInfo WhereInfo; @@ -681,7 +682,7 @@ struct Db { ** A thread must be holding a mutex on the corresponding Btree in order ** to access Schema content. This implies that the thread must also be ** holding a mutex on the sqlite3 connection pointer that owns the Btree. -** For a TEMP Schema, on the connection mutex is required. +** For a TEMP Schema, only the connection mutex is required. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ @@ -802,7 +803,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ Db *aDb; /* All backends */ int flags; /* Miscellaneous flags. See below */ - int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ + unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ u8 autoCommit; /* The auto-commit flag. */ @@ -811,6 +812,7 @@ struct sqlite3 { u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ + u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ @@ -869,7 +871,7 @@ struct sqlite3 { #endif #ifndef SQLITE_OMIT_VIRTUALTABLE Hash aModule; /* populated by sqlite3_create_module() */ - Table *pVTab; /* vtab with active Connect/Create method */ + VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ int nVTrans; /* Allocated size of aVTrans */ VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ @@ -953,6 +955,7 @@ struct sqlite3 { #define SQLITE_IndexCover 0x10 /* Disable index covering table */ #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ +#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* @@ -1232,6 +1235,8 @@ struct VTable { Module *pMod; /* Pointer to module implementation */ sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ + int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ }; @@ -2226,9 +2231,8 @@ struct Parse { ** each recursion */ int nVar; /* Number of '?' variables seen in the SQL so far */ - int nVarExpr; /* Number of used slots in apVarExpr[] */ - int nVarExprAlloc; /* Number of allocated slots in apVarExpr[] */ - Expr **apVarExpr; /* Pointers to :aaa and $aaaa wildcard expressions */ + int nzVar; /* Number of available slots in azVar[] */ + char **azVar; /* Pointers to names of parameters */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ int nAlias; /* Number of aliased result set columns */ int nAliasAlloc; /* Number of allocated slots for aAlias[] */ @@ -2420,6 +2424,7 @@ struct Sqlite3Config { int bMemstat; /* True to enable memory status */ int bCoreMutex; /* True to enable core mutexing */ int bFullMutex; /* True to enable full mutexing */ + int bOpenUri; /* True to interpret filenames as URIs */ int mxStrlen; /* Maximum string length */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ @@ -2448,6 +2453,7 @@ struct Sqlite3Config { int nRefInitMutex; /* Number of users of pInitMutex */ void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ + int bLocaltimeFault; /* True to fail localtime() calls */ }; /* @@ -2669,6 +2675,8 @@ void sqlite3AddColumnType(Parse*,Token*); void sqlite3AddDefaultValue(Parse*,ExprSpan*); void sqlite3AddCollateType(Parse*, Token*); void sqlite3EndTable(Parse*,Token*,Token*,Select*); +int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); Bitvec *sqlite3BitvecCreate(u32); int sqlite3BitvecTest(Bitvec*, u32); @@ -2873,7 +2881,7 @@ int sqlite3GetInt32(const char *, int*); int sqlite3Atoi(const char*); int sqlite3Utf16ByteLen(const void *pData, int nChar); int sqlite3Utf8CharLen(const char *pData, int nByte); -int sqlite3Utf8Read(const u8*, const u8**); +u32 sqlite3Utf8Read(const u8*, const u8**); /* ** Routines to read and write variable-length integers. These used to @@ -2919,6 +2927,7 @@ char sqlite3ExprAffinity(Expr *pExpr); int sqlite3Atoi64(const char*, i64*, int, u8); void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); @@ -2934,6 +2943,12 @@ int sqlite3AddInt64(i64*,i64); int sqlite3SubInt64(i64*,i64); int sqlite3MulInt64(i64*,i64); int sqlite3AbsInt32(int); +#ifdef SQLITE_ENABLE_8_3_NAMES +void sqlite3FileSuffix3(const char*, char*); +#else +# define sqlite3FileSuffix3(X,Y) +#endif +u8 sqlite3GetBoolean(const char *z); const void *sqlite3ValueText(sqlite3_value*, u8); int sqlite3ValueBytes(sqlite3_value*, u8); @@ -3043,6 +3058,7 @@ void sqlite3AutoLoadExtensions(sqlite3*); # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) +# define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK #else void sqlite3VtabClear(sqlite3 *db, Table*); int sqlite3VtabSync(sqlite3 *db, char **); @@ -3051,6 +3067,7 @@ void sqlite3AutoLoadExtensions(sqlite3*); void sqlite3VtabLock(VTable *); void sqlite3VtabUnlock(VTable *); void sqlite3VtabUnlockList(sqlite3*); + int sqlite3VtabSavepoint(sqlite3 *, int, int); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif void sqlite3VtabMakeWritable(Parse*,Table*); diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 575651d7..ad4d27a1 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -761,7 +761,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ case SQLITE_INTEGER: { sqlite_int64 v = sqlite3_value_int64(pIn); if( v>=-2147483647 && v<=2147483647 ){ - pVal = Tcl_NewIntObj(v); + pVal = Tcl_NewIntObj((int)v); }else{ pVal = Tcl_NewWideIntObj(v); } @@ -1441,7 +1441,7 @@ static Tcl_Obj *dbEvalColumnValue(DbEvalContext *p, int iCol){ case SQLITE_INTEGER: { sqlite_int64 v = sqlite3_column_int64(pStmt, iCol); if( v>=-2147483647 && v<=2147483647 ){ - return Tcl_NewIntObj(v); + return Tcl_NewIntObj((int)v); }else{ return Tcl_NewWideIntObj(v); } @@ -2367,7 +2367,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ } if( zNull && len>0 ){ pDb->zNull = Tcl_Alloc( len + 1 ); - strncpy(pDb->zNull, zNull, len); + memcpy(pDb->zNull, zNull, len); pDb->zNull[len] = '\0'; }else{ pDb->zNull = 0; @@ -3585,6 +3585,10 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetestfuzzer_Init(Tcl_Interp*); extern int Sqlitetestwholenumber_Init(Tcl_Interp*); +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + extern int Sqlitetestfts3_Init(Tcl_Interp *interp); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS extern int Zipvfs_Init(Tcl_Interp*); Zipvfs_Init(interp); @@ -3625,6 +3629,10 @@ static void init_all(Tcl_Interp *interp){ Sqlitetestfuzzer_Init(interp); Sqlitetestwholenumber_Init(interp); +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + Sqlitetestfts3_Init(interp); +#endif + Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); #ifdef SQLITE_SSE diff --git a/src/test1.c b/src/test1.c index 8a0d09a7..7fdf54b1 100644 --- a/src/test1.c +++ b/src/test1.c @@ -14,6 +14,7 @@ ** testing of the SQLite library. */ #include "sqliteInt.h" +#include "vdbeInt.h" #include "tcl.h" #include #include @@ -162,6 +163,9 @@ const char *sqlite3TestErrorName(int rc){ case SQLITE_IOERR_CHECKRESERVEDLOCK: zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; default: zName = "SQLITE_Unknown"; break; } return zName; @@ -2326,6 +2330,32 @@ static int test_stmt_readonly( return TCL_OK; } +/* +** Usage: uses_stmt_journal STMT +** +** Return true if STMT uses a statement journal. +*/ +static int uses_stmt_journal( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; + int rc; + + if( objc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " STMT", 0); + return TCL_ERROR; + } + + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + rc = sqlite3_stmt_readonly(pStmt); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(((Vdbe *)pStmt)->usesStmtJournal)); + return TCL_OK; +} + /* ** Usage: sqlite3_reset STMT @@ -3818,6 +3848,76 @@ static int test_open( return TCL_OK; } +/* +** Usage: sqlite3_open_v2 FILENAME FLAGS VFS +*/ +static int test_open_v2( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zFilename; + const char *zVfs; + int flags = 0; + sqlite3 *db; + int rc; + char zBuf[100]; + + int nFlag; + Tcl_Obj **apFlag; + int i; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME FLAGS VFS"); + return TCL_ERROR; + } + zFilename = Tcl_GetString(objv[1]); + zVfs = Tcl_GetString(objv[3]); + if( zVfs[0]==0x00 ) zVfs = 0; + + rc = Tcl_ListObjGetElements(interp, objv[2], &nFlag, &apFlag); + if( rc!=TCL_OK ) return rc; + for(i=0; i #include #include +#include /* ** Size of the write buffer used by journal files in bytes. diff --git a/src/test_journal.c b/src/test_journal.c index ca4c5c38..68869723 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -14,13 +14,7 @@ ** an existing VFS. The code in this file attempts to verify that SQLite ** correctly populates and syncs a journal file before writing to a ** corresponding database file. -*/ -#if SQLITE_TEST /* This file is used for testing only */ - -#include "sqlite3.h" -#include "sqliteInt.h" - -/* +** ** INTERFACE ** ** The public interface to this wrapper VFS is two functions: @@ -99,6 +93,10 @@ ** ** c) The journal file is deleted using xDelete. */ +#if SQLITE_TEST /* This file is used for testing only */ + +#include "sqlite3.h" +#include "sqliteInt.h" /* ** Maximum pathname length supported by the jt backend. diff --git a/src/test_malloc.c b/src/test_malloc.c index c63ded70..5023dca4 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1174,6 +1174,35 @@ static int test_config_error( return TCL_OK; } +/* +** tclcmd: sqlite3_config_uri BOOLEAN +** +** Invoke sqlite3_config() or sqlite3_db_config() with invalid +** opcodes and verify that they return errors. +*/ +static int test_config_uri( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + int bOpenUri; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BOOL"); + return TCL_ERROR; + } + if( Tcl_GetBooleanFromObj(interp, objv[1], &bOpenUri) ){ + return TCL_ERROR; + } + + rc = sqlite3_config(SQLITE_CONFIG_URI, bOpenUri); + Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + + return TCL_OK; +} + /* ** Usage: ** @@ -1422,6 +1451,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_memstatus", test_config_memstatus ,0 }, { "sqlite3_config_lookaside", test_config_lookaside ,0 }, { "sqlite3_config_error", test_config_error ,0 }, + { "sqlite3_config_uri", test_config_uri ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/test_multiplex.c b/src/test_multiplex.c index d8a7db86..f709c9a9 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -11,13 +11,36 @@ ************************************************************************* ** ** This file contains a VFS "shim" - a layer that sits in between the -** pager and the real VFS. +** pager and the real VFS - that breaks up a very large database file +** into two or more smaller files on disk. This is useful, for example, +** in order to support large, multi-gigabyte databases on older filesystems +** that limit the maximum file size to 2 GiB. ** -** This particular shim enforces a multiplex system on DB files. -** This shim shards/partitions a single DB file into smaller -** "chunks" such that the total DB file size may exceed the maximum -** file size of the underlying file system. +** USAGE: ** +** Compile this source file and link it with your application. Then +** at start-time, invoke the following procedure: +** +** int sqlite3_multiplex_initialize( +** const char *zOrigVfsName, // The underlying real VFS +** int makeDefault // True to make multiplex the default VFS +** ); +** +** The procedure call above will create and register a new VFS shim named +** "multiplex". The multiplex VFS will use the VFS named by zOrigVfsName to +** do the actual disk I/O. (The zOrigVfsName parameter may be NULL, in +** which case the default VFS at the moment sqlite3_multiplex_initialize() +** is called will be used as the underlying real VFS.) +** +** If the makeDefault parameter is TRUE then multiplex becomes the new +** default VFS. Otherwise, you can use the multiplex VFS by specifying +** "multiplex" as the 4th parameter to sqlite3_open_v2() or by employing +** URI filenames and adding "vfs=multiplex" as a parameter to the filename +** URI. +** +** The multiplex VFS allows databases up to 32 GiB in size. But it splits +** the files up into 1 GiB pieces, so that they will work even on filesystems +** that do not support large files. */ #include "sqlite3.h" #include @@ -185,13 +208,77 @@ static void multiplexLeave(void){ sqlite3_mutex_leave(gMultiplex.pMutex); } ** than the actual length of the string. For very long strings (greater ** than 1GiB) the value returned might be less than the true string length. */ -int multiplexStrlen30(const char *z){ +static int multiplexStrlen30(const char *z){ const char *z2 = z; if( z==0 ) return 0; while( *z2 ){ z2++; } return 0x3fffffff & (int)(z2 - z); } +/* +** Create a temporary file name in zBuf. zBuf must be big enough to +** hold at pOrigVfs->mxPathname characters. This function departs +** from the traditional temporary name generation in the os_win +** and os_unix VFS in several ways, but is necessary so that +** the file name is known for temporary files (like those used +** during vacuum.) +** +** N.B. This routine assumes your underlying VFS is ok with using +** "/" as a directory seperator. This is the default for UNIXs +** and is allowed (even mixed) for most versions of Windows. +*/ +static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ + static char zChars[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789"; + int i,j; + int attempts = 0; + int exists = 0; + int rc = SQLITE_ERROR; + + /* Check that the output buffer is large enough for + ** pVfs->mxPathname characters. + */ + if( pOrigVfs->mxPathname <= nBuf ){ + char *zTmp = sqlite3_malloc(pOrigVfs->mxPathname); + if( zTmp==0 ) return SQLITE_NOMEM; + + /* sqlite3_temp_directory should always be less than + ** pVfs->mxPathname characters. + */ + sqlite3_snprintf(pOrigVfs->mxPathname, + zTmp, + "%s/", + sqlite3_temp_directory ? sqlite3_temp_directory : "."); + rc = pOrigVfs->xFullPathname(pOrigVfs, zTmp, nBuf, zBuf); + sqlite3_free(zTmp); + if( rc ) return rc; + + /* Check that the output buffer is large enough for the temporary file + ** name. + */ + j = multiplexStrlen30(zBuf); + if( (j + 8 + 1 + 3 + 1) <= nBuf ){ + /* Make 3 attempts to generate a unique name. */ + do { + attempts++; + sqlite3_randomness(8, &zBuf[j]); + for(i=0; i<8; i++){ + zBuf[j+i] = (char)zChars[ ((unsigned char)zBuf[j+i])%(sizeof(zChars)-1) ]; + } + memcpy(&zBuf[j+i], ".tmp", 5); + rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists); + } while ( (rc==SQLITE_OK) && exists && (attempts<3) ); + if( rc==SQLITE_OK && exists ){ + rc = SQLITE_ERROR; + } + } + } + + return rc; +} + /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ @@ -295,12 +382,12 @@ static int multiplexOpen( int flags, /* Flags to control the opening */ int *pOutFlags /* Flags showing results of opening */ ){ - int rc; /* Result code */ + int rc = SQLITE_OK; /* Result code */ multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ sqlite3_file *pSubOpen; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int nName = multiplexStrlen30(zName); + int nName; int i; int sz; @@ -311,23 +398,39 @@ static int multiplexOpen( */ multiplexEnter(); pMultiplexOpen = (multiplexConn*)pConn; - /* allocate space for group */ - sz = sizeof(multiplexGroup) /* multiplexGroup */ - + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ - + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ - + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ - + nName + 1; /* zName */ + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use. This will be handled by the + ** original xOpen method. We just need to allocate space for + ** it. + */ + if( !zName ){ + rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, gMultiplex.zName); + zName = gMultiplex.zName; + } + + if( rc==SQLITE_OK ){ + /* allocate space for group */ + nName = multiplexStrlen30(zName); + sz = sizeof(multiplexGroup) /* multiplexGroup */ + + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ + + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ + + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ + + nName + 1; /* zName */ #ifndef SQLITE_MULTIPLEX_EXT_OVWR - sz += SQLITE_MULTIPLEX_EXT_SZ; - assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); + sz += SQLITE_MULTIPLEX_EXT_SZ; + assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); #else - assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); - assert(nName < pOrigVfs->mxPathname); + assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); + assert(nName < pOrigVfs->mxPathname); #endif - pGroup = sqlite3_malloc( sz ); - if( pGroup==0 ){ - rc=SQLITE_NOMEM; - }else{ + pGroup = sqlite3_malloc( sz ); + if( pGroup==0 ){ + rc=SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ /* assign pointers to extra space allocated */ char *p = (char *)&pGroup[1]; pMultiplexOpen->pGroup = pGroup; @@ -411,7 +514,7 @@ static int multiplexDelete( } rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists){ + if( rc2==SQLITE_OK && exists ){ /* if it exists, delete it */ rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir); if( rc2!=SQLITE_OK ) rc = rc2; diff --git a/src/test_quota.c b/src/test_quota.c index 3c6db4d7..9b0e4a9d 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -323,7 +323,7 @@ static int quotaOpen( pFile=pFile->pNext){} if( pFile==0 ){ int nName = strlen(zName); - pFile = sqlite3_malloc( sizeof(*pFile) + nName + 1 ); + pFile = (quotaFile *)sqlite3_malloc( sizeof(*pFile) + nName + 1 ); if( pFile==0 ){ quotaLeave(); pSubOpen->pMethods->xClose(pSubOpen); @@ -683,7 +683,7 @@ int sqlite3_quota_set( quotaLeave(); return SQLITE_OK; } - pGroup = sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 ); + pGroup = (quotaGroup *)sqlite3_malloc( sizeof(*pGroup) + nPattern + 1 ); if( pGroup==0 ){ quotaLeave(); return SQLITE_NOMEM; diff --git a/src/test_thread.c b/src/test_thread.c index ef191bc2..3a13dd66 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -404,9 +404,9 @@ static int clock_seconds_proc( */ typedef struct UnlockNotification UnlockNotification; struct UnlockNotification { - int fired; /* True after unlock event has occured */ - pthread_cond_t cond; /* Condition variable to wait on */ - pthread_mutex_t mutex; /* Mutex to protect structure */ + int fired; /* True after unlock event has occurred */ + pthread_cond_t cond; /* Condition variable to wait on */ + pthread_mutex_t mutex; /* Mutex to protect structure */ }; /* diff --git a/src/test_vfs.c b/src/test_vfs.c index 64b3cb57..a8b53526 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -10,10 +10,6 @@ ** ****************************************************************************** ** -*/ -#if SQLITE_TEST /* This file is used for testing only */ - -/* ** This file contains the implementation of the Tcl [testvfs] command, ** used to create SQLite VFS implementations with various properties and ** instrumentation to support testing SQLite. @@ -28,6 +24,7 @@ ** -mxpathname INTEGER (Value for sqlite3_vfs.mxPathname) ** -iversion INTEGER (Value for sqlite3_vfs.iVersion) */ +#if SQLITE_TEST /* This file is used for testing only */ #include "sqlite3.h" #include "sqliteInt.h" @@ -82,8 +79,6 @@ struct Testvfs { sqlite3_vfs *pVfs; /* The testvfs registered with SQLite */ Tcl_Interp *interp; /* Interpreter to run script in */ Tcl_Obj *pScript; /* Script to execute */ - int nScript; /* Number of elements in array apScript */ - Tcl_Obj **apScript; /* Array version of pScript */ TestvfsBuffer *pBuffer; /* List of shared buffers */ int isNoshm; @@ -114,20 +109,21 @@ struct Testvfs { ** + Simulating IO errors, and ** + Invoking the Tcl callback script. */ -#define TESTVFS_SHMOPEN_MASK 0x00000001 -#define TESTVFS_SHMLOCK_MASK 0x00000010 -#define TESTVFS_SHMMAP_MASK 0x00000020 -#define TESTVFS_SHMBARRIER_MASK 0x00000040 -#define TESTVFS_SHMCLOSE_MASK 0x00000080 +#define TESTVFS_SHMOPEN_MASK 0x00000001 +#define TESTVFS_SHMLOCK_MASK 0x00000010 +#define TESTVFS_SHMMAP_MASK 0x00000020 +#define TESTVFS_SHMBARRIER_MASK 0x00000040 +#define TESTVFS_SHMCLOSE_MASK 0x00000080 -#define TESTVFS_OPEN_MASK 0x00000100 -#define TESTVFS_SYNC_MASK 0x00000200 -#define TESTVFS_DELETE_MASK 0x00000400 -#define TESTVFS_CLOSE_MASK 0x00000800 -#define TESTVFS_WRITE_MASK 0x00001000 -#define TESTVFS_TRUNCATE_MASK 0x00002000 -#define TESTVFS_ACCESS_MASK 0x00004000 -#define TESTVFS_ALL_MASK 0x00007FFF +#define TESTVFS_OPEN_MASK 0x00000100 +#define TESTVFS_SYNC_MASK 0x00000200 +#define TESTVFS_DELETE_MASK 0x00000400 +#define TESTVFS_CLOSE_MASK 0x00000800 +#define TESTVFS_WRITE_MASK 0x00001000 +#define TESTVFS_TRUNCATE_MASK 0x00002000 +#define TESTVFS_ACCESS_MASK 0x00004000 +#define TESTVFS_FULLPATHNAME_MASK 0x00008000 +#define TESTVFS_ALL_MASK 0x0001FFFF #define TESTVFS_MAX_PAGES 1024 @@ -269,48 +265,26 @@ static void tvfsExecTcl( Tcl_Obj *arg3 ){ int rc; /* Return code from Tcl_EvalObj() */ - int nArg; /* Elements in eval'd list */ - int nScript; - Tcl_Obj ** ap; - + Tcl_Obj *pEval; assert( p->pScript ); - if( !p->apScript ){ - int nByte; - int i; - if( TCL_OK!=Tcl_ListObjGetElements(p->interp, p->pScript, &nScript, &ap) ){ - Tcl_BackgroundError(p->interp); - Tcl_ResetResult(p->interp); - return; - } - p->nScript = nScript; - nByte = (nScript+TESTVFS_MAX_ARGS)*sizeof(Tcl_Obj *); - p->apScript = (Tcl_Obj **)ckalloc(nByte); - memset(p->apScript, 0, nByte); - for(i=0; iapScript[i] = ap[i]; - } - } + assert( zMethod ); + assert( p ); + assert( arg2==0 || arg1!=0 ); + assert( arg3==0 || arg2!=0 ); - p->apScript[p->nScript] = Tcl_NewStringObj(zMethod, -1); - p->apScript[p->nScript+1] = arg1; - p->apScript[p->nScript+2] = arg2; - p->apScript[p->nScript+3] = arg3; + pEval = Tcl_DuplicateObj(p->pScript); + Tcl_IncrRefCount(p->pScript); + Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zMethod, -1)); + if( arg1 ) Tcl_ListObjAppendElement(p->interp, pEval, arg1); + if( arg2 ) Tcl_ListObjAppendElement(p->interp, pEval, arg2); + if( arg3 ) Tcl_ListObjAppendElement(p->interp, pEval, arg3); - for(nArg=p->nScript; p->apScript[nArg]; nArg++){ - Tcl_IncrRefCount(p->apScript[nArg]); - } - - rc = Tcl_EvalObjv(p->interp, nArg, p->apScript, TCL_EVAL_GLOBAL); + rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); if( rc!=TCL_OK ){ Tcl_BackgroundError(p->interp); Tcl_ResetResult(p->interp); } - - for(nArg=p->nScript; p->apScript[nArg]; nArg++){ - Tcl_DecrRefCount(p->apScript[nArg]); - p->apScript[nArg] = 0; - } } @@ -545,7 +519,7 @@ static int tvfsOpen( /* Evaluate the Tcl script: ** - ** SCRIPT xOpen FILENAME + ** SCRIPT xOpen FILENAME KEY-VALUE-ARGS ** ** If the script returns an SQLite error code other than SQLITE_OK, an ** error is returned to the caller. If it returns SQLITE_OK, the new @@ -554,7 +528,19 @@ static int tvfsOpen( */ Tcl_ResetResult(p->interp); if( p->pScript && p->mask&TESTVFS_OPEN_MASK ){ - tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0); + Tcl_Obj *pArg = Tcl_NewObj(); + Tcl_IncrRefCount(pArg); + if( flags&SQLITE_OPEN_MAIN_DB ){ + const char *z = &zName[strlen(zName)+1]; + while( *z ){ + Tcl_ListObjAppendElement(0, pArg, Tcl_NewStringObj(z, -1)); + z += strlen(z) + 1; + Tcl_ListObjAppendElement(0, pArg, Tcl_NewStringObj(z, -1)); + z += strlen(z) + 1; + } + } + tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0); + Tcl_DecrRefCount(pArg); if( tvfsResultCode(p, &rc) ){ if( rc!=SQLITE_OK ) return rc; }else{ @@ -663,6 +649,14 @@ static int tvfsFullPathname( int nOut, char *zOut ){ + Testvfs *p = (Testvfs *)pVfs->pAppData; + if( p->pScript && p->mask&TESTVFS_FULLPATHNAME_MASK ){ + int rc; + tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0); + if( tvfsResultCode(p, &rc) ){ + if( rc!=SQLITE_OK ) return rc; + } + } return sqlite3OsFullPathname(PARENTVFS(pVfs), zPath, nOut, zOut); } @@ -1028,18 +1022,19 @@ static int testvfs_obj_cmd( char *zName; int mask; } vfsmethod [] = { - { "xShmOpen", TESTVFS_SHMOPEN_MASK }, - { "xShmLock", TESTVFS_SHMLOCK_MASK }, - { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, - { "xShmUnmap", TESTVFS_SHMCLOSE_MASK }, - { "xShmMap", TESTVFS_SHMMAP_MASK }, - { "xSync", TESTVFS_SYNC_MASK }, - { "xDelete", TESTVFS_DELETE_MASK }, - { "xWrite", TESTVFS_WRITE_MASK }, - { "xTruncate", TESTVFS_TRUNCATE_MASK }, - { "xOpen", TESTVFS_OPEN_MASK }, - { "xClose", TESTVFS_CLOSE_MASK }, - { "xAccess", TESTVFS_ACCESS_MASK }, + { "xShmOpen", TESTVFS_SHMOPEN_MASK }, + { "xShmLock", TESTVFS_SHMLOCK_MASK }, + { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, + { "xShmUnmap", TESTVFS_SHMCLOSE_MASK }, + { "xShmMap", TESTVFS_SHMMAP_MASK }, + { "xSync", TESTVFS_SYNC_MASK }, + { "xDelete", TESTVFS_DELETE_MASK }, + { "xWrite", TESTVFS_WRITE_MASK }, + { "xTruncate", TESTVFS_TRUNCATE_MASK }, + { "xOpen", TESTVFS_OPEN_MASK }, + { "xClose", TESTVFS_CLOSE_MASK }, + { "xAccess", TESTVFS_ACCESS_MASK }, + { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; @@ -1076,9 +1071,6 @@ static int testvfs_obj_cmd( int nByte; if( p->pScript ){ Tcl_DecrRefCount(p->pScript); - ckfree((char *)p->apScript); - p->apScript = 0; - p->nScript = 0; p->pScript = 0; } Tcl_GetStringFromObj(objv[2], &nByte); @@ -1232,7 +1224,6 @@ static void testvfs_obj_del(ClientData cd){ Testvfs *p = (Testvfs *)cd; if( p->pScript ) Tcl_DecrRefCount(p->pScript); sqlite3_vfs_unregister(p->pVfs); - ckfree((char *)p->apScript); ckfree((char *)p->pVfs); ckfree((char *)p); } diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index 073eab63..5e94f5cf 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -12,6 +12,100 @@ ** ** This file contains code implements a VFS shim that writes diagnostic ** output for each VFS call, similar to "strace". +** +** USAGE: +** +** This source file exports a single symbol which is the name of a +** function: +** +** int vfstrace_register( +** const char *zTraceName, // Name of the newly constructed VFS +** const char *zOldVfsName, // Name of the underlying VFS +** int (*xOut)(const char*,void*), // Output routine. ex: fputs +** void *pOutArg, // 2nd argument to xOut. ex: stderr +** int makeDefault // Make the new VFS the default +** ); +** +** Applications that want to trace their VFS usage must provide a callback +** function with this prototype: +** +** int traceOutput(const char *zMessage, void *pAppData); +** +** This function will "output" the trace messages, where "output" can +** mean different things to different applications. The traceOutput function +** for the command-line shell (see shell.c) is "fputs" from the standard +** library, which means that all trace output is written on the stream +** specified by the second argument. In the case of the command-line shell +** the second argument is stderr. Other applications might choose to output +** trace information to a file, over a socket, or write it into a buffer. +** +** The vfstrace_register() function creates a new "shim" VFS named by +** the zTraceName parameter. A "shim" VFS is an SQLite backend that does +** not really perform the duties of a true backend, but simply filters or +** interprets VFS calls before passing them off to another VFS which does +** the actual work. In this case the other VFS - the one that does the +** real work - is identified by the second parameter, zOldVfsName. If +** the the 2nd parameter is NULL then the default VFS is used. The common +** case is for the 2nd parameter to be NULL. +** +** The third and fourth parameters are the pointer to the output function +** and the second argument to the output function. For the SQLite +** command-line shell, when the -vfstrace option is used, these parameters +** are fputs and stderr, respectively. +** +** The fifth argument is true (non-zero) to cause the newly created VFS +** to become the default VFS. The common case is for the fifth parameter +** to be true. +** +** The call to vfstrace_register() simply creates the shim VFS that does +** tracing. The application must also arrange to use the new VFS for +** all database connections that are created and for which tracing is +** desired. This can be done by specifying the trace VFS using URI filename +** notation, or by specifying the trace VFS as the 4th parameter to +** sqlite3_open_v2() or by making the trace VFS be the default (by setting +** the 5th parameter of vfstrace_register() to 1). +** +** +** ENABLING VFSTRACE IN A COMMAND-LINE SHELL +** +** The SQLite command line shell implemented by the shell.c source file +** can be used with this module. To compile in -vfstrace support, first +** gather this file (test_vfstrace.c), the shell source file (shell.c), +** and the SQLite amalgamation source files (sqlite3.c, sqlite3.h) into +** the working directory. Then compile using a command like the following: +** +** gcc -o sqlite3 -Os -I. -DSQLITE_ENABLE_VFSTRACE \ +** -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ +** -DHAVE_READLINE -DHAVE_USLEEP=1 \ +** shell.c test_vfstrace.c sqlite3.c -ldl -lreadline -lncurses +** +** The gcc command above works on Linux and provides (in addition to the +** -vfstrace option) support for FTS3 and FTS4, RTREE, and command-line +** editing using the readline library. The command-line shell does not +** use threads so we added -DSQLITE_THREADSAFE=0 just to make the code +** run a little faster. For compiling on a Mac, you'll probably need +** to omit the -DHAVE_READLINE, the -lreadline, and the -lncurses options. +** The compilation could be simplified to just this: +** +** gcc -DSQLITE_ENABLE_VFSTRACE \ +** shell.c test_vfstrace.c sqlite3.c -ldl -lpthread +** +** In this second example, all unnecessary options have been removed +** Note that since the code is now threadsafe, we had to add the -lpthread +** option to pull in the pthreads library. +** +** To cross-compile for windows using MinGW, a command like this might +** work: +** +** /opt/mingw/bin/i386-mingw32msvc-gcc -o sqlite3.exe -Os -I \ +** -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_VFSTRACE \ +** shell.c test_vfstrace.c sqlite3.c +** +** Similar compiler commands will work on different systems. The key +** invariants are (1) you must have -DSQLITE_ENABLE_VFSTRACE so that +** the shell.c source file will know to include the -vfstrace command-line +** option and (2) you must compile and link the three source files +** shell,c, test_vfstrace.c, and sqlite3.c. */ #include #include diff --git a/src/tokenize.c b/src/tokenize.c index c624efdc..b3298927 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -353,13 +353,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='x' ); testcase( z[0]=='X' ); if( z[1]=='\'' ){ *tokenType = TK_BLOB; - for(i=2; (c=z[i])!=0 && c!='\''; i++){ - if( !sqlite3Isxdigit(c) ){ - *tokenType = TK_ILLEGAL; - } + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } } - if( i%2 || !c ) *tokenType = TK_ILLEGAL; - if( c ) i++; + if( z[i] ) i++; return i; } /* Otherwise fall through to the next case */ @@ -412,9 +411,8 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); - assert( pParse->nVarExpr==0 ); - assert( pParse->nVarExprAlloc==0 ); - assert( pParse->apVarExpr==0 ); + assert( pParse->nzVar==0 ); + assert( pParse->azVar==0 ); enableLookaside = db->lookaside.bEnabled; if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; while( !db->mallocFailed && zSql[i]!=0 ){ @@ -508,7 +506,8 @@ abort_parse: } sqlite3DeleteTrigger(db, pParse->pNewTrigger); - sqlite3DbFree(db, pParse->apVarExpr); + for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); + sqlite3DbFree(db, pParse->azVar); sqlite3DbFree(db, pParse->aAlias); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; diff --git a/src/trigger.c b/src/trigger.c index 0f3f5bad..6242de5e 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -301,9 +301,8 @@ void sqlite3FinishTrigger( pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf( - db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC - ); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); } if( db->init.busy ){ diff --git a/src/update.c b/src/update.c index 315034d8..d445f01f 100644 --- a/src/update.c +++ b/src/update.c @@ -23,7 +23,8 @@ static void updateVirtualTable( ExprList *pChanges, /* The columns to change in the UPDATE statement */ Expr *pRowidExpr, /* Expression used to recompute the rowid */ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ ); #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -243,7 +244,7 @@ void sqlite3Update( } for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; - if( chngRowid ){ + if( hasFK || chngRowid ){ reg = ++pParse->nMem; }else{ reg = 0; @@ -267,7 +268,7 @@ void sqlite3Update( /* Virtual tables must be handled separately */ if( IsVirtual(pTab) ){ updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere); + pWhere, onError); pWhere = 0; pTabList = 0; goto update_cleanup; @@ -597,7 +598,8 @@ static void updateVirtualTable( ExprList *pChanges, /* The columns to change in the UPDATE statement */ Expr *pRowid, /* Expression used to recompute the rowid */ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ - Expr *pWhere /* WHERE clause of the UPDATE statement */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ ExprList *pEList = 0; /* The result set of the SELECT statement */ @@ -654,6 +656,7 @@ static void updateVirtualTable( } sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, pTab->nCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); sqlite3VdbeJumpHere(v, addr); diff --git a/src/utf.c b/src/utf.c index 95182694..17f3a09a 100644 --- a/src/utf.c +++ b/src/utf.c @@ -163,7 +163,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { || (c&0xFFFFF800)==0xD800 \ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ } -int sqlite3Utf8Read( +u32 sqlite3Utf8Read( const unsigned char *zIn, /* First byte of UTF-8 character */ const unsigned char **pzNext /* Write first byte past UTF-8 char here */ ){ diff --git a/src/util.c b/src/util.c index 1c9b401f..de735772 100644 --- a/src/util.c +++ b/src/util.c @@ -983,13 +983,12 @@ void sqlite3Put4byte(unsigned char *p, u32 v){ -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* ** Translate a single byte of Hex into an integer. ** This routine only works if h really is a valid hexadecimal ** character: 0..9a..fA..F */ -static u8 hexToInt(int h){ +u8 sqlite3HexToInt(int h){ assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); #ifdef SQLITE_ASCII h += 9*(1&(h>>6)); @@ -999,7 +998,6 @@ static u8 hexToInt(int h){ #endif return (u8)(h & 0xf); } -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ #if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* @@ -1016,7 +1014,7 @@ void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ n--; if( zBlob ){ for(i=0; i test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +*/ +void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ + const char *zOk; + zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); + if( zOk && sqlite3GetBoolean(zOk) ){ + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memcpy(&z[i+1], &z[sz-3], 4); + } +} +#endif diff --git a/src/vdbe.c b/src/vdbe.c index 5376b08a..d965b967 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -564,6 +564,7 @@ int sqlite3VdbeExec( Mem *pOut = 0; /* Output operand */ int iCompare = 0; /* Result of last OP_Compare operation */ int *aPermute = 0; /* Permutation of columns for OP_Compare */ + i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ int origPc; /* Program counter at start of opcode */ @@ -796,7 +797,7 @@ case OP_Yield: { /* in1 */ /* Opcode: HaltIfNull P1 P2 P3 P4 * ** -** Check the value in register P3. If is is NULL then Halt using +** Check the value in register P3. If it is NULL then Halt using ** parameter P1, P2, and P4 as if this were a Halt instruction. If the ** value in register P3 is not NULL, then this routine is a no-op. */ @@ -833,6 +834,7 @@ case OP_Halt: { p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); pc = sqlite3VdbeFrameRestore(pFrame); + lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ /* Instruction pc is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt @@ -986,6 +988,7 @@ case OP_Variable: { /* out2-prerelease */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; @@ -1393,16 +1396,9 @@ case OP_Function: { assert( pOp[-1].opcode==OP_CollSeq ); ctx.pColl = pOp[-1].p4.pColl; } + db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&ctx.s); - goto no_mem; - } + lastRowid = db->lastRowid; /* If any auxiliary data functions have been called by this user function, ** immediately call the destructor for any non-static values. @@ -1413,6 +1409,16 @@ case OP_Function: { pOp->p4type = P4_VDBEFUNC; } + if( db->mallocFailed ){ + /* Even though a malloc() has failed, the implementation of the + ** user function may have called an sqlite3_result_XXX() function + ** to return a value. The following call releases any resources + ** associated with such a value. + */ + sqlite3VdbeMemRelease(&ctx.s); + goto no_mem; + } + /* If the function returned an error, throw an exception */ if( ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); @@ -1714,7 +1720,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either ** true or false and is never NULL. If both operands are NULL then the result ** of comparison is false. If either operand is NULL then the result is true. -** If neither operand is NULL the the result is the same as it would be if +** If neither operand is NULL the result is the same as it would be if ** the SQLITE_NULLEQ flag were omitted from P5. */ /* Opcode: Eq P1 P2 P3 P4 P5 @@ -1726,7 +1732,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ ** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either ** true or false and is never NULL. If both operands are NULL then the result ** of comparison is true. If either operand is NULL then the result is false. -** If neither operand is NULL the the result is the same as it would be if +** If neither operand is NULL the result is the same as it would be if ** the SQLITE_NULLEQ flag were omitted from P5. */ /* Opcode: Le P1 P2 P3 P4 P5 @@ -1762,7 +1768,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pIn3 = &aMem[pOp->p3]; flags1 = pIn1->flags; flags3 = pIn3->flags; - if( (pIn1->flags | pIn3->flags)&MEM_Null ){ + if( (flags1 | flags3)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ /* If SQLITE_NULLEQ is set (which will only happen if the operator is @@ -1770,7 +1776,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** or not both operands are null. */ assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); - res = (pIn1->flags & pIn3->flags & MEM_Null)==0; + res = (flags1 & flags3 & MEM_Null)==0; }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, ** then the result is always NULL. @@ -2005,13 +2011,13 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ /* Opcode: If P1 P2 P3 * * ** -** Jump to P2 if the value in register P1 is true. The value is +** Jump to P2 if the value in register P1 is true. The value ** is considered true if it is numeric and non-zero. If the value ** in P1 is NULL then take the jump if P3 is true. */ /* Opcode: IfNot P1 P2 P3 * * ** -** Jump to P2 if the value in register P1 is False. The value is +** Jump to P2 if the value in register P1 is False. The value ** is considered true if it has a numeric value of zero. If the value ** in P1 is NULL then take the jump if P3 is true. */ @@ -2580,6 +2586,17 @@ case OP_Savepoint: { }else{ nName = sqlite3Strlen30(zName); +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* This call is Ok even if this savepoint is actually a transaction + ** savepoint (and therefore should not prompt xSavepoint()) callbacks. + ** If this is a transaction savepoint being opened, it is guaranteed + ** that the db->aVTrans[] array is empty. */ + assert( db->autoCommit==0 || db->nVTrans==0 ); + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, + db->nStatement+db->nSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif + /* Create a new savepoint structure. */ pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); if( pNew ){ @@ -2686,6 +2703,11 @@ case OP_Savepoint: { }else{ db->nDeferredCons = pSavepoint->nDeferredCons; } + + if( !isTransaction ){ + rc = sqlite3VtabSavepoint(db, p1, iSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } } } @@ -2821,7 +2843,11 @@ case OP_Transaction: { db->nStatement++; p->iStatement = db->nSavepoint + db->nStatement; } - rc = sqlite3BtreeBeginStmt(pBt, p->iStatement); + + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement-1); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginStmt(pBt, p->iStatement); + } /* Store the current value of the database handles deferred constraint ** counter. If the statement transaction needs to be rolled back, @@ -3132,7 +3158,7 @@ case OP_OpenEphemeral: { pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; - rc = sqlite3BtreeOpen(0, db, &pCx->pBt, + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(pCx->pBt, 1); @@ -3609,7 +3635,7 @@ case OP_IsUnique: { /* jump, in3 */ /* Opcode: NotExists P1 P2 P3 * * ** -** Use the content of register P3 as a integer key. If a record +** Use the content of register P3 as an integer key. If a record ** with that key does not exist in table of P1, then jump to P2. ** If the record does exist, then fall through. The cursor is left ** pointing to the record if it exists. @@ -3685,7 +3711,7 @@ case OP_Sequence: { /* out2-prerelease */ ** If P3>0 then P3 is a register in the root frame of this VDBE that holds ** the largest previously generated record number. No new record numbers are ** allowed to be less than this value. When this value reaches its maximum, -** a SQLITE_FULL error is generated. The P3 register is updated with the ' +** an SQLITE_FULL error is generated. The P3 register is updated with the ' ** generated record number. This P3 mechanism is used to help implement the ** AUTOINCREMENT feature. */ @@ -3792,7 +3818,7 @@ case OP_NewRowid: { /* out2-prerelease */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ /* on the first attempt, simply do one more than previous */ - v = db->lastRowid; + v = lastRowid; v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ v++; /* ensure non-zero */ cnt = 0; @@ -3902,7 +3928,7 @@ case OP_InsertInt: { } if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = iKey; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey; if( pData->flags & MEM_Null ){ pData->z = 0; pData->n = 0; @@ -4309,7 +4335,7 @@ case OP_Next: { /* jump */ /* Opcode: IdxInsert P1 P2 P3 * P5 ** -** Register P2 holds a SQL index key made using the +** Register P2 holds an SQL index key made using the ** MakeRecord instructions. This opcode writes that key ** into the index P1. Data for the entry is nil. ** @@ -4990,7 +5016,7 @@ case OP_Program: { /* jump */ p->nFrame++; pFrame->pParent = p->pFrame; - pFrame->lastRowid = db->lastRowid; + pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; p->nChange = 0; p->pFrame = pFrame; @@ -5773,11 +5799,15 @@ case OP_VUpdate: { Mem **apArg; Mem *pX; + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace + ); pVtab = pOp->p4.pVtab->pVtab; pModule = (sqlite3_module *)pVtab->pModule; nArg = pOp->p2; assert( pOp->p4type==P4_VTAB ); if( ALWAYS(pModule->xUpdate) ){ + u8 vtabOnConflict = db->vtabOnConflict; apArg = p->apArg; pX = &aMem[pOp->p3]; for(i=0; ivtabOnConflict = pOp->p5; rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); + db->vtabOnConflict = vtabOnConflict; importVtabErrMsg(p, pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); - db->lastRowid = rowid; + db->lastRowid = lastRowid = rowid; + } + if( rc==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( pOp->p5==OE_Ignore ){ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ + p->nChange++; } - p->nChange++; } break; } @@ -5844,20 +5884,20 @@ case OP_MaxPgcnt: { /* out2-prerelease */ */ case OP_Trace: { char *zTrace; + char *z; - zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); - if( zTrace ){ - if( db->xTrace ){ - char *z = sqlite3VdbeExpandSql(p, zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); - } -#ifdef SQLITE_DEBUG - if( (db->flags & SQLITE_SqlTrace)!=0 ){ - sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); - } -#endif /* SQLITE_DEBUG */ + if( db->xTrace && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ + z = sqlite3VdbeExpandSql(p, zTrace); + db->xTrace(db->pTraceArg, z); + sqlite3DbFree(db, z); } +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); + } +#endif /* SQLITE_DEBUG */ break; } #endif @@ -5941,6 +5981,7 @@ vdbe_error_halt: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: + db->lastRowid = lastRowid; sqlite3VdbeLeave(p); return rc; diff --git a/src/vdbe.h b/src/vdbe.h index 43044533..e66ee302 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -172,6 +172,7 @@ int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); +void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); @@ -185,7 +186,7 @@ int sqlite3VdbeMakeLabel(Vdbe*); void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeDelete(Vdbe*); void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); -void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); +void sqlite3VdbeMakeReady(Vdbe*,Parse*); int sqlite3VdbeFinalize(Vdbe*); void sqlite3VdbeResolveLabel(Vdbe*, int); int sqlite3VdbeCurrentAddr(Vdbe*); @@ -194,6 +195,7 @@ int sqlite3VdbeCurrentAddr(Vdbe*); void sqlite3VdbeTrace(Vdbe*,FILE*); #endif void sqlite3VdbeResetStepResult(Vdbe*); +void sqlite3VdbeRewind(Vdbe*); int sqlite3VdbeReset(Vdbe*); void sqlite3VdbeSetNumCols(Vdbe*,int); int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 5e96c6f9..0aeb3af7 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -287,11 +287,11 @@ struct Vdbe { Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ ynVar nVar; /* Number of entries in aVar[] */ + ynVar nzVar; /* Number of entries in azVar[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ u8 explain; /* True if EXPLAIN present on SQL command */ u8 changeCntOn; /* True to update the change-counter */ u8 expired; /* True if the VM needs to be recompiled */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 90baaccc..a87b5c9a 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -102,7 +102,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; sqlite3_mutex_enter(v->db->mutex); rc = sqlite3VdbeReset(v); - sqlite3VdbeMakeReady(v, -1, 0, 0, 0, 0, 0); + sqlite3VdbeRewind(v); assert( (rc & (v->db->errMask))==rc ); rc = sqlite3ApiExit(v->db, rc); sqlite3_mutex_leave(v->db->mutex); @@ -459,6 +459,14 @@ end_of_step: return (rc&db->errMask); } +/* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 5 +#endif + /* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, @@ -477,7 +485,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ db = v->db; sqlite3_mutex_enter(db->mutex); while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < 5 + && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); v->expired = 0; @@ -1167,32 +1175,6 @@ int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ return p ? p->nVar : 0; } -/* -** Create a mapping from variable numbers to variable names -** in the Vdbe.azVar[] array, if such a mapping does not already -** exist. -*/ -static void createVarMap(Vdbe *p){ - if( !p->okVar ){ - int j; - Op *pOp; - sqlite3_mutex_enter(p->db->mutex); - /* The race condition here is harmless. If two threads call this - ** routine on the same Vdbe at the same time, they both might end - ** up initializing the Vdbe.azVar[] array. That is a little extra - ** work but it results in the same answer. - */ - for(j=0, pOp=p->aOp; jnOp; j++, pOp++){ - if( pOp->opcode==OP_Variable ){ - assert( pOp->p1>0 && pOp->p1<=p->nVar ); - p->azVar[pOp->p1-1] = pOp->p4.z; - } - } - p->okVar = 1; - sqlite3_mutex_leave(p->db->mutex); - } -} - /* ** Return the name of a wildcard parameter. Return NULL if the index ** is out of range or if the wildcard is unnamed. @@ -1201,10 +1183,9 @@ static void createVarMap(Vdbe *p){ */ const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; - if( p==0 || i<1 || i>p->nVar ){ + if( p==0 || i<1 || i>p->nzVar ){ return 0; } - createVarMap(p); return p->azVar[i-1]; } @@ -1218,9 +1199,8 @@ int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ if( p==0 ){ return 0; } - createVarMap(p); if( zName ){ - for(i=0; inVar; i++){ + for(i=0; inzVar; i++){ const char *z = p->azVar[i]; if( z && memcmp(z,zName,nName)==0 && z[nName]==0 ){ return i+1; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 4d4bb224..989a8003 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -156,13 +156,6 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ pOp->p3 = p3; pOp->p4.p = 0; pOp->p4type = P4_NOTUSED; - p->expired = 0; - if( op==OP_ParseSchema ){ - /* Any program that uses the OP_ParseSchema opcode needs to lock - ** all btrees. */ - int j; - for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); - } #ifdef SQLITE_DEBUG pOp->zComment = 0; if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); @@ -201,6 +194,20 @@ int sqlite3VdbeAddOp4( return addr; } +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also local all btrees. +** +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. +*/ +void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ + int j; + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); +} + /* ** Add an opcode that includes the p4 value as an integer. */ @@ -1391,34 +1398,13 @@ static void *allocSpace( } /* -** Prepare a virtual machine for execution. This involves things such -** as allocating stack space and initializing the program counter. -** After the VDBE has be prepped, it can be executed by one or more -** calls to sqlite3VdbeExec(). -** -** This is the only way to move a VDBE from VDBE_MAGIC_INIT to -** VDBE_MAGIC_RUN. -** -** This function may be called more than once on a single virtual machine. -** The first call is made while compiling the SQL statement. Subsequent -** calls are made as part of the process of resetting a statement to be -** re-executed (from a call to sqlite3_reset()). The nVar, nMem, nCursor -** and isExplain parameters are only passed correct values the first time -** the function is called. On subsequent calls, from sqlite3_reset(), nVar -** is passed -1 and nMem, nCursor and isExplain are all passed zero. +** Rewind the VDBE back to the beginning in preparation for +** running it. */ -void sqlite3VdbeMakeReady( - Vdbe *p, /* The VDBE */ - int nVar, /* Number of '?' see in the SQL statement */ - int nMem, /* Number of memory cells to allocate */ - int nCursor, /* Number of cursors to allocate */ - int nArg, /* Maximum number of args in SubPrograms */ - int isExplain, /* True if the EXPLAIN keywords is present */ - int usesStmtJournal /* True to set Vdbe.usesStmtJournal */ -){ - int n; - sqlite3 *db = p->db; - +void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif assert( p!=0 ); assert( p->magic==VDBE_MAGIC_INIT ); @@ -1429,6 +1415,71 @@ void sqlite3VdbeMakeReady( /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ p->magic = VDBE_MAGIC_RUN; +#ifdef SQLITE_DEBUG + for(i=1; inMem; i++){ + assert( p->aMem[i].db==p->db ); + } +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->magic = VDBE_MAGIC_RUN; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif +} + +/* +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating stack space and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exact once on a each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, futher calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. +*/ +void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ +){ + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int n; /* Loop counter */ + u8 *zCsr; /* Memory available for allocation */ + u8 *zEnd; /* First byte past allocated memory */ + int nByte; /* How much extra memory is needed */ + + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->magic==VDBE_MAGIC_INIT ); + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + /* For each cursor required, also allocate a memory cell. Memory ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by ** the vdbe program. Instead they are used to allocate space for @@ -1441,91 +1492,69 @@ void sqlite3VdbeMakeReady( nMem += nCursor; /* Allocate space for memory registers, SQL variables, VDBE cursors and - ** an array to marshal SQL function arguments in. This is only done the - ** first time this function is called for a given VDBE, not when it is - ** being called from sqlite3_reset() to reset the virtual machine. + ** an array to marshal SQL function arguments in. */ - if( nVar>=0 && ALWAYS(db->mallocFailed==0) ){ - u8 *zCsr = (u8 *)&p->aOp[p->nOp]; /* Memory avaliable for alloation */ - u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; /* First byte past available mem */ - int nByte; /* How much extra memory needed */ + zCsr = (u8*)&p->aOp[p->nOp]; /* Memory avaliable for allocation */ + zEnd = (u8*)&p->aOp[p->nOpAlloc]; /* First byte past end of zCsr[] */ - resolveP2Values(p, &nArg); - p->usesStmtJournal = (u8)usesStmtJournal; - if( isExplain && nMem<10 ){ - nMem = 10; + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain && nMem<10 ){ + nMem = 10; + } + memset(zCsr, 0, zEnd-zCsr); + zCsr += (zCsr - (u8*)0)&7; + assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; + + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ + do { + nByte = 0; + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); + if( nByte ){ + p->pFree = sqlite3DbMallocZero(db, nByte); } - memset(zCsr, 0, zEnd-zCsr); - zCsr += (zCsr - (u8*)0)&7; - assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + zCsr = p->pFree; + zEnd = &zCsr[nByte]; + }while( nByte && !db->mallocFailed ); - /* Memory for registers, parameters, cursor, etc, is allocated in two - ** passes. On the first pass, we try to reuse unused space at the - ** end of the opcode array. If we are unable to satisfy all memory - ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the rest using a fresh allocation. - ** - ** This two-pass approach that reuses as much memory as possible from - ** the leftover space at the end of the opcode array can significantly - ** reduce the amount of memory held by a prepared statement. - */ - do { - nByte = 0; - p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), - &zCsr, zEnd, &nByte); - if( nByte ){ - p->pFree = sqlite3DbMallocZero(db, nByte); - } - zCsr = p->pFree; - zEnd = &zCsr[nByte]; - }while( nByte && !db->mallocFailed ); - - p->nCursor = (u16)nCursor; - if( p->aVar ){ - p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ - p->aMem[n].flags = MEM_Null; - p->aMem[n].db = db; - } + p->nCursor = (u16)nCursor; + if( p->aVar ){ + p->nVar = (ynVar)nVar; + for(n=0; naVar[n].flags = MEM_Null; + p->aVar[n].db = db; } } -#ifdef SQLITE_DEBUG - for(n=1; nnMem; n++){ - assert( p->aMem[n].db==db ); + if( p->azVar ){ + p->nzVar = pParse->nzVar; + memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); + memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); } -#endif - - p->pc = -1; - p->rc = SQLITE_OK; - p->errorAction = OE_Abort; - p->explain |= isExplain; - p->magic = VDBE_MAGIC_RUN; - p->nChange = 0; - p->cacheCtr = 1; - p->minWriteFileFormat = 255; - p->iStatement = 0; - p->nFkConstraint = 0; -#ifdef VDBE_PROFILE - { - int i; - for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + if( p->aMem ){ + p->aMem--; /* aMem[] goes from 1..nMem */ + p->nMem = nMem; /* not from 0..nMem-1 */ + for(n=1; n<=nMem; n++){ + p->aMem[n].flags = MEM_Null; + p->aMem[n].db = db; } } -#endif + p->explain = pParse->explain; + sqlite3VdbeRewind(p); } /* @@ -1799,6 +1828,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( !zMaster ){ return SQLITE_NOMEM; } + sqlite3FileSuffix3(zMainFile, zMaster); rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); }while( rc==SQLITE_OK && res ); if( rc==SQLITE_OK ){ @@ -2013,6 +2043,15 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ db->nStatement--; p->iStatement = 0; + if( rc==SQLITE_OK ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + /* If the statement transaction is being rolled back, also restore the ** database handles deferred constraint counter to the value it had when ** the statement transaction was opened. */ @@ -2192,17 +2231,11 @@ int sqlite3VdbeHalt(Vdbe *p){ ** do so. If this operation returns an error, and the current statement ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the ** current statement error code. - ** - ** Note that sqlite3VdbeCloseStatement() can only fail if eStatementOp - ** is SAVEPOINT_ROLLBACK. But if p->rc==SQLITE_OK then eStatementOp - ** must be SAVEPOINT_RELEASE. Hence the NEVER(p->rc==SQLITE_OK) in - ** the following code. */ if( eStatementOp ){ rc = sqlite3VdbeCloseStatement(p, eStatementOp); if( rc ){ - assert( eStatementOp==SAVEPOINT_ROLLBACK ); - if( NEVER(p->rc==SQLITE_OK) || p->rc==SQLITE_CONSTRAINT ){ + if( p->rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT ){ p->rc = rc; sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; @@ -2395,6 +2428,7 @@ void sqlite3VdbeDeleteAuxData(VdbeFunc *pVdbeFunc, int mask){ */ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; + int i; assert( p->db==0 || p->db==db ); releaseMemArray(p->aVar, p->nVar); releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); @@ -2403,6 +2437,7 @@ void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } + for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aLabel); sqlite3DbFree(db, p->aColName); @@ -2848,7 +2883,7 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( idx += getVarint32(&aKey[idx], serial_type); pMem->enc = pKeyInfo->enc; pMem->db = pKeyInfo->db; - pMem->flags = 0; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->zMalloc = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; @@ -2863,6 +2898,7 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( ** This routine destroys a UnpackedRecord object. */ void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ +#ifdef SQLITE_DEBUG int i; Mem *pMem; @@ -2876,6 +2912,7 @@ void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ */ if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); } +#endif if( p->flags & UNPACKED_NEED_FREE ){ sqlite3DbFree(p->pKeyInfo->db, p); } @@ -2929,7 +2966,7 @@ int sqlite3VdbeRecordCompare( /* Compilers may complain that mem1.u.i is potentially uninitialized. ** We could initialize it, as shown here, to silence those complaints. - ** But in fact, mem1.u.i will never actually be used initialized, and doing + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing ** the unnecessary initialization has a measurable negative performance ** impact, since this routine is a very high runner. And so, we choose ** to ignore the compiler warnings and leave this variable uninitialized. diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 18fdd465..a8728e6d 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -297,7 +297,10 @@ int sqlite3_blob_open( sqlite3VdbeChangeP4(v, 3+flags, SQLITE_INT_TO_PTR(pTab->nCol+1),P4_INT32); sqlite3VdbeChangeP2(v, 7, pTab->nCol); if( !db->mallocFailed ){ - sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); + pParse->nVar = 1; + pParse->nMem = 1; + pParse->nTab = 1; + sqlite3VdbeMakeReady(v, pParse); } } diff --git a/src/vtab.c b/src/vtab.c index b052de23..223ef4e7 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -14,6 +14,18 @@ #ifndef SQLITE_OMIT_VIRTUALTABLE #include "sqliteInt.h" +/* +** Before a virtual table xCreate() or xConnect() method is invoked, the +** sqlite3.pVtabCtx member variable is set to point to an instance of +** this struct allocated on the stack. It is used by the implementation of +** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which +** are invoked only from within xCreate and xConnect methods. +*/ +struct VtabCtx { + Table *pTab; + VTable *pVTable; +}; + /* ** The actual function that does the work of creating a new module. ** This function implements the sqlite3_create_module() and @@ -42,13 +54,13 @@ static int createModule( pMod->xDestroy = xDestroy; pDel = (Module *)sqlite3HashInsert(&db->aModule, zCopy, nName, (void*)pMod); if( pDel && pDel->xDestroy ){ + sqlite3ResetInternalSchema(db, -1); pDel->xDestroy(pDel->pAux); } sqlite3DbFree(db, pDel); if( pDel==pMod ){ db->mallocFailed = 1; } - sqlite3ResetInternalSchema(db, -1); }else if( xDestroy ){ xDestroy(pAux); } @@ -371,7 +383,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); } @@ -434,6 +446,7 @@ static int vtabCallConstructor( int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), char **pzErr ){ + VtabCtx sCtx; VTable *pVTable; int rc; const char *const*azArg = (const char *const*)pTab->azModuleArg; @@ -453,12 +466,14 @@ static int vtabCallConstructor( pVTable->db = db; pVTable->pMod = pMod; - assert( !db->pVTab ); - assert( xConstruct ); - db->pVTab = pTab; - /* Invoke the virtual table constructor */ + assert( &db->pVtabCtx ); + assert( xConstruct ); + sCtx.pTab = pTab; + sCtx.pVTable = pVTable; + db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + db->pVtabCtx = 0; if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ @@ -474,7 +489,7 @@ static int vtabCallConstructor( ** the sqlite3_vtab object if successful. */ pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; - if( db->pVTab ){ + if( sCtx.pTab ){ const char *zFormat = "vtable constructor did not declare schema: %s"; *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); sqlite3VtabUnlock(pVTable); @@ -522,7 +537,6 @@ static int vtabCallConstructor( } sqlite3DbFree(db, zModuleName); - db->pVTab = 0; return rc; } @@ -563,11 +577,11 @@ int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ return rc; } - /* -** Add the virtual table pVTab to the array sqlite3.aVTrans[]. +** Grow the db->aVTrans[] array so that there is room for at least one +** more v-table. Return SQLITE_NOMEM if a malloc fails, or SQLITE_OK otherwise. */ -static int addToVTrans(sqlite3 *db, VTable *pVTab){ +static int growVTrans(sqlite3 *db){ const int ARRAY_INCR = 5; /* Grow the sqlite3.aVTrans array if required */ @@ -582,10 +596,17 @@ static int addToVTrans(sqlite3 *db, VTable *pVTab){ db->aVTrans = aVTrans; } + return SQLITE_OK; +} + +/* +** Add the virtual table pVTab to the array sqlite3.aVTrans[]. Space should +** have already been reserved using growVTrans(). +*/ +static void addToVTrans(sqlite3 *db, VTable *pVTab){ /* Add pVtab to the end of sqlite3.aVTrans */ db->aVTrans[db->nVTrans++] = pVTab; sqlite3VtabLock(pVTab); - return SQLITE_OK; } /* @@ -623,7 +644,10 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ /* Justification of ALWAYS(): The xConstructor method is required to ** create a valid sqlite3_vtab if it returns SQLITE_OK. */ if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ - rc = addToVTrans(db, sqlite3GetVTable(db, pTab)); + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + addToVTrans(db, sqlite3GetVTable(db, pTab)); + } } return rc; @@ -642,8 +666,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ char *zErr = 0; sqlite3_mutex_enter(db->mutex); - pTab = db->pVTab; - if( !pTab ){ + if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ sqlite3Error(db, SQLITE_MISUSE, 0); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; @@ -670,7 +693,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ pParse->pNewTable->nCol = 0; pParse->pNewTable->aCol = 0; } - db->pVTab = 0; + db->pVtabCtx->pTab = 0; }else{ sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); @@ -740,6 +763,7 @@ static void callFinaliser(sqlite3 *db, int offset){ x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset); if( x ) x(p); } + pVTab->iSavepoint = 0; sqlite3VtabUnlock(pVTab); } sqlite3DbFree(db, db->aVTrans); @@ -822,7 +846,6 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( pModule->xBegin ){ int i; - /* If pVtab is already in the aVTrans array, return early */ for(i=0; inVTrans; i++){ if( db->aVTrans[i]==pVTab ){ @@ -830,10 +853,62 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ } } - /* Invoke the xBegin method */ - rc = pModule->xBegin(pVTab->pVtab); + /* Invoke the xBegin method. If successful, add the vtab to the + ** sqlite3.aVTrans[] array. */ + rc = growVTrans(db); if( rc==SQLITE_OK ){ - rc = addToVTrans(db, pVTab); + rc = pModule->xBegin(pVTab->pVtab); + if( rc==SQLITE_OK ){ + addToVTrans(db, pVTab); + } + } + } + return rc; +} + +/* +** Invoke either the xSavepoint, xRollbackTo or xRelease method of all +** virtual tables that currently have an open transaction. Pass iSavepoint +** as the second argument to the virtual table method invoked. +** +** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with +** an open transaction is invoked. +** +** If any virtual table method returns an error code other than SQLITE_OK, +** processing is abandoned and the error returned to the caller of this +** function immediately. If all calls to virtual table methods are successful, +** SQLITE_OK is returned. +*/ +int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ + int rc = SQLITE_OK; + + assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); + assert( iSavepoint>=0 ); + if( db->aVTrans ){ + int i; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + const sqlite3_module *pMod = pVTab->pMod->pModule; + if( pMod->iVersion>=2 ){ + int (*xMethod)(sqlite3_vtab *, int); + switch( op ){ + case SAVEPOINT_BEGIN: + xMethod = pMod->xSavepoint; + pVTab->iSavepoint = iSavepoint+1; + break; + case SAVEPOINT_ROLLBACK: + xMethod = pMod->xRollbackTo; + break; + default: + xMethod = pMod->xRelease; + break; + } + if( xMethod && pVTab->iSavepoint>iSavepoint ){ + rc = xMethod(db->aVTrans[i]->pVtab, iSavepoint); + } + } } } return rc; @@ -937,4 +1012,55 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ } } +/* +** Return the ON CONFLICT resolution mode in effect for the virtual +** table update operation currently in progress. +** +** The results of this routine are undefined unless it is called from +** within an xUpdate method. +*/ +int sqlite3_vtab_on_conflict(sqlite3 *db){ + static const unsigned char aMap[] = { + SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE + }; + assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); + assert( OE_Ignore==4 && OE_Replace==5 ); + assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); + return (int)aMap[db->vtabOnConflict-1]; +} + +/* +** Call from within the xCreate() or xConnect() methods to provide +** the SQLite core with additional information about the behavior +** of the virtual table being implemented. +*/ +int sqlite3_vtab_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { + VtabCtx *p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + p->pVTable->bConstraint = (u8)va_arg(ap, int); + } + break; + } + default: + rc = SQLITE_MISUSE_BKPT; + break; + } + va_end(ap); + + if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + sqlite3_mutex_leave(db->mutex); + return rc; +} + #endif /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/src/wal.c b/src/wal.c index 51ea18fb..b9a03dff 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,6 +412,7 @@ struct Wal { sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ + i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u32 szPage; /* Database page size */ @@ -419,7 +420,7 @@ struct Wal { u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ - u8 readOnly; /* True if the WAL file is open read-only */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -435,6 +436,13 @@ struct Wal { #define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 +/* +** Possible values for WAL.readOnly +*/ +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ + /* ** Each page of the wal-index mapping contains a hash-table made up of ** an array of HASHTABLE_NSLOT elements of the following type. @@ -528,6 +536,10 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); + if( rc==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + rc = SQLITE_OK; + } } } @@ -1234,6 +1246,7 @@ int sqlite3WalOpen( sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ + i64 mxWalSize, /* Truncate WAL to this size on reset */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ @@ -1266,6 +1279,7 @@ int sqlite3WalOpen( pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; pRet->readLock = -1; + pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); @@ -1273,7 +1287,7 @@ int sqlite3WalOpen( flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ - pRet->readOnly = 1; + pRet->readOnly = WAL_RDONLY; } if( rc!=SQLITE_OK ){ @@ -1287,6 +1301,13 @@ int sqlite3WalOpen( return rc; } +/* +** Change the size to which the WAL file is trucated on each reset. +*/ +void sqlite3WalLimit(Wal *pWal, i64 iLimit){ + if( pWal ) pWal->mxWalSize = iLimit; +} + /* ** Find the smallest page number out of all pages held in the WAL that ** has not been returned by any prior invocation of this method on the @@ -1907,21 +1928,28 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** with a writer. So get a WRITE lock and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); - if( badHdr && SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; + if( badHdr ){ + if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; } + }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } /* If the header is read successfully, check the version number to make @@ -2108,7 +2136,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } /* There was once an "if" here. The extra "{" is to preserve indentation. */ { - if( mxReadMark < pWal->hdr.mxFrame || mxI==0 ){ + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkhdr.mxFrame || mxI==0) + ){ for(i=1; ireadOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); @@ -2522,6 +2552,24 @@ static int walRestartLog(Wal *pWal){ */ int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + + /* Limit the size of WAL file if the journal_size_limit PRAGMA is + ** set to a non-negative value. Log errors encountered + ** during the truncation attempt. */ + if( pWal->mxWalSize>=0 ){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->pWalFd, &sz); + if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){ + rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } + } + pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); @@ -2747,6 +2795,7 @@ int sqlite3WalCheckpoint( assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ diff --git a/src/wal.h b/src/wal.h index 2039c701..a62b23bb 100644 --- a/src/wal.h +++ b/src/wal.h @@ -21,6 +21,7 @@ #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalLimit(x,y) # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) @@ -46,9 +47,12 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *zName, int, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); +/* Set the limiting size of a WAL file. */ +void sqlite3WalLimit(Wal*, i64); + /* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database ** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and diff --git a/test/8_3_names.test b/test/8_3_names.test new file mode 100644 index 00000000..418685e6 --- /dev/null +++ b/test/8_3_names.test @@ -0,0 +1,197 @@ +# 2011 May 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the SQLITE_ENABLE_8_3_NAMES feature that forces all +# filename extensions to be limited to 3 characters. Some embedded +# systems need this to work around microsoft FAT patents, but this +# feature should be disabled on most deployments. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !8_3_names { + finish_test + return +} + +db close +sqlite3_shutdown +sqlite3_config_uri 1 + +do_test 8_3_names-1.0 { + forcedelete test.db test.nal test.db-journal + sqlite3 db test.db + db eval { + PRAGMA cache_size=10; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(20000)); + BEGIN; + DELETE FROM t1; + INSERT INTO t1 VALUES(randomblob(15000)); + } + file exists test.db-journal +} 1 +do_test 8_3_names-1.1 { + file exists test.nal +} 0 +do_test 8_3_names-1.2 { + db eval { + ROLLBACK; + SELECT length(x) FROM t1 + } +} 20000 + +db close +do_test 8_3_names-2.0 { + forcedelete test.db test.nal test.db-journal + sqlite3 db file:./test.db?8_3_names=1 + db eval { + PRAGMA cache_size=10; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(20000)); + BEGIN; + DELETE FROM t1; + INSERT INTO t1 VALUES(randomblob(15000)); + } + file exists test.db-journal +} 0 +do_test 8_3_names-2.1 { + file exists test.nal +} 1 +forcedelete test2.db test2.nal test2.db-journal +file copy test.db test2.db +file copy test.nal test2.nal +do_test 8_3_names-2.2 { + db eval { + COMMIT; + SELECT length(x) FROM t1 + } +} 15000 +do_test 8_3_names-2.3 { + sqlite3 db2 file:./test2.db?8_3_names=1 + db2 eval { + PRAGMA integrity_check; + SELECT length(x) FROM t1; + } +} {ok 20000} + +db close +do_test 8_3_names-3.0 { + forcedelete test.db test.nal test.db-journal + sqlite3 db file:./test.db?8_3_names=0 + db eval { + PRAGMA cache_size=10; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(20000)); + BEGIN; + DELETE FROM t1; + INSERT INTO t1 VALUES(randomblob(15000)); + } + file exists test.db-journal +} 1 +do_test 8_3_names-3.1 { + file exists test.nal +} 0 +forcedelete test2.db test2.nal test2.db-journal +file copy test.db test2.db +file copy test.db-journal test2.db-journal +do_test 8_3_names-3.2 { + db eval { + COMMIT; + SELECT length(x) FROM t1 + } +} 15000 +do_test 8_3_names-3.3 { + sqlite3 db2 file:./test2.db?8_3_names=0 + db2 eval { + PRAGMA integrity_check; + SELECT length(x) FROM t1; + } +} {ok 20000} + +########################################################################## +# Master journals. +# +db close +forcedelete test.db test2.db +do_test 8_3_names-4.0 { + sqlite3 db file:./test.db?8_3_names=1 + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + ATTACH 'file:./test2.db?8_3_names=1' AS db2; + CREATE TABLE db2.t2(y); + INSERT INTO t2 VALUES(2); + BEGIN; + INSERT INTO t1 VALUES(3); + INSERT INTO t2 VALUES(4); + COMMIT; + SELECT * FROM t1, t2 ORDER BY x, y + } +} {1 2 1 4 3 2 3 4} + + +########################################################################## +# WAL mode. +# +ifcapable !wal { + finish_test + return +} +db close +forcedelete test.db +do_test 8_3_names-5.0 { + sqlite3 db file:./test.db?8_3_names=1 + register_wholenumber_module db + db eval { + PRAGMA journal_mode=WAL; + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE nums USING wholenumber; + INSERT INTO t1 SELECT value FROM nums WHERE value BETWEEN 1 AND 1000; + BEGIN; + UPDATE t1 SET x=x*2; + } + sqlite3 db2 file:./test.db?8_3_names=1 + register_wholenumber_module db2 + db2 eval { + BEGIN; + SELECT sum(x) FROM t1; + } +} {500500} + +do_test 8_3_names-5.1 { + file exists test.db-wal +} 0 +do_test 8_3_names-5.2 { + file exists test.wal +} 1 +do_test 8_3_names-5.3 { + file exists test.db-shm +} 0 +do_test 8_3_names-5.4 { + file exists test.shm +} 1 + + +do_test 8_3_names-5.5 { + db eval { + COMMIT; + SELECT sum(x) FROM t1; + } +} {1001000} +do_test 8_3_names-5.6 { + db2 eval { + SELECT sum(x) FROM t1; + } +} {500500} + + +finish_test diff --git a/test/alter.test b/test/alter.test index d4b72a6a..359034d5 100644 --- a/test/alter.test +++ b/test/alter.test @@ -650,7 +650,7 @@ do_test alter-6.7 { # Ticket #1665: Make sure ALTER TABLE ADD COLUMN works on a table # that includes a COLLATE clause. # -do_test alter-7.1 { +do_realnum_test alter-7.1 { execsql { CREATE TABLE t1(a TEXT COLLATE BINARY); ALTER TABLE t1 ADD COLUMN b INTEGER COLLATE NOCASE; diff --git a/test/cast.test b/test/cast.test index 7a239439..f47f4bb2 100644 --- a/test/cast.test +++ b/test/cast.test @@ -234,7 +234,7 @@ do_test cast-3.1 { do_test cast-3.2 { execsql {SELECT CAST(9223372036854774800 AS numeric)} } 9223372036854774800 -do_test cast-3.3 { +do_realnum_test cast-3.3 { execsql {SELECT CAST(9223372036854774800 AS real)} } 9.22337203685477e+18 do_test cast-3.4 { @@ -246,7 +246,7 @@ do_test cast-3.5 { do_test cast-3.6 { execsql {SELECT CAST(-9223372036854774800 AS numeric)} } -9223372036854774800 -do_test cast-3.7 { +do_realnum_test cast-3.7 { execsql {SELECT CAST(-9223372036854774800 AS real)} } -9.22337203685477e+18 do_test cast-3.8 { @@ -258,7 +258,7 @@ do_test cast-3.11 { do_test cast-3.12 { execsql {SELECT CAST('9223372036854774800' AS numeric)} } 9223372036854774800 -do_test cast-3.13 { +do_realnum_test cast-3.13 { execsql {SELECT CAST('9223372036854774800' AS real)} } 9.22337203685477e+18 ifcapable long_double { @@ -272,7 +272,7 @@ do_test cast-3.15 { do_test cast-3.16 { execsql {SELECT CAST('-9223372036854774800' AS numeric)} } -9223372036854774800 -do_test cast-3.17 { +do_realnum_test cast-3.17 { execsql {SELECT CAST('-9223372036854774800' AS real)} } -9.22337203685477e+18 ifcapable long_double { @@ -287,7 +287,7 @@ if {[db eval {PRAGMA encoding}]=="UTF-8"} { do_test cast-3.22 { execsql {SELECT CAST(x'39323233333732303336383534373734383030' AS numeric)} } 9223372036854774800 - do_test cast-3.23 { + do_realnum_test cast-3.23 { execsql {SELECT CAST(x'39323233333732303336383534373734383030' AS real)} } 9.22337203685477e+18 ifcapable long_double { diff --git a/test/e_createtable.test b/test/e_createtable.test index c67195ed..f61db1cd 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -989,9 +989,9 @@ do_execsql_test e_createtable-3.7.4 { SELECT quote(a), quote(b) FROM t6; } {1 2 'X' 3 1 4 'X' 5} -# EVIDENCE-OF: R-18683-56219 If the default value of a column is -# CURRENT_TIME, CURRENT_DATE or CURRENT_DATETIME, then the value used in -# the new row is a text representation of the current UTC date and/or +# EVIDENCE-OF: R-15363-55230 If the default value of a column is +# CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP, then the value used +# in the new row is a text representation of the current UTC date and/or # time. # # This is difficult to test literally without knowing what time the diff --git a/test/e_delete.test b/test/e_delete.test index de391b11..c77d4442 100644 --- a/test/e_delete.test +++ b/test/e_delete.test @@ -440,7 +440,7 @@ do_delete_tests e_delete-3.9 -repair rebuild_t1 -query { } -# EVIDENCE-OF: R-26627-30313 The ORDER BY clause on an DELETE statement +# EVIDENCE-OF: R-07548-13422 The ORDER BY clause on a DELETE statement # is used only to determine which rows fall within the LIMIT. The order # in which rows are deleted is arbitrary and is not influenced by the # ORDER BY clause. diff --git a/test/e_expr.test b/test/e_expr.test index 1e768535..b61b42da 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -1598,8 +1598,8 @@ do_expr_test e_expr-31.1.2 { CAST(1.99999 AS INTEGER) } integer 1 do_expr_test e_expr-31.1.3 { CAST(-1.99999 AS INTEGER) } integer -1 do_expr_test e_expr-31.1.4 { CAST(-0.99999 AS INTEGER) } integer 0 -# EVIDENCE-OF: R-06126-36021 If an REAL is too large to be represented -# as an INTEGER then the result of the cast is the largest negative +# EVIDENCE-OF: R-49503-28105 If a REAL is too large to be represented as +# an INTEGER then the result of the cast is the largest negative # integer: -9223372036854775808. # do_expr_test e_expr-31.2.1 { CAST(2e+50 AS INT) } integer -9223372036854775808 @@ -1845,4 +1845,3 @@ foreach {tn expr} { finish_test - diff --git a/test/e_insert.test b/test/e_insert.test index 2192f2e9..fe8bfcf8 100644 --- a/test/e_insert.test +++ b/test/e_insert.test @@ -152,7 +152,7 @@ do_insert_tests e_insert-1.2 -error { 4 "INSERT INTO a2 VALUES(1,2,3,4,5)" {a2 3 5} } -# EVIDENCE-OF: R-52422-65517 In this case the result of evaluting the +# EVIDENCE-OF: R-04006-57648 In this case the result of evaluating the # left-most expression in the VALUES list is inserted into the left-most # column of the new row, and so on. # diff --git a/test/e_select.test b/test/e_select.test index 79e334ce..6194872f 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -1768,9 +1768,9 @@ do_select_tests e_select-8.5 { } } -# EVIDENCE-OF: R-27923-38747 Otherwise, if the ORDER BY expression is -# any other expression, it is evaluated and the the returned value used -# to order the output rows. +# EVIDENCE-OF: R-65068-27207 Otherwise, if the ORDER BY expression is +# any other expression, it is evaluated and the returned value used to +# order the output rows. # # EVIDENCE-OF: R-03421-57988 If the SELECT statement is a simple SELECT, # then an ORDER BY may contain any arbitrary expressions. diff --git a/test/e_update.test b/test/e_update.test index d8032ce0..c14b8456 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -233,9 +233,9 @@ do_update_tests e_update-1.5 -query { {3 1 4 1 5 9 2 6 5} } -# EVIDENCE-OF: R-09060-20018 If a single column-name appears more than +# EVIDENCE-OF: R-34751-18293 If a single column-name appears more than # once in the list of assignment expressions, all but the rightmost -# occurence is ignored. +# occurrence is ignored. # do_update_tests e_update-1.6 -query { SELECT * FROM t2 @@ -605,4 +605,3 @@ do_update_tests e_update-3.5 -query { SELECT x FROM t8 ; DELETE FROM t8 } { } ;# ifcapable update_delete_limit finish_test - diff --git a/test/e_uri.test b/test/e_uri.test new file mode 100644 index 00000000..5275ec1b --- /dev/null +++ b/test/e_uri.test @@ -0,0 +1,456 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_uri + +db close + +proc parse_uri {uri} { + testvfs tvfs2 + testvfs tvfs + tvfs filter xOpen + tvfs script parse_uri_open_cb + + set ::uri_open [list] + set DB [sqlite3_open_v2 $uri { + SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_WAL + } tvfs] + sqlite3_close $DB + tvfs delete + tvfs2 delete + + set ::uri_open +} +proc parse_uri_open_cb {method file arglist} { + set ::uri_open [list $file $arglist] +} + +proc open_uri_error {uri} { + set flags {SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_WAL} + set DB [sqlite3_open_v2 $uri $flags ""] + set e [sqlite3_errmsg $DB] + sqlite3_close $DB + set e +} + +# EVIDENCE-OF: R-35840-33204 If URI filename interpretation is enabled, +# and the filename argument begins with "file:", then the filename is +# interpreted as a URI. +# +# EVIDENCE-OF: R-24124-56960 URI filename interpretation is enabled if +# the SQLITE_OPEN_URI flag is set in the fourth argument to +# sqlite3_open_v2(), or if it has been enabled globally using the +# SQLITE_CONFIG_URI option with the sqlite3_config() method or by the +# SQLITE_USE_URI compile-time option. +# +if {$tcl_platform(platform) == "unix"} { + set flags [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE] + + # Tests with SQLITE_CONFIG_URI configured to false. URI intepretation is + # only enabled if the SQLITE_OPEN_URI flag is specified. + sqlite3_shutdown + sqlite3_config_uri 0 + do_test 1.1 { + forcedelete file:test.db test.db + set DB [sqlite3_open_v2 file:test.db [concat $flags SQLITE_OPEN_URI] ""] + list [file exists file:test.db] [file exists test.db] + } {0 1} + do_test 1.2 { + forcedelete file:test.db2 test.db2 + set STMT [sqlite3_prepare $DB "ATTACH 'file:test.db2' AS aux" -1 dummy] + sqlite3_step $STMT + sqlite3_finalize $STMT + list [file exists file:test.db2] [file exists test.db2] + } {0 1} + sqlite3_close $DB + do_test 1.3 { + forcedelete file:test.db test.db + set DB [sqlite3_open_v2 file:test.db [concat $flags] ""] + list [file exists file:test.db] [file exists test.db] + } {1 0} + do_test 1.4 { + forcedelete file:test.db2 test.db2 + set STMT [sqlite3_prepare $DB "ATTACH 'file:test.db2' AS aux" -1 dummy] + sqlite3_step $STMT + sqlite3_finalize $STMT + list [file exists file:test.db2] [file exists test.db2] + } {1 0} + sqlite3_close $DB + + # Tests with SQLITE_CONFIG_URI configured to true. URI intepretation is + # enabled with or without SQLITE_OPEN_URI. + # + sqlite3_shutdown + sqlite3_config_uri 1 + do_test 1.5 { + forcedelete file:test.db test.db + set DB [sqlite3_open_v2 file:test.db [concat $flags SQLITE_OPEN_URI] ""] + list [file exists file:test.db] [file exists test.db] + } {0 1} + do_test 1.6 { + forcedelete file:test.db2 test.db2 + set STMT [sqlite3_prepare $DB "ATTACH 'file:test.db2' AS aux" -1 dummy] + sqlite3_step $STMT + sqlite3_finalize $STMT + list [file exists file:test.db2] [file exists test.db2] + } {0 1} + sqlite3_close $DB + do_test 1.7 { + forcedelete file:test.db test.db + set DB [sqlite3_open_v2 file:test.db [concat $flags] ""] + list [file exists file:test.db] [file exists test.db] + } {0 1} + do_test 1.8 { + forcedelete file:test.db2 test.db2 + set STMT [sqlite3_prepare $DB "ATTACH 'file:test.db2' AS aux" -1 dummy] + sqlite3_step $STMT + sqlite3_finalize $STMT + list [file exists file:test.db2] [file exists test.db2] + } {0 1} + sqlite3_close $DB +} + +# ensure uri processing enabled for the rest of the tests +sqlite3_shutdown +sqlite3_config_uri 1 + +# EVIDENCE-OF: R-17482-00398 If the authority is not an empty string or +# "localhost", an error is returned to the caller. +# +if {$tcl_platform(platform) == "unix"} { + set flags [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_URI] + foreach {tn uri error} " + 1 {file://localhost[pwd]/test.db} {not an error} + 2 {file://[pwd]/test.db} {not an error} + 3 {file://x[pwd]/test.db} {invalid uri authority: x} + 4 {file://invalid[pwd]/test.db} {invalid uri authority: invalid} + " { + do_test 2.$tn { + set DB [sqlite3_open_v2 $uri $flags ""] + set e [sqlite3_errmsg $DB] + sqlite3_close $DB + set e + } $error + } +} + +# EVIDENCE-OF: R-45981-25528 The fragment component of a URI, if +# present, is ignored. +# +# It is difficult to test that something is ignored correctly. So these tests +# just show that adding a fragment does not interfere with the pathname or +# parameters passed through to the VFS xOpen() methods. +# +foreach {tn uri parse} " + 1 {file:test.db#abc} {[pwd]/test.db {}} + 2 {file:test.db?a=b#abc} {[pwd]/test.db {a b}} + 3 {file:test.db?a=b#?c=d} {[pwd]/test.db {a b}} +" { + do_filepath_test 3.$tn { parse_uri $uri } $parse +} + +# EVIDENCE-OF: R-62557-09390 SQLite uses the path component of the URI +# as the name of the disk file which contains the database. +# +# EVIDENCE-OF: R-28659-11035 If the path begins with a '/' character, +# then it is interpreted as an absolute path. +# +# EVIDENCE-OF: R-46234-61323 If the path does not begin with a '/' +# (meaning that the authority section is omitted from the URI) then the +# path is interpreted as a relative path. +# +foreach {tn uri parse} " + 1 {file:test.db} {[pwd]/test.db {}} + 2 {file:/test.db} {/test.db {}} + 3 {file:///test.db} {/test.db {}} + 4 {file://localhost/test.db} {/test.db {}} + 5 {file:/a/b/c/test.db} {/a/b/c/test.db {}} +" { + do_filepath_test 4.$tn { parse_uri $uri } $parse +} + +# EVIDENCE-OF: R-01612-30877 The "vfs" parameter may be used to specify +# the name of a VFS object that provides the operating system interface +# that should be used to access the database file on disk. +# +# The above is tested by cases 1.* below. +# +# EVIDENCE-OF: R-52293-58497 If this option is set to an empty string +# the default VFS object is used. +# +# The above is tested by cases 2.* below. +# +# EVIDENCE-OF: R-31855-18665 If sqlite3_open_v2() is used and the vfs +# option is present, then the VFS specified by the option takes +# precedence over the value passed as the fourth parameter to +# sqlite3_open_v2(). +# +# The above is tested by cases 3.* below. +# +proc vfs_open_cb {name args} { + set ::vfs $name +} +foreach {name default} {vfs1 0 vfs2 0 vfs3 1} { + testvfs $name -default $default + $name filter xOpen + $name script [list vfs_open_cb $name] +} +foreach {tn uri defvfs vfs} { + 1.1 "file:test.db?vfs=vfs1" "" vfs1 + 1.2 "file:test.db?vfs=vfs2" "" vfs2 + + 2.1 "file:test.db" vfs1 vfs1 + 2.2 "file:test.db?vfs=" vfs1 vfs3 + + 3.1 "file:test.db?vfs=vfs1" vfs2 vfs1 + 3.2 "file:test.db?vfs=vfs2" vfs1 vfs2 + 3.3 "file:test.db?xvfs=vfs1" vfs2 vfs2 + 3.4 "file:test.db?xvfs=vfs2" vfs1 vfs1 +} { + do_test 5.$tn { + set flags [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_URI] + sqlite3_close [ + sqlite3_open_v2 $uri $flags $defvfs + ] + set ::vfs + } $vfs +} +vfs1 delete +vfs2 delete +vfs3 delete + +# EVIDENCE-OF: R-48365-36308 Specifying an unknown VFS is an error. +# +set flags [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_URI] +do_test 6.1 { + set DB [sqlite3_open_v2 file:test.db?vfs=nosuchvfs $flags ""] + set errmsg [sqlite3_errmsg $DB] + sqlite3_close $DB + set errmsg +} {no such vfs: nosuchvfs} + + +# EVIDENCE-OF: R-60479-64270 The mode parameter may be set to either +# "ro", "rw" or "rwc". Attempting to set it to any other value is an +# error +# +sqlite3 db test.db +db close +foreach {tn uri error} " + 1 {file:test.db?mode=ro} {not an error} + 2 {file:test.db?mode=rw} {not an error} + 3 {file:test.db?mode=rwc} {not an error} + 4 {file:test.db?mode=Ro} {no such access mode: Ro} + 5 {file:test.db?mode=Rw} {no such access mode: Rw} + 6 {file:test.db?mode=Rwc} {no such access mode: Rwc} +" { + do_test 7.$tn { open_uri_error $uri } $error +} + + +# EVIDENCE-OF: R-09651-31805 If "ro" is specified, then the database is +# opened for read-only access, just as if the SQLITE_OPEN_READONLY flag +# had been set in the third argument to sqlite3_prepare_v2(). +# +# EVIDENCE-OF: R-40137-26050 If the mode option is set to "rw", then the +# database is opened for read-write (but not create) access, as if +# SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had been set. +# +# EVIDENCE-OF: R-26845-32976 Value "rwc" is equivalent to setting both +# SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. +# +foreach {tn uri read write create} { + 1 {file:test.db?mode=ro} 1 0 0 + 2 {file:test.db?mode=rw} 1 1 0 + 3 {file:test.db?mode=rwc} 1 1 1 +} { + set RES(c,0) {1 {unable to open database file}} + set RES(c,1) {0 {}} + set RES(w,0) {1 {attempt to write a readonly database}} + set RES(w,1) {0 {}} + set RES(r,0) {1 {this never happens}} + set RES(r,1) {0 {a b}} + + # Test CREATE access: + forcedelete test.db + do_test 8.$tn.c { list [catch { sqlite3 db $uri } msg] $msg } $RES(c,$create) + catch { db close } + + sqlite3 db test.db + db eval { CREATE TABLE t1(a, b) ; INSERT INTO t1 VALUES('a', 'b') ;} + db close + + # Test READ access: + do_test 8.$tn.r { + sqlite3 db $uri + catchsql { SELECT * FROM t1 } + } $RES(r,$read) + + # Test WRITE access: + do_test 8.$tn.w { + sqlite3 db $uri + catchsql { INSERT INTO t1 VALUES(1, 2) } + } $RES(w,$write) + + catch {db close} +} + +# EVIDENCE-OF: R-56032-32287 If sqlite3_open_v2() is used, it is an +# error to specify a value for the mode parameter that is less +# restrictive than that specified by the flags passed as the third +# parameter. +# +forcedelete test.db +sqlite3 db test.db +db close +foreach {tn uri flags error} { + 1 {file:test.db?mode=ro} ro {not an error} + 2 {file:test.db?mode=ro} rw {not an error} + 3 {file:test.db?mode=ro} rwc {not an error} + + 4 {file:test.db?mode=rw} ro {access mode not allowed: rw} + 5 {file:test.db?mode=rw} rw {not an error} + 6 {file:test.db?mode=rw} rwc {not an error} + + 7 {file:test.db?mode=rwc} ro {access mode not allowed: rwc} + 8 {file:test.db?mode=rwc} rw {access mode not allowed: rwc} + 9 {file:test.db?mode=rwc} rwc {not an error} +} { + set f(ro) [list SQLITE_OPEN_READONLY SQLITE_OPEN_URI] + set f(rw) [list SQLITE_OPEN_READWRITE SQLITE_OPEN_URI] + set f(rwc) [list SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_URI] + + set DB [sqlite3_open_v2 $uri $f($flags) ""] + set e [sqlite3_errmsg $DB] + sqlite3_close $DB + + do_test 9.$tn { set e } $error +} + +# EVIDENCE-OF: R-23182-54295 The cache parameter may be set to either +# "shared" or "private". +sqlite3 db test.db +db close +foreach {tn uri error} " + 1 {file:test.db?cache=private} {not an error} + 2 {file:test.db?cache=shared} {not an error} + 3 {file:test.db?cache=yes} {no such cache mode: yes} + 4 {file:test.db?cache=} {no such cache mode: } +" { + do_test 10.$tn { open_uri_error $uri } $error +} + +# EVIDENCE-OF: R-23027-03515 Setting it to "shared" is equivalent to +# setting the SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed +# to sqlite3_open_v2(). +# +# EVIDENCE-OF: R-49793-28525 Setting the cache parameter to "private" is +# equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +# +# EVIDENCE-OF: R-19510-48080 If sqlite3_open_v2() is used and the +# "cache" parameter is present in a URI filename, its value overrides +# any behaviour requested by setting SQLITE_OPEN_PRIVATECACHE or +# SQLITE_OPEN_SHAREDCACHE flag. +# +set orig [sqlite3_enable_shared_cache] +foreach {tn uri flags shared_default isshared} { + 1.1 "file:test.db" "" 0 0 + 1.2 "file:test.db" "" 1 1 + 1.3 "file:test.db" private 0 0 + 1.4 "file:test.db" private 1 0 + 1.5 "file:test.db" shared 0 1 + 1.6 "file:test.db" shared 1 1 + + 2.1 "file:test.db?cache=private" "" 0 0 + 2.2 "file:test.db?cache=private" "" 1 0 + 2.3 "file:test.db?cache=private" private 0 0 + 2.4 "file:test.db?cache=private" private 1 0 + 2.5 "file:test.db?cache=private" shared 0 0 + 2.6 "file:test.db?cache=private" shared 1 0 + + 3.1 "file:test.db?cache=shared" "" 0 1 + 3.2 "file:test.db?cache=shared" "" 1 1 + 3.3 "file:test.db?cache=shared" private 0 1 + 3.4 "file:test.db?cache=shared" private 1 1 + 3.5 "file:test.db?cache=shared" shared 0 1 + 3.6 "file:test.db?cache=shared" shared 1 1 +} { + forcedelete test.db + sqlite3_enable_shared_cache 1 + sqlite3 db test.db + sqlite3_enable_shared_cache 0 + + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('ok'); + } + + unset -nocomplain f + set f() {SQLITE_OPEN_READWRITE SQLITE_OPEN_CREATE SQLITE_OPEN_URI} + set f(shared) [concat $f() SQLITE_OPEN_SHAREDCACHE] + set f(private) [concat $f() SQLITE_OPEN_PRIVATECACHE] + + sqlite3_enable_shared_cache $shared_default + set DB [sqlite3_open_v2 $uri $f($flags) ""] + + set STMT [sqlite3_prepare $DB "SELECT * FROM t1" -1 dummy] + + db eval { + BEGIN; + INSERT INTO t1 VALUES('ko'); + } + + sqlite3_step $STMT + sqlite3_finalize $STMT + + set RES(0) {not an error} + set RES(1) {database table is locked: t1} + + do_test 11.$tn { sqlite3_errmsg $DB } $RES($isshared) + + sqlite3_close $DB + db close +} +sqlite3_enable_shared_cache $orig + +# EVIDENCE-OF: R-63472-46769 Specifying an unknown parameter in the +# query component of a URI is not an error. +# +do_filepath_test 12.1 { + parse_uri file://localhost/test.db?an=unknown¶meter=is&ok= +} {/test.db {an unknown parameter is ok {}}} +do_filepath_test 12.2 { + parse_uri file://localhost/test.db?an&unknown¶meter&is&ok +} {/test.db {an {} unknown {} parameter {} is {} ok {}}} + +# EVIDENCE-OF: R-27458-04043 URI hexadecimal escape sequences (%HH) are +# supported within the path and query components of a URI. +# +# EVIDENCE-OF: R-52765-50368 Before the path or query components of a +# URI filename are interpreted, they are encoded using UTF-8 and all +# hexadecimal escape sequences replaced by a single byte containing the +# corresponding octet. +# +# The second of the two statements above is tested by creating a +# multi-byte utf-8 character using a sequence of %HH escapes. +# +foreach {tn uri parse} " + 1 {file:/test.%64%62} {/test.db {}} + 2 {file:/test.db?%68%65%6c%6c%6f=%77%6f%72%6c%64} {/test.db {hello world}} + 3 {file:/%C3%BF.db} {/\xFF.db {}} +" { + do_filepath_test 13.$tn { parse_uri $uri } $parse +} + +finish_test diff --git a/test/enc4.test b/test/enc4.test index de2a1b88..7de6e849 100644 --- a/test/enc4.test +++ b/test/enc4.test @@ -1,139 +1,137 @@ -# 2010 Sept 29 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The focus of -# this file is testing the SQLite routines used for converting between the -# various suported unicode encodings (UTF-8, UTF-16, UTF-16le and -# UTF-16be). -# -# $Id: enc4.test,v 1.0 2010/09/29 08:29:32 shaneh Exp $ - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -# If UTF16 support is disabled, ignore the tests in this file -# -ifcapable {!utf16} { - finish_test - return -} - -db close - -# The three unicode encodings understood by SQLite. -set encodings [list UTF-8 UTF-16le UTF-16be] - -# initial value to use in SELECT -set inits [list 1 1.0 1. 1e0] - -# vals -set vals [list\ -"922337203685477580792233720368547758079223372036854775807"\ -"100000000000000000000000000000000000000000000000000000000"\ -"1.0000000000000000000000000000000000000000000000000000000"\ -] - -set i 1 -foreach enc $encodings { - - file delete -force test.db - sqlite3 db test.db - db eval "PRAGMA encoding = \"$enc\"" - - do_test enc4-$i.1 { - db eval {PRAGMA encoding} - } $enc - - set j 1 - foreach init $inits { - - do_test enc4-$i.$j.2 { - set S [sqlite3_prepare_v2 db "SELECT $init+?" -1 dummy] - sqlite3_expired $S - } {0} - - set k 1 - foreach val $vals { - for {set x 1} {$x<18} {incr x} { - set part [expr $init + [string range $val 0 [expr $x-1]]] - regsub {e\+0} $part {e+} part - regsub {^1e} $part {1.0e} part - - do_test enc4-$i.$j.$k.3.$x { - sqlite3_reset $S - sqlite3_bind_text $S 1 $val $x - sqlite3_step $S - sqlite3_column_text $S 0 - } [list $part] - - do_test enc4-$i.$j.$k.4.$x { - sqlite3_reset $S - sqlite3_bind_text16 $S 1 [encoding convertto unicode $val] [expr $x*2] - sqlite3_step $S - sqlite3_column_text $S 0 - } [list $part] - } - - incr k - } - - do_test enc4-$i.$j.5 { - sqlite3_finalize $S - } {SQLITE_OK} - - incr j - } - - db close - incr i -} - -file delete -force test.db -sqlite3 db test.db - -do_test enc4-4.1 { - db eval "select 1+1." -} {2.0} - -do_test enc4-4.2.1 { - set S [sqlite3_prepare_v2 db "SELECT 1+1." -1 dummy] - sqlite3_step $S - sqlite3_column_text $S 0 -} {2.0} - -do_test enc4-4.2.2 { - sqlite3_finalize $S -} {SQLITE_OK} - -do_test enc4-4.3.1 { - set S [sqlite3_prepare_v2 db "SELECT 1+?" -1 dummy] - sqlite3_bind_text $S 1 "1." 2 - sqlite3_step $S - sqlite3_column_text $S 0 -} {2.0} - -do_test enc4-4.3.2 { - sqlite3_finalize $S -} {SQLITE_OK} - -do_test enc4-4.4.1 { - set S [sqlite3_prepare_v2 db "SELECT 1+?" -1 dummy] - sqlite3_bind_text $S 1 "1.0" 2 - sqlite3_step $S - sqlite3_column_text $S 0 -} {2.0} - -do_test enc4-4.4.2 { - sqlite3_finalize $S -} {SQLITE_OK} - -db close - -finish_test +# 2010 Sept 29 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The focus of +# this file is testing the SQLite routines used for converting between the +# various suported unicode encodings (UTF-8, UTF-16, UTF-16le and +# UTF-16be). +# +# $Id: enc4.test,v 1.0 2010/09/29 08:29:32 shaneh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If UTF16 support is disabled, ignore the tests in this file +# +ifcapable {!utf16} { + finish_test + return +} + +db close + +# The three unicode encodings understood by SQLite. +set encodings [list UTF-8 UTF-16le UTF-16be] + +# initial value to use in SELECT +set inits [list 1 1.0 1. 1e0] + +# vals +set vals [list\ +"922337203685477580792233720368547758079223372036854775807"\ +"100000000000000000000000000000000000000000000000000000000"\ +"1.0000000000000000000000000000000000000000000000000000000"\ +] + +set i 1 +foreach enc $encodings { + + file delete -force test.db + sqlite3 db test.db + db eval "PRAGMA encoding = \"$enc\"" + + do_test enc4-$i.1 { + db eval {PRAGMA encoding} + } $enc + + set j 1 + foreach init $inits { + + do_test enc4-$i.$j.2 { + set S [sqlite3_prepare_v2 db "SELECT $init+?" -1 dummy] + sqlite3_expired $S + } {0} + + set k 1 + foreach val $vals { + for {set x 1} {$x<16} {incr x} { + set part [expr $init + [string range $val 0 [expr $x-1]]] + + do_realnum_test enc4-$i.$j.$k.3.$x { + sqlite3_reset $S + sqlite3_bind_text $S 1 $val $x + sqlite3_step $S + sqlite3_column_text $S 0 + } [list $part] + + do_realnum_test enc4-$i.$j.$k.4.$x { + sqlite3_reset $S + sqlite3_bind_text16 $S 1 [encoding convertto unicode $val] [expr $x*2] + sqlite3_step $S + sqlite3_column_text $S 0 + } [list $part] + } + + incr k + } + + do_test enc4-$i.$j.5 { + sqlite3_finalize $S + } {SQLITE_OK} + + incr j + } + + db close + incr i +} + +file delete -force test.db +sqlite3 db test.db + +do_test enc4-4.1 { + db eval "select 1+1." +} {2.0} + +do_test enc4-4.2.1 { + set S [sqlite3_prepare_v2 db "SELECT 1+1." -1 dummy] + sqlite3_step $S + sqlite3_column_text $S 0 +} {2.0} + +do_test enc4-4.2.2 { + sqlite3_finalize $S +} {SQLITE_OK} + +do_test enc4-4.3.1 { + set S [sqlite3_prepare_v2 db "SELECT 1+?" -1 dummy] + sqlite3_bind_text $S 1 "1." 2 + sqlite3_step $S + sqlite3_column_text $S 0 +} {2.0} + +do_test enc4-4.3.2 { + sqlite3_finalize $S +} {SQLITE_OK} + +do_test enc4-4.4.1 { + set S [sqlite3_prepare_v2 db "SELECT 1+?" -1 dummy] + sqlite3_bind_text $S 1 "1.0" 2 + sqlite3_step $S + sqlite3_column_text $S 0 +} {2.0} + +do_test enc4-4.4.2 { + sqlite3_finalize $S +} {SQLITE_OK} + +db close + +finish_test diff --git a/test/expr.test b/test/expr.test index 5a3d1675..cc4c9c67 100644 --- a/test/expr.test +++ b/test/expr.test @@ -32,6 +32,11 @@ proc test_expr {name settings expr result} { execsql {BEGIN; UPDATE test1 SET %s; SELECT %s FROM test1; ROLLBACK;} } $settings $expr] $result } +proc test_realnum_expr {name settings expr result} { + do_realnum_test $name [format { + execsql {BEGIN; UPDATE test1 SET %s; SELECT %s FROM test1; ROLLBACK;} + } $settings $expr] $result +} test_expr expr-1.1 {i1=10, i2=20} {i1+i2} 30 test_expr expr-1.2 {i1=10, i2=20} {i1-i2} -10 @@ -164,7 +169,7 @@ ifcapable floatingpoint { } if {[working_64bit_int]} { - test_expr expr-1.106 {i1=0} {-9223372036854775808/-1} 9.22337203685478e+18 + test_realnum_expr expr-1.106 {i1=0} {-9223372036854775808/-1} 9.22337203685478e+18 } test_expr expr-1.107 {i1=0} {-9223372036854775808%-1} 0 @@ -203,100 +208,100 @@ test_expr expr-1.126 {i1=8, i2=8} \ ifcapable floatingpoint {if {[working_64bit_int]} { test_expr expr-1.200\ {i1=9223372036854775806, i2=1} {i1+i2} 9223372036854775807 - test_expr expr-1.201\ + test_realnum_expr expr-1.201\ {i1=9223372036854775806, i2=2} {i1+i2} 9.22337203685478e+18 - test_expr expr-1.202\ + test_realnum_expr expr-1.202\ {i1=9223372036854775806, i2=100000} {i1+i2} 9.22337203685488e+18 - test_expr expr-1.203\ + test_realnum_expr expr-1.203\ {i1=9223372036854775807, i2=0} {i1+i2} 9223372036854775807 - test_expr expr-1.204\ + test_realnum_expr expr-1.204\ {i1=9223372036854775807, i2=1} {i1+i2} 9.22337203685478e+18 - test_expr expr-1.205\ + test_realnum_expr expr-1.205\ {i2=9223372036854775806, i1=1} {i1+i2} 9223372036854775807 - test_expr expr-1.206\ + test_realnum_expr expr-1.206\ {i2=9223372036854775806, i1=2} {i1+i2} 9.22337203685478e+18 - test_expr expr-1.207\ + test_realnum_expr expr-1.207\ {i2=9223372036854775806, i1=100000} {i1+i2} 9.22337203685488e+18 - test_expr expr-1.208\ + test_realnum_expr expr-1.208\ {i2=9223372036854775807, i1=0} {i1+i2} 9223372036854775807 - test_expr expr-1.209\ + test_realnum_expr expr-1.209\ {i2=9223372036854775807, i1=1} {i1+i2} 9.22337203685478e+18 - test_expr expr-1.210\ + test_realnum_expr expr-1.210\ {i1=-9223372036854775807, i2=-1} {i1+i2} -9223372036854775808 - test_expr expr-1.211\ + test_realnum_expr expr-1.211\ {i1=-9223372036854775807, i2=-2} {i1+i2} -9.22337203685478e+18 - test_expr expr-1.212\ + test_realnum_expr expr-1.212\ {i1=-9223372036854775807, i2=-100000} {i1+i2} -9.22337203685488e+18 - test_expr expr-1.213\ + test_realnum_expr expr-1.213\ {i1=-9223372036854775808, i2=0} {i1+i2} -9223372036854775808 - test_expr expr-1.214\ + test_realnum_expr expr-1.214\ {i1=-9223372036854775808, i2=-1} {i1+i2} -9.22337203685478e+18 - test_expr expr-1.215\ + test_realnum_expr expr-1.215\ {i2=-9223372036854775807, i1=-1} {i1+i2} -9223372036854775808 - test_expr expr-1.216\ + test_realnum_expr expr-1.216\ {i2=-9223372036854775807, i1=-2} {i1+i2} -9.22337203685478e+18 - test_expr expr-1.217\ + test_realnum_expr expr-1.217\ {i2=-9223372036854775807, i1=-100000} {i1+i2} -9.22337203685488e+18 - test_expr expr-1.218\ + test_realnum_expr expr-1.218\ {i2=-9223372036854775808, i1=0} {i1+i2} -9223372036854775808 - test_expr expr-1.219\ + test_realnum_expr expr-1.219\ {i2=-9223372036854775808, i1=-1} {i1+i2} -9.22337203685478e+18 - test_expr expr-1.220\ + test_realnum_expr expr-1.220\ {i1=9223372036854775806, i2=-1} {i1-i2} 9223372036854775807 - test_expr expr-1.221\ + test_realnum_expr expr-1.221\ {i1=9223372036854775806, i2=-2} {i1-i2} 9.22337203685478e+18 - test_expr expr-1.222\ + test_realnum_expr expr-1.222\ {i1=9223372036854775806, i2=-100000} {i1-i2} 9.22337203685488e+18 - test_expr expr-1.223\ + test_realnum_expr expr-1.223\ {i1=9223372036854775807, i2=0} {i1-i2} 9223372036854775807 - test_expr expr-1.224\ + test_realnum_expr expr-1.224\ {i1=9223372036854775807, i2=-1} {i1-i2} 9.22337203685478e+18 - test_expr expr-1.225\ + test_realnum_expr expr-1.225\ {i2=-9223372036854775806, i1=1} {i1-i2} 9223372036854775807 - test_expr expr-1.226\ + test_realnum_expr expr-1.226\ {i2=-9223372036854775806, i1=2} {i1-i2} 9.22337203685478e+18 - test_expr expr-1.227\ + test_realnum_expr expr-1.227\ {i2=-9223372036854775806, i1=100000} {i1-i2} 9.22337203685488e+18 - test_expr expr-1.228\ + test_realnum_expr expr-1.228\ {i2=-9223372036854775807, i1=0} {i1-i2} 9223372036854775807 - test_expr expr-1.229\ + test_realnum_expr expr-1.229\ {i2=-9223372036854775807, i1=1} {i1-i2} 9.22337203685478e+18 - test_expr expr-1.230\ + test_realnum_expr expr-1.230\ {i1=-9223372036854775807, i2=1} {i1-i2} -9223372036854775808 - test_expr expr-1.231\ + test_realnum_expr expr-1.231\ {i1=-9223372036854775807, i2=2} {i1-i2} -9.22337203685478e+18 - test_expr expr-1.232\ + test_realnum_expr expr-1.232\ {i1=-9223372036854775807, i2=100000} {i1-i2} -9.22337203685488e+18 - test_expr expr-1.233\ + test_realnum_expr expr-1.233\ {i1=-9223372036854775808, i2=0} {i1-i2} -9223372036854775808 - test_expr expr-1.234\ + test_realnum_expr expr-1.234\ {i1=-9223372036854775808, i2=1} {i1-i2} -9.22337203685478e+18 - test_expr expr-1.235\ + test_realnum_expr expr-1.235\ {i2=9223372036854775807, i1=-1} {i1-i2} -9223372036854775808 - test_expr expr-1.236\ + test_realnum_expr expr-1.236\ {i2=9223372036854775807, i1=-2} {i1-i2} -9.22337203685478e+18 - test_expr expr-1.237\ + test_realnum_expr expr-1.237\ {i2=9223372036854775807, i1=-100000} {i1-i2} -9.22337203685488e+18 - test_expr expr-1.238\ + test_realnum_expr expr-1.238\ {i2=9223372036854775807, i1=0} {i1-i2} -9223372036854775807 - test_expr expr-1.239\ + test_realnum_expr expr-1.239\ {i2=9223372036854775807, i1=-1} {i1-i2} -9223372036854775808 - test_expr expr-1.250\ + test_realnum_expr expr-1.250\ {i1=4294967296, i2=2147483648} {i1*i2} 9.22337203685478e+18 - test_expr expr-1.251\ + test_realnum_expr expr-1.251\ {i1=4294967296, i2=2147483647} {i1*i2} 9223372032559808512 - test_expr expr-1.252\ + test_realnum_expr expr-1.252\ {i1=-4294967296, i2=2147483648} {i1*i2} -9223372036854775808 - test_expr expr-1.253\ + test_realnum_expr expr-1.253\ {i1=-4294967296, i2=2147483647} {i1*i2} -9223372032559808512 - test_expr expr-1.254\ + test_realnum_expr expr-1.254\ {i1=4294967296, i2=-2147483648} {i1*i2} -9223372036854775808 - test_expr expr-1.255\ + test_realnum_expr expr-1.255\ {i1=4294967296, i2=-2147483647} {i1*i2} -9223372032559808512 - test_expr expr-1.256\ + test_realnum_expr expr-1.256\ {i1=-4294967296, i2=-2147483648} {i1*i2} 9.22337203685478e+18 - test_expr expr-1.257\ + test_realnum_expr expr-1.257\ {i1=-4294967296, i2=-2147483647} {i1*i2} 9223372032559808512 }} @@ -883,7 +888,7 @@ do_test expr-12.2 { } {1 {near ")": syntax error}} ifcapable floatingpoint { - do_test expr-13.1 { + do_realnum_test expr-13.1 { execsql { SELECT 12345678901234567890; } @@ -908,12 +913,12 @@ if {[working_64bit_int]} { # If the value is too large, use String->Float conversion. # ifcapable floatingpoint { - do_test expr-13.4 { + do_realnum_test expr-13.4 { execsql { SELECT 0+'9223372036854775808' } } {9.22337203685478e+18} - do_test expr-13.5 { + do_realnum_test expr-13.5 { execsql { SELECT '9223372036854775808'+0 } @@ -923,12 +928,12 @@ ifcapable floatingpoint { # Use String->float conversion if the value is explicitly a floating # point value. # -do_test expr-13.6 { +do_realnum_test expr-13.6 { execsql { SELECT 0+'9223372036854775807.0' } } {9.22337203685478e+18} -do_test expr-13.7 { +do_realnum_test expr-13.7 { execsql { SELECT '9223372036854775807.0'+0 } diff --git a/test/fkey3.test b/test/fkey3.test index 88b5aad3..6a11f888 100644 --- a/test/fkey3.test +++ b/test/fkey3.test @@ -21,6 +21,8 @@ ifcapable {!foreignkey||!trigger} { return } +set testprefix fkey3 + # Create a table and some data to work with. # do_test fkey3-1.1 { @@ -77,4 +79,108 @@ do_test fkey3-2.1 { } } {1 100 1 101 2 100 2 101} + +#------------------------------------------------------------------------- +# The following tests - fkey-3.* - test some edge cases to do with +# inserting rows into tables that have foreign keys where the parent +# table is the same as the child table. Especially cases where the +# new row being inserted matches itself. +# +do_execsql_test 3.1.1 { + CREATE TABLE t3(a, b, c, d, + UNIQUE(a, b), + FOREIGN KEY(c, d) REFERENCES t3(a, b) + ); + INSERT INTO t3 VALUES(1, 2, 1, 2); +} {} +do_catchsql_test 3.1.2 { + INSERT INTO t3 VALUES(NULL, 2, 5, 2); +} {1 {foreign key constraint failed}} +do_catchsql_test 3.1.3 { + INSERT INTO t3 VALUES(NULL, 3, 5, 2); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.2.1 { + CREATE TABLE t4(a UNIQUE, b REFERENCES t4(a)); +} +do_catchsql_test 3.2.2 { + INSERT INTO t4 VALUES(NULL, 1); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY, b REFERENCES t5(a)); + INSERT INTO t5 VALUES(NULL, 1); +} {} +do_catchsql_test 3.3.2 { + INSERT INTO t5 VALUES(NULL, 3); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.4.1 { + CREATE TABLE t6(a INTEGER PRIMARY KEY, b, c, d, + FOREIGN KEY(c, d) REFERENCES t6(a, b) + ); + CREATE UNIQUE INDEX t6i ON t6(b, a); +} +do_execsql_test 3.4.2 { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {} +do_execsql_test 3.4.3 { INSERT INTO t6 VALUES(2, 'a', 2, 'a'); } {} +do_execsql_test 3.4.4 { INSERT INTO t6 VALUES(NULL, 'a', 1, 'a'); } {} +do_execsql_test 3.4.5 { INSERT INTO t6 VALUES(5, 'a', 2, 'a'); } {} +do_catchsql_test 3.4.6 { + INSERT INTO t6 VALUES(NULL, 'a', 65, 'a'); +} {1 {foreign key constraint failed}} + +do_execsql_test 3.4.7 { + INSERT INTO t6 VALUES(100, 'one', 100, 'one'); + DELETE FROM t6 WHERE a = 100; +} +do_execsql_test 3.4.8 { + INSERT INTO t6 VALUES(100, 'one', 100, 'one'); + UPDATE t6 SET c = 1, d = 'a' WHERE a = 100; + DELETE FROM t6 WHERE a = 100; +} + +do_execsql_test 3.5.1 { + CREATE TABLE t7(a, b, c, d INTEGER PRIMARY KEY, + FOREIGN KEY(c, d) REFERENCES t7(a, b) + ); + CREATE UNIQUE INDEX t7i ON t7(a, b); +} +do_execsql_test 3.5.2 { INSERT INTO t7 VALUES('x', 1, 'x', NULL) } {} +do_execsql_test 3.5.3 { INSERT INTO t7 VALUES('x', 2, 'x', 2) } {} +do_catchsql_test 3.5.4 { + INSERT INTO t7 VALUES('x', 450, 'x', NULL); +} {1 {foreign key constraint failed}} +do_catchsql_test 3.5.5 { + INSERT INTO t7 VALUES('x', 450, 'x', 451); +} {1 {foreign key constraint failed}} + + +do_execsql_test 3.6.1 { + CREATE TABLE t8(a, b, c, d, e, FOREIGN KEY(c, d) REFERENCES t8(a, b)); + CREATE UNIQUE INDEX t8i1 ON t8(a, b); + CREATE UNIQUE INDEX t8i2 ON t8(c); + INSERT INTO t8 VALUES(1, 1, 1, 1, 1); +} +do_catchsql_test 3.6.2 { + UPDATE t8 SET d = 2; +} {1 {foreign key constraint failed}} +do_execsql_test 3.6.3 { UPDATE t8 SET d = 1; } +do_execsql_test 3.6.4 { UPDATE t8 SET e = 2; } + +do_catchsql_test 3.6.5 { + CREATE TABLE TestTable ( + id INTEGER PRIMARY KEY, + name text, + source_id integer not null, + parent_id integer, + + foreign key(source_id, parent_id) references TestTable(source_id, id) + ); + CREATE UNIQUE INDEX testindex on TestTable(source_id, id); + PRAGMA foreign_keys=1; + INSERT INTO TestTable VALUES (1, 'parent', 1, null); + INSERT INTO TestTable VALUES (2, 'child', 1, 1); + UPDATE TestTable SET parent_id=1000 where id=2; +} {1 {foreign key constraint failed}} + finish_test diff --git a/test/fts3atoken.test b/test/fts3atoken.test index cf9574e8..554259d0 100644 --- a/test/fts3atoken.test +++ b/test/fts3atoken.test @@ -24,6 +24,8 @@ ifcapable !fts3 { return } +set ::testprefix fts3token + proc escape_string {str} { set out "" foreach char [split $str ""] { @@ -165,10 +167,21 @@ ifcapable icu { do_icu_test fts3token-4.6 MiddleOfTheOcean $input $output do_icu_test fts3token-4.7 th_TH $input $output do_icu_test fts3token-4.8 en_US $input $output + + do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); + insert into x1 (name) values (NULL); + insert into x1 (name) values (NULL); + delete from x1; + } } + do_test fts3token-internal { execsql { SELECT fts3_tokenizer_internal_test() } } {ok} + finish_test + + diff --git a/test/fts3auto.test b/test/fts3auto.test new file mode 100644 index 00000000..484fd6ac --- /dev/null +++ b/test/fts3auto.test @@ -0,0 +1,658 @@ +# 2011 June 10 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set testprefix fts3auto +set sfep $sqlite_fts3_enable_parentheses +set sqlite_fts3_enable_parentheses 1 + +#-------------------------------------------------------------------------- +# Start of Tcl infrastructure used by tests. The entry points are: +# +# do_fts3query_test +# fts3_make_deferrable +# fts3_zero_long_segments +# + +# +# do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR +# +# This proc runs several test cases on FTS3/4 table $TABLE using match +# expression $MATCHEXPR. All documents in $TABLE must be formatted so that +# they can be "tokenized" using the Tcl list commands (llength, lindex etc.). +# The name and column names used by $TABLE must not require any quoting or +# escaping when used in SQL statements. +# +# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, +# except that the ":token" syntax is not supported. Tcl list +# commands are used to tokenize the expression. Any parenthesis must appear +# either as separate list elements, or as the first (for opening) or last +# (for closing) character of a list element. i.e. the expression "(a OR b)c" +# will not be parsed correctly, but "( a OR b) c" will. +# +# Available OPTIONS are: +# +# -deferred TOKENLIST +# +# If the "deferred" option is supplied, it is passed a list of tokens that +# are deferred by FTS and result in the relevant matchinfo() stats being an +# approximation. +# +set sqlite_fts3_enable_parentheses 1 +proc do_fts3query_test {tn args} { + + set nArg [llength $args] + if {$nArg < 2 || ($nArg % 2)} { + set cmd do_fts3query_test + error "wrong # args: should be \"$cmd ?-deferred LIST? TABLE MATCHEXPR\"" + } + set tbl [lindex $args [expr $nArg-2]] + set match [lindex $args [expr $nArg-1]] + set deferred [list] + + foreach {k v} [lrange $args 0 [expr $nArg-3]] { + switch -- $k { + -deferred { + set deferred $v + } + default { + error "bad option \"$k\": must be -deferred" + } + } + } + + get_near_results $tbl $match $deferred aMatchinfo + + set matchinfo_asc [list] + foreach docid [lsort -integer -incr [array names aMatchinfo]] { + lappend matchinfo_asc $docid $aMatchinfo($docid) + } + set matchinfo_desc [list] + foreach docid [lsort -integer -decr [array names aMatchinfo]] { + lappend matchinfo_desc $docid $aMatchinfo($docid) + } + + set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" + + do_execsql_test $tn$title.1 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC + " [lsort -integer -incr [array names aMatchinfo]] + + do_execsql_test $tn$title.2 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC + " [lsort -integer -decr [array names aMatchinfo]] + + do_execsql_test $tn$title.3 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid DESC + " $matchinfo_desc + + do_execsql_test $tn$title.4 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid ASC + " $matchinfo_asc +} + +# fts3_make_deferrable TABLE TOKEN +# +proc fts3_make_deferrable {tbl token} { + + set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] + set name [sqlite3_column_name $stmt 0] + sqlite3_finalize $stmt + + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl ($name) VALUES(\$doc)" + } + execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT + + return [expr $nRow*$pgsz] +} + +# fts3_zero_long_segments TABLE ?LIMIT? +# +proc fts3_zero_long_segments {tbl limit} { + execsql " + UPDATE ${tbl}_segments + SET block = zeroblob(length(block)) + WHERE length(block)>$limit + " + return [db changes] +} + + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +proc fix_phrase_expr {cols expr colfiltervar} { + upvar $colfiltervar iColFilter + + set out [list] + foreach t $expr { + if {[string match *:* $t]} { + set col [lindex [split $t :] 0] + set t [lindex [split $t :] 1] + set iCol [lsearch $cols $col] + if {$iCol<0} { error "unknown column: $col" } + if {$iColFilter < 0} { + set iColFilter $iCol + } elseif {$iColFilter != $iCol} { + set iColFilter [llength $cols] + } + } + lappend out $t + } + + return $out +} + +proc fix_near_expr {cols expr colfiltervar} { + upvar $colfiltervar iColFilter + + set iColFilter -1 + + set out [list] + lappend out [fix_phrase_expr $cols [lindex $expr 0] iColFilter] + foreach {a b} [lrange $expr 1 end] { + if {[string match -nocase near $a]} { set a 10 } + if {[string match -nocase near/* $a]} { set a [string range $a 5 end] } + lappend out $a + lappend out [fix_phrase_expr $cols $b iColFilter] + } + return $out +} + +proc get_single_near_results {tbl expr deferred arrayvar nullvar} { + upvar $arrayvar aMatchinfo + upvar $nullvar nullentry + catch {array unset aMatchinfo} + + set cols [list] + set miss [list] + db eval "PRAGMA table_info($tbl)" A { lappend cols $A(name) ; lappend miss 0 } + set expr [fix_near_expr $cols $expr iColFilter] + + # Calculate the expected results using [fts3_near_match]. The following + # loop populates the "hits" and "counts" arrays as follows: + # + # 1. For each document in the table that matches the NEAR expression, + # hits($docid) is set to 1. The set of docids that match the expression + # can therefore be found using [array names hits]. + # + # 2. For each column of each document in the table, counts($docid,$iCol) + # is set to the -phrasecountvar output. + # + set res [list] + catch { array unset hits } + db eval "SELECT docid, * FROM $tbl" d { + set iCol 0 + foreach col [lrange $d(*) 1 end] { + set docid $d(docid) + if {$iColFilter<0 || $iCol==$iColFilter} { + set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)] + if {$hit} { set hits($docid) 1 } + } else { + set counts($docid,$iCol) $miss + } + incr iCol + } + } + set nPhrase [expr ([llength $expr]+1)/2] + set nCol $iCol + + # This block populates the nHit and nDoc arrays. For each phrase/column + # in the query/table, array elements are set as follows: + # + # nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in + # column $iCol. + # + # nDoc($iPhrase,$iCol) - Number of documents with at least one hit for + # phrase $iPhrase in column $iCol. + # + for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit($iPhrase,$iCol) 0 + set nDoc($iPhrase,$iCol) 0 + } + } + foreach key [array names counts] { + set iCol [lindex [split $key ,] 1] + set iPhrase 0 + foreach c $counts($key) { + if {$c>0} { incr nDoc($iPhrase,$iCol) 1 } + incr nHit($iPhrase,$iCol) $c + incr iPhrase + } + } + + if {[llength $deferred] && [llength $expr]==1} { + set phrase [lindex $expr 0] + set rewritten [list] + set partial 0 + foreach tok $phrase { + if {[lsearch $deferred $tok]>=0} { + lappend rewritten * + } else { + lappend rewritten $tok + set partial 1 + } + } + if {$partial==0} { + set tblsize [db one "SELECT count(*) FROM $tbl"] + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit(0,$iCol) $tblsize + set nDoc(0,$iCol) $tblsize + } + } elseif {$rewritten != $phrase} { + while {[lindex $rewritten end] == "*"} { + set rewritten [lrange $rewritten 0 end-1] + } + while {[lindex $rewritten 0] == "*"} { + set rewritten [lrange $rewritten 1 end] + } + get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry + foreach docid [array names hits] { + set aMatchinfo($docid) $aRewrite($docid) + } + return + } + } + + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to + # contain the output of matchinfo('x') for the document. + # + foreach docid [array names hits] { + set mi [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend mi [lindex $counts($docid,$iCol) $iPhrase] + lappend mi $nHit($iPhrase,$iCol) + lappend mi $nDoc($iPhrase,$iCol) + } + } + set aMatchinfo($docid) $mi + } + + # Set up the nullentry output. + # + set nullentry [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol) + } + } +} + + +proc matching_brackets {expr} { + if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { + return 0 + } + + set iBracket 1 + set nExpr [string length $expr] + for {set i 1} {$iBracket && $i < $nExpr} {incr i} { + set c [string range $expr $i $i] + if {$c == "("} {incr iBracket} + if {$c == ")"} {incr iBracket -1} + } + + return [expr ($iBracket==0 && $i==$nExpr)] +} + +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} { + upvar $arrayvar aMatchinfo + if {$nullvar != ""} { upvar $nullvar nullentry } + + set expr [string trim $expr] + while { [matching_brackets $expr] } { + set expr [string trim [string range $expr 1 end-1]] + } + + set prec(NOT) 1 + set prec(AND) 2 + set prec(OR) 3 + + set currentprec 0 + set iBracket 0 + set expr_length [llength $expr] + for {set i 0} {$i < $expr_length} {incr i} { + set op [lindex $expr $i] + if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } { + set opidx $i + set currentprec $prec($op) + } else { + for {set j 0} {$j < [string length $op]} {incr j} { + set c [string range $op $j $j] + if {$c == "("} { incr iBracket +1 } + if {$c == ")"} { incr iBracket -1 } + } + } + } + if {$iBracket!=0} { error "mismatched brackets in: $expr" } + + if {[info exists opidx]==0} { + get_single_near_results $tbl $expr $deferred aMatchinfo nullentry + } else { + set eLeft [lrange $expr 0 [expr $opidx-1]] + set eRight [lrange $expr [expr $opidx+1] end] + + get_near_results $tbl $eLeft $deferred aLeft nullleft + get_near_results $tbl $eRight $deferred aRight nullright + + switch -- [lindex $expr $opidx] { + "NOT" { + foreach hit [array names aLeft] { + if {0==[info exists aRight($hit)]} { + set aMatchinfo($hit) $aLeft($hit) + } + } + set nullentry $nullleft + } + + "AND" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + } + } + set nullentry [concat $nullleft $nullright] + } + + "OR" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + unset aRight($hit) + } else { + set aMatchinfo($hit) [concat $aLeft($hit) $nullright] + } + } + foreach hit [array names aRight] { + set aMatchinfo($hit) [concat $nullleft $aRight($hit)] + } + + set nullentry [concat $nullleft $nullright] + } + } + } +} + + +# End of test procs. Actual tests are below this line. +#-------------------------------------------------------------------------- + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-1.* - focus on testing the Tcl +# command [fts3_near_match], which is used by other tests in this file. +# +proc test_fts3_near_match {tn doc expr res} { + fts3_near_match $doc $expr -phrasecountvar p + uplevel do_test [list $tn] [list [list set {} $p]] [list $res] +} + +test_fts3_near_match 1.1.1 {a b c a b} a {2} +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} +test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} +test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} +test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} + +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} + +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} + +#-------------------------------------------------------------------------- +# Test cases fts3auto-2.* run some simple tests using the +# [do_fts3query_test] proc. +# +foreach {tn create} { + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC)" + 3 "fts4(a, b, order=ASC)" + 4 "fts4(a, b, prefix=1)" + 5 "fts4(a, b, order=DESC, prefix=1)" + 6 "fts4(a, b, order=ASC, prefix=1)" +} { + do_test 2.$tn.1 { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + for {set i 0} {$i<32} {incr i} { + set doc [list] + if {$i&0x01} {lappend doc one} + if {$i&0x02} {lappend doc two} + if {$i&0x04} {lappend doc three} + if {$i&0x08} {lappend doc four} + if {$i&0x10} {lappend doc five} + execsql { INSERT INTO t1 VALUES($doc, null) } + } + } {} + + foreach {tn2 expr} { + 1 {one} + 2 {one NEAR/1 five} + 3 {t*} + 4 {t* NEAR/0 five} + 5 {o* NEAR/1 f*} + 6 {one NEAR five NEAR two NEAR four NEAR three} + 7 {one NEAR xyz} + 8 {one OR two} + 9 {one AND two} + 10 {one NOT two} + 11 {one AND two OR three} + 12 {three OR one AND two} + 13 {(three OR one) AND two} + 14 {(three OR one) AND two NOT (five NOT four)} + 15 {"one two"} + 16 {"one two" NOT "three four"} + } { + do_fts3query_test 2.$tn.2.$tn2 t1 $expr + } +} + +#-------------------------------------------------------------------------- +# Some test cases involving deferred tokens. +# + +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + do_execsql_test 3.$tn.1 { + INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k'); + INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a'); + INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b'); + INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c'); + INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d'); + INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e'); + INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k'); + INSERT INTO t1(docid, x) VALUES(5, 'a d g j'); + INSERT INTO t1(docid, x) VALUES(6, 'c a b'); + } + + set limit [fts3_make_deferrable t1 c] + + do_fts3query_test 3.$tn.2.1 t1 {a OR c} + + do_test 3.$tn.3 { + fts3_zero_long_segments t1 $limit + } {1} + + foreach {tn2 expr def} { + 1 {a NEAR c} {} + 2 {a AND c} c + 3 {"a c"} c + 4 {"c a"} c + 5 {"a c" NEAR/1 g} {} + 6 {"a c" NEAR/0 g} {} + } { + do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr + } +} + +#-------------------------------------------------------------------------- +# +foreach {tn create} { + 1 "fts4(x, y)" + 2 "fts4(x, y, order=DESC)" + 3 "fts4(x, y, order=DESC, prefix=2)" +} { + + execsql [subst { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING $create; + INSERT INTO t1 VALUES('one two five four five', ''); + INSERT INTO t1 VALUES('', 'one two five four five'); + INSERT INTO t1 VALUES('one two', 'five four five'); + }] + + do_fts3query_test 4.$tn.1.1 t1 {one AND five} + do_fts3query_test 4.$tn.1.2 t1 {one NEAR five} + do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five} + + do_test 4.$tn.2 { + set limit [fts3_make_deferrable t1 five] + execsql { INSERT INTO t1(t1) VALUES('optimize') } + expr {[fts3_zero_long_segments t1 $limit]>0} + } {1} + + do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} + do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} + do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} + + do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} + + do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} + do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} + do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} + do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} + do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} +} + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-5.* - focus on using prefix indexes. +# +set chunkconfig [fts3_configure_incr_load 1 1] +foreach {tn create pending} { + 1 "fts4(a, b)" 1 + 2 "fts4(a, b, order=ASC, prefix=1)" 1 + 3 "fts4(a, b, order=ASC, prefix=1,3)" 0 + 4 "fts4(a, b, order=DESC, prefix=2,4)" 0 + 5 "fts4(a, b, order=DESC, prefix=1)" 0 + 6 "fts4(a, b, order=ASC, prefix=1,3)" 0 +} { + + execsql [subst { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + if {$pending} {execsql BEGIN} + + foreach {a b} { + "the song of songs which is solomons" + "let him kiss me with the kisses of his mouth for thy love is better than wine" + "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee" + "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee" + "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon" + "look not upon me because i am black because the sun hath looked upon me my mothers children were angry with me they made me the keeper of the vineyards but mine own vineyard have i not kept" + "tell me o thou whom my soul loveth where thou feedest where thou makest thy flock to rest at noon for why should i be as one that turneth aside by the flocks of thy companions?" + "if thou know not o thou fairest among women go thy way forth by the footsteps of the flock and feed thy kids beside the shepherds tents" + "i have compared thee o my love to a company of horses in pharaohs chariots" + "thy cheeks are comely with rows of jewels thy neck with chains of gold" + "we will make thee borders of gold with studs of silver" + "while the king sitteth at his table my spikenard sendeth forth the smell thereof" + "a bundle of myrrh is my wellbeloved unto me he shall lie all night betwixt my breasts" + "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" + "behold thou art fair my love behold thou art fair thou hast doves eyes" + "behold thou art fair my beloved yea pleasant also our bed is green" + "the beams of our house are cedar and our rafters of fir" + } { + execsql {INSERT INTO t1(a, b) VALUES($a, $b)} + } + + + do_fts3query_test 5.$tn.1.1 t1 {s*} + do_fts3query_test 5.$tn.1.2 t1 {so*} + do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} + do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} + do_fts3query_test 5.$tn.1.5 t1 {a*} + do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*} + do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"} + + if {$pending} {execsql COMMIT} +} +eval fts3_configure_incr_load $chunkconfig + +foreach {tn pending create} { + 1 0 "fts4(a, b, c, d)" + 2 1 "fts4(a, b, c, d)" + 3 0 "fts4(a, b, c, d, order=DESC)" + 4 1 "fts4(a, b, c, d, order=DESC)" +} { + execsql [subst { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + + if {$pending} { execsql BEGIN } + + foreach {a b c d} { + "A B C" "D E F" "G H I" "J K L" + "B C D" "E F G" "H I J" "K L A" + "C D E" "F G H" "I J K" "L A B" + "D E F" "G H I" "J K L" "A B C" + "E F G" "H I J" "K L A" "B C D" + "F G H" "I J K" "L A B" "C D E" + } { + execsql { INSERT INTO t1 VALUES($a, $b, $c, $d) } + } + + do_fts3query_test 6.$tn.1 t1 {b:G} + do_fts3query_test 6.$tn.2 t1 {b:G AND c:I} + do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I} + do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C} + do_fts3query_test 6.$tn.5 t1 {a:G OR b:G} + + catchsql { COMMIT } +} + +set sqlite_fts3_enable_parentheses $sfep +finish_test + diff --git a/test/fts3aux1.test b/test/fts3aux1.test index 5359521a..adda5863 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -38,10 +38,10 @@ do_execsql_test 1.2 { six 1 1 three 4 6 two 1 1 } -do_execsql_test 1.3 { - DELETE FROM t1; +do_execsql_test 1.3.1 { DELETE FROM t1; } +do_execsql_test 1.3.2 { SELECT term, documents, occurrences FROM terms WHERE col = '*'; -} {} +} do_execsql_test 1.4 { INSERT INTO t1 VALUES('a b a b a b a'); diff --git a/test/fts3conf.test b/test/fts3conf.test new file mode 100644 index 00000000..ce410277 --- /dev/null +++ b/test/fts3conf.test @@ -0,0 +1,139 @@ +# 2011 April 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3conf + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + + +proc fts3_integrity {tn db tbl} { + + if {[sqlite3_get_autocommit $db]==0} { + error "fts3_integrity does not work with an open transaction" + } + + set sql [db one {SELECT sql FROM sqlite_master WHERE name = $tbl}] + regexp -nocase {[^(]* using (.*)} $sql -> tail + set cols [list] + $db eval "PRAGMA table_info($tbl)" { + lappend cols $name + } + set cols [join [concat docid $cols] ,] + + $db eval [subst { + CREATE VIRTUAL TABLE fts3check USING fts4term($tbl); + CREATE VIRTUAL TABLE temp.fts3check2 USING $tail; + INSERT INTO temp.fts3check2($cols) SELECT docid, * FROM $tbl; + CREATE VIRTUAL TABLE temp.fts3check3 USING fts4term(fts3check2); + }] + + set m1 [$db one {SELECT md5sum(term, docid, col, pos) FROM fts3check}] + set m2 [$db one {SELECT md5sum(term, docid, col, pos) FROM fts3check3}] + + $db eval { + DROP TABLE fts3check; + DROP TABLE temp.fts3check2; + DROP TABLE temp.fts3check3; + } + + uplevel [list do_test $tn [list set {} $m1] $m2] +} + +do_execsql_test 1.0.1 { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1(rowid, x) VALUES(1, 'a b c d'); + INSERT INTO t1(rowid, x) VALUES(2, 'e f g h'); + + CREATE TABLE source(a, b); + INSERT INTO source VALUES(4, 'z'); + INSERT INTO source VALUES(2, 'y'); +} +db_save_and_close + +set T1 "INTO t1(rowid, x) VALUES(1, 'x')" +set T2 "INTO t1(rowid, x) SELECT * FROM source" + +set T3 "t1 SET docid = 2 WHERE docid = 1" +set T4 "t1 SET docid = CASE WHEN docid = 1 THEN 4 ELSE 3 END WHERE docid <=2" + +foreach {tn sql uses constraint data} [subst { + 1 "INSERT OR ROLLBACK $T1" 0 1 {{a b c d} {e f g h}} + 2 "INSERT OR ABORT $T1" 0 1 {{a b c d} {e f g h} {i j k l}} + 3 "INSERT OR FAIL $T1" 0 1 {{a b c d} {e f g h} {i j k l}} + 4 "INSERT OR IGNORE $T1" 0 0 {{a b c d} {e f g h} {i j k l}} + 5 "INSERT OR REPLACE $T1" 0 0 {x {e f g h} {i j k l}} + + 6 "INSERT OR ROLLBACK $T2" 1 1 {{a b c d} {e f g h}} + 7 "INSERT OR ABORT $T2" 1 1 {{a b c d} {e f g h} {i j k l}} + 8 "INSERT OR FAIL $T2" 1 1 {{a b c d} {e f g h} {i j k l} z} + 9 "INSERT OR IGNORE $T2" 1 0 {{a b c d} {e f g h} {i j k l} z} + 10 "INSERT OR REPLACE $T2" 1 0 {{a b c d} y {i j k l} z} + + 11 "UPDATE OR ROLLBACK $T3" 1 1 {{a b c d} {e f g h}} + 12 "UPDATE OR ABORT $T3" 1 1 {{a b c d} {e f g h} {i j k l}} + 13 "UPDATE OR FAIL $T3" 1 1 {{a b c d} {e f g h} {i j k l}} + 14 "UPDATE OR IGNORE $T3" 1 0 {{a b c d} {e f g h} {i j k l}} + 15 "UPDATE OR REPLACE $T3" 1 0 {{a b c d} {i j k l}} + + 16 "UPDATE OR ROLLBACK $T4" 1 1 {{a b c d} {e f g h}} + 17 "UPDATE OR ABORT $T4" 1 1 {{a b c d} {e f g h} {i j k l}} + 18 "UPDATE OR FAIL $T4" 1 1 {{e f g h} {i j k l} {a b c d}} + 19 "UPDATE OR IGNORE $T4" 1 0 {{e f g h} {i j k l} {a b c d}} + 20 "UPDATE OR REPLACE $T4" 1 0 {{e f g h} {a b c d}} +}] { + db_restore_and_reopen + execsql { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(3, 'i j k l'); + } + set R(0) {0 {}} + set R(1) {1 {constraint failed}} + do_catchsql_test 1.$tn.1 $sql $R($constraint) + do_catchsql_test 1.$tn.2 { SELECT * FROM t1 } [list 0 $data] + catchsql COMMIT + + fts3_integrity 1.$tn.3 db t1 + + do_test 1.$tn.4 [list sql_uses_stmt db $sql] $uses +} + +do_execsql_test 2.1.1 { + DELETE FROM t1; + BEGIN; + INSERT INTO t1 VALUES('a b c'); + SAVEPOINT a; + INSERT INTO t1 VALUES('x y z'); + ROLLBACK TO a; + COMMIT; +} +fts3_integrity 2.1.2 db t1 + +do_catchsql_test 2.2.1 { + DELETE FROM t1; + BEGIN; + INSERT INTO t1(docid, x) VALUES(0, 'a b c'); + INSERT INTO t1(docid, x) VALUES(1, 'a b c'); + REPLACE INTO t1(docid, x) VALUES('zero', 'd e f'); +} {1 {datatype mismatch}} +do_execsql_test 2.2.2 { COMMIT } +do_execsql_test 2.2.3 { SELECT * FROM t1 } {{a b c} {a b c}} +fts3_integrity 2.2.4 db t1 + +finish_test diff --git a/test/fts3corrupt.test b/test/fts3corrupt.test index b8b45c39..ea4c9a9d 100644 --- a/test/fts3corrupt.test +++ b/test/fts3corrupt.test @@ -40,6 +40,7 @@ do_test fts3corrupt-1.2 { do_catchsql_test 1.3 { INSERT INTO t1 VALUES('world'); } {1 {database disk image is malformed}} +do_test 1.3.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB do_execsql_test 1.4 { DROP TABLE t1; } @@ -69,6 +70,7 @@ do_test fts3corrupt-2.1 { do_catchsql_test 2.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'hello' } {1 {database disk image is malformed}} +do_test 2.2.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB do_execsql_test 3.0 { DROP TABLE t1; @@ -86,6 +88,7 @@ do_test fts3corrupt-3.1 { do_catchsql_test 3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'world' } {1 {database disk image is malformed}} +do_test 3.2.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB do_execsql_test 4.0 { @@ -111,6 +114,7 @@ do_catchsql_test 4.2 { UPDATE t1_segdir SET root = X'FFFFFFFFFFFFFFFF'; SELECT rowid FROM t1 WHERE t1 MATCH 'world'; } {1 {database disk image is malformed}} +do_test 4.2.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB set blob [binary format cca*cca*cca*cca*cca*cca*cca*cca*cca*cca*a* \ 22 120 [string repeat a 120] \ @@ -130,6 +134,7 @@ do_catchsql_test 4.3 { UPDATE t1_segdir SET root = $blob; SELECT rowid FROM t1 WHERE t1 MATCH 'world'; } {1 {database disk image is malformed}} +do_test 4.3.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB # Test a special kind of corruption, where the %_stat table contains # an invalid entry. At one point this could lead to a division-by-zero @@ -152,10 +157,12 @@ do_catchsql_test 5.2 { UPDATE t1_stat SET value = X'0000'; SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*'; } {1 {database disk image is malformed}} +do_test 5.2.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB do_catchsql_test 5.3 { UPDATE t1_stat SET value = NULL; SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*'; } {1 {database disk image is malformed}} +do_test 5.3.1 { sqlite3_extended_errcode db } SQLITE_CORRUPT_VTAB finish_test diff --git a/test/fts3defer.test b/test/fts3defer.test index 1c9056fd..4bc0b0a7 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -20,6 +20,8 @@ ifcapable !fts3 { set sqlite_fts3_enable_parentheses 1 +set fts3_simple_deferred_tokens_only 1 + set ::testprefix fts3defer #-------------------------------------------------------------------------- @@ -257,7 +259,6 @@ foreach {tn setup} { do_select_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' } {100} -if {$tn==3} breakpoint do_select_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' } {7 70 98} @@ -282,13 +283,16 @@ if {$tn==3} breakpoint do_select_test 1.10 { SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' } {10 13 17 31 35 51 58 88 89 90 93 100} - do_select_test 1.11 { - SELECT rowid FROM t1 - WHERE t1 MATCH '( - zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR - zk OR zkhdvkw OR zm OR zsmhnf - ) vgsld' - } {10 13 17 31 35 51 58 88 89 90 93 100} + + if { $fts3_simple_deferred_tokens_only==0 } { + do_select_test 1.11 { + SELECT rowid FROM t1 + WHERE t1 MATCH '( + zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR + zk OR zkhdvkw OR zm OR zsmhnf + ) vgsld' + } {10 13 17 31 35 51 58 88 89 90 93 100} + } do_select_test 2.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' @@ -364,6 +368,7 @@ if {$tn==3} breakpoint foreach DO_MALLOC_TEST $dmt_modes { # Phrase search. + # do_select_test 5.$DO_MALLOC_TEST.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' } {8 15 36 64 67 72} @@ -416,9 +421,11 @@ if {$tn==3} breakpoint do_select_test 6.2.2 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' } {15 26 92 96} - do_select_test 6.2.3 { - SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' - } {8 15 26 92 96} + if {$fts3_simple_deferred_tokens_only==0} { + do_select_test 6.2.3 { + SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' + } {8 15 26 92 96} + } } set testprefix fts3defer diff --git a/test/fts3defer2.test b/test/fts3defer2.test index 84418033..92a4491e 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -48,6 +48,10 @@ do_execsql_test 1.1.4 { UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } {2} +do_execsql_test 1.2.0 { + SELECT content FROM t1 WHERE t1 MATCH 'f (e a)'; +} {{a b c d e f a x y}} + do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} @@ -58,7 +62,7 @@ do_execsql_test 1.2.2 { } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9] ] do_execsql_test 1.2.3 { @@ -67,7 +71,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } @@ -99,8 +103,14 @@ foreach {tn sql} { [list 2 1 1 54 54 1 3 3 54 372 7] \ ] - set sqlite_fts3_enable_parentheses 1 do_execsql_test 2.2.$tn.2 { + SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z'; + } [list \ + [list 1 2 2 1 54 54] \ + ] + + set sqlite_fts3_enable_parentheses 1 + do_execsql_test 2.2.$tn.3 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)'; } [list \ [list 1 2 2 1 2 2 1 54 54] \ diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 8f194e72..40366b6a 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -68,7 +68,11 @@ do_execsql_test 3.1 { do_execsql_test 3.2 { CREATE VIRTUAL TABLE xx USING FTS4; +} +do_execsql_test 3.3 { SELECT * FROM xx WHERE xx MATCH 'abc'; +} +do_execsql_test 3.4 { SELECT * FROM xx WHERE xx MATCH 'a b c'; } @@ -240,9 +244,13 @@ do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; -do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { - x {{5 8 2 5 5 5} {3 8 2 3 5 5}} - s {2 1} +# It used to be that the second 'a' token would be deferred. That doesn't +# work any longer. +if 0 { + do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 5 5} {3 8 2 3 5 5}} + s {2 1} + } } do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } diff --git a/test/fts3prefix.test b/test/fts3prefix.test new file mode 100644 index 00000000..f5e31f32 --- /dev/null +++ b/test/fts3prefix.test @@ -0,0 +1,203 @@ +# 2011 May 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3prefix + +ifcapable !fts3 { + finish_test + return +} + +# This proc tests that the prefixes index appears to represent the same content +# as the terms index. +# +proc fts3_terms_and_prefixes {db tbl prefixlengths} { + + set iIndex 0 + foreach len $prefixlengths { + incr iIndex + $db eval { + DROP TABLE IF EXISTS fts3check1; + DROP TABLE IF EXISTS fts3check2; + } + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);" + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);" + + $db eval { + DROP TABLE IF EXISTS temp.terms; + DROP TABLE IF EXISTS temp.prefixes; + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; + CREATE INDEX temp.idx ON prefixes(term); + DROP TABLE fts3check1; + DROP TABLE fts3check2; + } + + set nExpect 0 + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { + if {[string length $a(term)]<$len} continue + incr nExpect + set prefix [string range $a(term) 0 [expr $len-1]] + set r [$db one { + SELECT count(*) FROM temp.prefixes WHERE + term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) + }] + if {$r != 1} { + error "$t, $a(docid), $a(col), $a(pos)" + } + } + + set nCount [$db one {SELECT count(*) FROM temp.prefixes}] + if {$nCount != $nExpect} { + error "prefixes.count(*) is $nCount expected $nExpect" + } + + execsql { DROP TABLE temp.prefixes } + execsql { DROP TABLE temp.terms } + + set list [list] + $db eval " + SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree + FROM ${tbl}_segdir GROUP BY tree + " { + lappend list [list $total $tree] + } + + if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } { + error "inconsistent tree structures: $list" + } + } + + return "" +} +proc fts3_tap_test {tn db tbl lens} { + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""] +} + +#------------------------------------------------------------------------- +# Test cases 1.* are a sanity check. They test that the prefixes index is +# being constructed correctly for the simplest possible case. +# +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6'); + + CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1); + CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2); + CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3); + CREATE VIRTUAL TABLE terms USING fts4term(t1); +} +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('sqlite mysql firebird'); +} +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s} +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql} +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite} +do_execsql_test 1.4 { + SELECT term FROM terms; +} {firebird mysql sqlite} + +fts3_tap_test 1.5 db t1 {1 3 6} + +#------------------------------------------------------------------------- +# A slightly more complicated dataset. This test also verifies that DELETE +# operations do not corrupt the prefixes index. +# +do_execsql_test 2.1 { + INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules'); + INSERT INTO t1 VALUES('that allows users to perform full-text searches on'); + INSERT INTO t1 VALUES('a set of documents. The most common (and'); + INSERT INTO t1 VALUES('effective) way to describe full-text searches is'); + INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with'); + INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users'); + INSERT INTO t1 VALUES('input a term, or series of terms, perhaps'); + INSERT INTO t1 VALUES('connected by a binary operator or grouped together'); + INSERT INTO t1 VALUES('into a phrase, and the full-text query system'); + INSERT INTO t1 VALUES('finds the set of documents that best matches those'); + INSERT INTO t1 VALUES('terms considering the operators and groupings the'); + INSERT INTO t1 VALUES('user has specified. This article describes the'); + INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.'); + INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search'); + INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); + INSERT INTO t1 VALUES('these older modules and their use should be'); + INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); + INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); + INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); + INSERT INTO t1 VALUES('of SQLite. '); +} +fts3_tap_test 2.2 db t1 {1 3 6} +do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } +fts3_tap_test 2.4 db t1 {1 3 6} + +do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } +fts3_tap_test 2.6 db t1 {1 3 6} + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3'); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); +} + +fts3_tap_test 3.2 db t2 {1 2 3} +do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} +fts3_tap_test 3.4 db t2 {1 2 3} + + +#------------------------------------------------------------------------- +# Simple tests for reading the prefix-index. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4"); + INSERT INTO t3 VALUES('one two three'); + INSERT INTO t3 VALUES('four five six'); + INSERT INTO t3 VALUES('seven eight nine'); +} +do_execsql_test 4.2 { + SELECT * FROM t3 WHERE t3 MATCH 'f*' +} {{four five six}} +do_execsql_test 4.3 { + SELECT * FROM t3 WHERE t3 MATCH 'four*' +} {{four five six}} +do_execsql_test 4.4 { + SELECT * FROM t3 WHERE t3 MATCH 's*' +} {{four five six} {seven eight nine}} +do_execsql_test 4.5 { + SELECT * FROM t3 WHERE t3 MATCH 'sev*' +} {{seven eight nine}} +do_execsql_test 4.6 { + SELECT * FROM t3 WHERE t3 MATCH 'one*' +} {{one two three}} + +finish_test diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0909cee6..97af5492 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} { # This [proc] is used to test the FTS3 matchinfo() function. # -proc simple_token_matchinfo {zToken} { +proc simple_token_matchinfo {zToken bDesc} { set nDoc(0) 0 set nDoc(1) 0 @@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} { set nHit(1) 0 set nHit(2) 0 + set dir -inc + if {$bDesc} { set dir -dec } foreach key [array names ::t1] { set value $::t1($key) @@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} { } set ret [list] - foreach docid [lsort -integer [array names a]] { + foreach docid [lsort -integer $dir [array names a]] { if { [lindex [lsort -integer $a($docid)] end] } { set matchinfo [list 1 3] foreach i {0 1 2} hit $a($docid) { @@ -262,22 +264,37 @@ proc mit {blob} { return $r } db func mit mit - set sqlite_fts3_enable_parentheses 1 -foreach nodesize {50 500 1000 2000} { +proc do_orderbydocid_test {tn sql res} { + uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] + uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ + [lsort -int -dec $res] + ] +} + +set NUM_TRIALS 100 + +foreach {nodesize order} { + 50 DESC + 50 ASC + 500 ASC + 1000 DESC + 2000 ASC +} { catch { array unset ::t1 } + set testname "$nodesize/$order" # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" + execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" for {set i 0} {$i < 100} {incr i} { insert_row $i } } - for {set iTest 1} {$iTest <= 100} {incr iTest} { + for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { catchsql COMMIT set DO_MALLOC_TEST 0 @@ -286,6 +303,8 @@ foreach nodesize {50 500 1000 2000} { set DO_MALLOC_TEST 1 set nRep 2 } + + set ::testprefix fts3rnd-1.$testname.$iTest # Delete one row, update one row and insert one row. # @@ -307,7 +326,7 @@ foreach nodesize {50 500 1000 2000} { if {0==($iTest%2)} { execsql COMMIT } if {0==($iTest%2)} { - do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok + #do_test 0 { fts3_integrity_check t1 } ok } # Pick 10 terms from the vocabulary. Check that the results of querying @@ -317,9 +336,14 @@ foreach nodesize {50 500 1000 2000} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { + do_select_test 1.$i.asc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term - } [simple_token_matchinfo $term] + ORDER BY docid ASC + } [simple_token_matchinfo $term 0] + do_select_test 1.$i.desc { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term + ORDER BY docid DESC + } [simple_token_matchinfo $term 1] } # This time, use the first two characters of each term as a term prefix @@ -329,7 +353,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set prefix [string range [random_term] 0 end-1] set match "${prefix}*" - do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { + do_orderbydocid_test 2.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $match] } @@ -339,7 +363,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { + do_orderbydocid_test 3.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -349,7 +373,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { + do_orderbydocid_test 4.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -362,17 +386,19 @@ foreach nodesize {50 500 1000 2000} { append query "[string range [random_term] 0 end-1]*" set match "\"$query\"" - do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { + do_orderbydocid_test 5.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $query] } - # A NEAR query with terms as the arguments. + # A NEAR query with terms as the arguments: + # + # ... MATCH '$term1 NEAR $term2' ... # for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] - do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { + do_orderbydocid_test 6.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms 10] } @@ -383,7 +409,7 @@ foreach nodesize {50 500 1000 2000} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] - do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { + do_orderbydocid_test 7.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms $nNear] } @@ -399,7 +425,7 @@ foreach nodesize {50 500 1000 2000} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc [simple_phrase $term1] [simple_phrase $term2]] } @@ -408,9 +434,9 @@ foreach nodesize {50 500 1000 2000} { # Set operations on NEAR queries. # foreach {tn op proc} { - 8 OR setop_or - 9 NOT setop_not - 10 AND setop_and + 11 OR setop_or + 12 NOT setop_not + 13 AND setop_and } { for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term] @@ -418,7 +444,7 @@ foreach nodesize {50 500 1000 2000} { set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc \ [simple_near [list $term1 $term2] 10] \ diff --git a/test/fts3sort.test b/test/fts3sort.test new file mode 100644 index 00000000..ce5b5df7 --- /dev/null +++ b/test/fts3sort.test @@ -0,0 +1,162 @@ +# 2011 May 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + + +proc build_database {nRow param} { + db close + forcedelete test.db + sqlite3 db test.db + + set vocab [list aa ab ac ba bb bc ca cb cc da] + expr srand(0) + + execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)" + for {set i 0} {$i < $nRow} {incr i} { + set v [expr int(rand()*1000000)] + set doc [list] + for {set div 1} {$div < 1000000} {set div [expr $div*10]} { + lappend doc [lindex $vocab [expr ($v/$div) % 10]] + } + execsql { INSERT INTO t1 VALUES($doc) } + } +} + +set testprefix fts3sort + +unset -nocomplain CONTROL +foreach {t param} { + 1 "" + 2 "order=asc" + 3 "order=desc" +} { + + set testprefix fts3sort-1.$t + + set nRow 1000 + do_test 1.0 { + build_database $nRow $param + execsql { SELECT count(*) FROM t1 } + } $nRow + + foreach {tn query} { + 1 "SELECT docid, * FROM t1" + 2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'" + 3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'" + 4 "SELECT docid, quote(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'a*'" + 5 "SELECT docid, quote(matchinfo(t1,'pcnxals')) FROM t1 WHERE t1 MATCH 'b*'" + 6 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a* b* c*'" + 7 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa OR da'" + 8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'" + 9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'" + 10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'" + 11 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'" + 12 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'" + 13 "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'" + 14 "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'" + } { + + unset -nocomplain A B C D + set A_list [list] + set B_list [list] + set C_list [list] + set D_list [list] + + unset -nocomplain X + db eval "$query ORDER BY rowid ASC" X { + set A($X(docid)) [array get X] + lappend A_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY rowid DESC" X { + set B($X(docid)) [array get X] + lappend B_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid ASC" X { + set C($X(docid)) [array get X] + lappend C_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid DESC" X { + set D($X(docid)) [array get X] + lappend D_list $X(docid) + } + + do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list] + do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list] + do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list] + do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list] + + unset -nocomplain DATA + unset -nocomplain X + db eval "$query" X { + set DATA($X(docid)) [array get X] + } + + do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]] + do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]] + do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]] + do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]] + + if {[info exists CONTROL($tn)]} { + do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]] + } else { + set CONTROL($tn) [lsort [array get DATA]] + } + } +} +unset -nocomplain CONTROL + +set testprefix fts3sort + +#------------------------------------------------------------------------- +# Tests for parsing the "order=asc" and "order=desc" directives. +# +foreach {tn param res} { + 1 "order=asc" {0 {}} + 2 "order=desc" {0 {}} + 3 "order=dec" {1 {unrecognized order: dec}} + 4 "order=xxx, order=asc" {1 {unrecognized order: xxx}} + 5 "order=desc, order=asc" {0 {}} +} { + execsql { DROP TABLE IF EXISTS t1 } + do_catchsql_test 2.1.$tn " + CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param) + " $res +} + +do_execsql_test 2.2 { + BEGIN; + CREATE VIRTUAL TABLE t2 USING fts4(order=desc); + INSERT INTO t2 VALUES('aa bb'); + INSERT INTO t2 VALUES('bb cc'); + INSERT INTO t2 VALUES('cc aa'); + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; + END; +} {3 1} +do_execsql_test 2.3 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; +} {3 1} + +finish_test + diff --git a/test/fts4aa.test b/test/fts4aa.test index c6fa3b9c..1131df3b 100644 --- a/test/fts4aa.test +++ b/test/fts4aa.test @@ -21,14 +21,12 @@ ifcapable !fts3 { finish_test return } -if {[db eval {SELECT sqlite_compileoption_used('ENABLE_FTS4')}]==0} { - finish_test - return -} -do_test fts4aa-1.0 { +# This procedure fills an existing FTS3/FTS4 table with many entries. +# The table needs to have a single column (other than docid) named "words". +# +proc fts4aa_fill_table {} { db eval { -CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter); BEGIN TRANSACTION; INSERT INTO t1(docid,words) VALUES(1001001,'In the beginning God created the heaven and the earth.'); INSERT INTO t1(docid,words) VALUES(1001002,'And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.'); @@ -1565,9 +1563,41 @@ INSERT INTO t1(docid,words) VALUES(1050025,'And Joseph took an oath of the child INSERT INTO t1(docid,words) VALUES(1050026,'So Joseph died, being an hundred and ten years old: and they embalmed him, and he was put in a coffin in Egypt.'); COMMIT; } +} + +# The following is a list of queries to perform against the above +# FTS3/FTS4 database. We will be trying these queries in various +# configurations to ensure that they always return the same answers. +# +set fts4aa_queries { + {abraham} + {the king} + {"the king"} + {abraham OR joseph} + {ab* OR jos*} + {lived t*} + {spake hebrew} + {melchizedek} + {t* melchizedek} + {melchizedek t*} +} +unset -nocomplain fts4aa_res + +# Set up the baseline results +# +do_test fts4aa-1.0 { + db eval { + CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter); + } + fts4aa_fill_table + foreach q $::fts4aa_queries { + set r [db eval {SELECT docid FROM t1 WHERE words MATCH $q ORDER BY docid}] + set ::fts4aa_res($q) $r + } } {} - +# Legacy test cases +# do_test fts4aa-1.1 { db eval { SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize @@ -1587,50 +1617,114 @@ proc mit {blob} { } db func mit mit -do_test fts4aa-2.1 { +do_test fts4aa-1.3 { db eval { - SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'melchizedek'; + SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'melchizedek'; } } {1014018 {1 1 1 1 1 1533 25 20}} -do_test fts4aa-2.2 { +do_test fts4aa-1.4 { db eval { - SELECT docid, mit(matchinfo(t1)) FROM t1 + SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'spake hebrew' ORDER BY docid; } } {1039014 {2 1 1 40 40 1 6 6 1533 25 42} 1039017 {2 1 1 40 40 1 6 6 1533 25 26}} -do_test fts4aa-2.3 { +do_test fts4aa-1.5 { db eval { - SELECT docid, mit(matchinfo(t1)) FROM t1 + SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'laban overtook jacob' ORDER BY docid; } } {1031025 {3 1 2 54 46 1 3 3 2 181 160 1533 25 24}} -do_test fts4aa-9.1 { +do_test fts4aa-1.6 { db eval { DELETE FROM t1 WHERE docid!=1050026; SELECT hex(size) FROM t1_docsize; SELECT hex(value) FROM t1_stat; } -} {17 0117} +} {17 01176F} -do_test fts4aa-9.2 { +do_test fts4aa-1.7 { db eval { SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize } } {} -do_test fts4aa-9.3 { +do_test fts4aa-1.8 { db eval { SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1 } } {} -do_test fts4aa-9.4 { +do_test fts4aa-1.9 { + # Note: Token 'in' is being deferred in the following query. db eval { - SELECT docid, mit(matchinfo(t1)) FROM t1 + SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'joseph died in egypt' ORDER BY docid; } -} {1050026 {4 1 1 1 1 1 1 1 2 2 1 1 1 1 1 23 23}} +} {1050026 {4 1 1 1 1 1 1 1 2 1 1 1 1 1 1 23 23}} + +# Should get the same search results from FTS3 +# +do_test fts4aa-2.0 { + db eval { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts3(words, tokenize porter); + } + fts4aa_fill_table +} {} +unset -nocomplain ii +set ii 0 +foreach {q r} [array get fts4aa_res] { + incr ii + do_test fts4aa-2.$ii { + db eval {SELECT docid FROM t1 WHERE words MATCH $::q ORDER BY docid} + } $r +} + +# Should get the same search results when the page size is very large +# +do_test fts4aa-3.0 { + db close + file delete -force test.db + sqlite3 db test.db + db eval { + PRAGMA page_size=65536; + CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter); + } + fts4aa_fill_table +} {} +unset -nocomplain ii +set ii 0 +foreach {q r} [array get fts4aa_res] { + incr ii + do_test fts4aa-3.$ii { + db eval {SELECT docid FROM t1 WHERE words MATCH $::q ORDER BY docid} + } $r +} + +# Should get the same search results when an authorizer prevents +# all PRAGMA statements. +# +proc no_pragma_auth {code arg1 arg2 arg3 arg4} { + if {$code=="SQLITE_PRAGMA"} {return SQLITE_DENY} + return SQLITE_OK; +} +do_test fts4aa-4.0 { + db auth ::no_pragma_auth + db eval { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter); + } + fts4aa_fill_table +} {} +unset -nocomplain ii +set ii 0 +foreach {q r} [array get fts4aa_res] { + incr ii + do_test fts4aa-4.$ii { + db eval {SELECT docid FROM t1 WHERE words MATCH $::q ORDER BY docid} + } $r +} finish_test diff --git a/test/hook.test b/test/hook.test index 6496d41e..c4dfb2e4 100644 --- a/test/hook.test +++ b/test/hook.test @@ -274,6 +274,34 @@ ifcapable compound&&attach { set ::update_hook } [list] } + +do_test hook-4.4 { + execsql { + CREATE TABLE t4(a UNIQUE, b); + INSERT INTO t4 VALUES(1, 'a'); + INSERT INTO t4 VALUES(2, 'b'); + } + set ::update_hook [list] + execsql { + REPLACE INTO t4 VALUES(1, 'c'); + } + set ::update_hook +} [list INSERT main t4 3 ] +do_execsql_test hook-4.4.1 { + SELECT * FROM t4 ORDER BY a; +} {1 c 2 b} +do_test hook-4.4.2 { + set ::update_hook [list] + execsql { + PRAGMA recursive_triggers = on; + REPLACE INTO t4 VALUES(1, 'd'); + } + set ::update_hook +} [list INSERT main t4 4 ] +do_execsql_test hook-4.4.3 { + SELECT * FROM t4 ORDER BY a; +} {1 d 2 b} + db update_hook {} # #---------------------------------------------------------------------------- diff --git a/test/insert4.test b/test/insert4.test index 0b069e99..44b428a1 100644 --- a/test/insert4.test +++ b/test/insert4.test @@ -326,4 +326,64 @@ do_test insert4-6.7 { } } {1 {constraint failed}} +# Ticket [6284df89debdfa61db8073e062908af0c9b6118e] +# Disable the xfer optimization if the destination table contains +# a foreign key constraint +# +ifcapable foreignkey { + do_test insert4-7.1 { + set ::sqlite3_xferopt_count 0 + execsql { + CREATE TABLE t7a(x INTEGER PRIMARY KEY); INSERT INTO t7a VALUES(123); + CREATE TABLE t7b(y INTEGER REFERENCES t7a); + CREATE TABLE t7c(z INT); INSERT INTO t7c VALUES(234); + INSERT INTO t7b SELECT * FROM t7c; + SELECT * FROM t7b; + } + } {234} + do_test insert4-7.2 { + set ::sqlite3_xferopt_count + } {1} + do_test insert4-7.3 { + set ::sqlite3_xferopt_count 0 + execsql { + DELETE FROM t7b; + PRAGMA foreign_keys=ON; + } + catchsql { + INSERT INTO t7b SELECT * FROM t7c; + } + } {1 {foreign key constraint failed}} + do_test insert4-7.4 { + execsql {SELECT * FROM t7b} + } {} + do_test insert4-7.5 { + set ::sqlite3_xferopt_count + } {0} + do_test insert4-7.6 { + set ::sqlite3_xferopt_count 0 + execsql { + DELETE FROM t7b; DELETE FROM t7c; + INSERT INTO t7c VALUES(123); + INSERT INTO t7b SELECT * FROM t7c; + SELECT * FROM t7b; + } + } {123} + do_test insert4-7.7 { + set ::sqlite3_xferopt_count + } {0} + do_test insert4-7.7 { + set ::sqlite3_xferopt_count 0 + execsql { + PRAGMA foreign_keys=OFF; + DELETE FROM t7b; + INSERT INTO t7b SELECT * FROM t7c; + SELECT * FROM t7b; + } + } {123} + do_test insert4-7.8 { + set ::sqlite3_xferopt_count + } {1} +} + finish_test diff --git a/test/lock_common.tcl b/test/lock_common.tcl index 21f58426..bc1eb86b 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -55,8 +55,8 @@ proc do_multiclient_test {varname script} { uplevel set $varname $tn uplevel $script - code2 { db2 close } - code3 { db3 close } + catch { code2 { db2 close } } + catch { code3 { db3 close } } catch { close $::code2_chan } catch { close $::code3_chan } catch { db close } diff --git a/test/malloc.test b/test/malloc.test index 678b2be0..9bd5314b 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -894,6 +894,18 @@ ifcapable stat2&&utf16 { } } +# Test that if an OOM error occurs, aux-data is still correctly destroyed. +# This test case was causing either a memory-leak or an assert() failure +# at one point, depending on the configuration. +# +do_malloc_test 39 -tclprep { + sqlite3 db test.db +} -sqlbody { + SELECT test_auxdata('abc', 'def'); +} -cleanup { + db close +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} diff --git a/test/multiplex.test b/test/multiplex.test index 518cbe37..9278e842 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -567,5 +567,46 @@ if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} { } } +#------------------------------------------------------------------------- +# Test that you can vacuum a multiplex'ed DB. + +ifcapable vacuum { + +sqlite3_multiplex_shutdown +do_test multiplex-6.0.0 { + multiplex_delete test.db + sqlite3_multiplex_initialize "" 1 + sqlite3 db test.db + multiplex_set db main 4096 16 +} {SQLITE_OK} + +do_test multiplex-6.1.0 { + execsql { + PRAGMA page_size=1024; + PRAGMA journal_mode=DELETE; + PRAGMA auto_vacuum=OFF; + } + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, randomblob($g_chunk_size)); + INSERT INTO t1 VALUES(2, randomblob($g_chunk_size)); + } +} {} +do_test multiplex-6.2.1 { file size [multiplex_name test.db 0] } [list $g_chunk_size] +do_test multiplex-6.2.2 { file size [multiplex_name test.db 1] } [list $g_chunk_size] + +do_test multiplex-6.3.0 { + execsql { VACUUM } +} {} + +do_test multiplex-6.99 { + db close + multiplex_delete test.db + sqlite3_multiplex_shutdown +} {SQLITE_OK} + +} + + catch { sqlite3_multiplex_shutdown } finish_test diff --git a/test/nan.test b/test/nan.test index 0e9462fc..3257a439 100644 --- a/test/nan.test +++ b/test/nan.test @@ -42,31 +42,31 @@ do_test nan-1.1.1 { db eval {SELECT x, typeof(x) FROM t1} } {{} null} if {$tcl_platform(platform) != "symbian"} { - do_test nan-1.1.2 { + do_realnum_test nan-1.1.2 { sqlite3_bind_double $::STMT 1 +Inf sqlite3_step $::STMT sqlite3_reset $::STMT db eval {SELECT x, typeof(x) FROM t1} } {{} null inf real} - do_test nan-1.1.3 { + do_realnum_test nan-1.1.3 { sqlite3_bind_double $::STMT 1 -Inf sqlite3_step $::STMT sqlite3_reset $::STMT db eval {SELECT x, typeof(x) FROM t1} } {{} null inf real -inf real} - do_test nan-1.1.4 { + do_realnum_test nan-1.1.4 { sqlite3_bind_double $::STMT 1 -NaN sqlite3_step $::STMT sqlite3_reset $::STMT db eval {SELECT x, typeof(x) FROM t1} } {{} null inf real -inf real {} null} - do_test nan-1.1.5 { + do_realnum_test nan-1.1.5 { sqlite3_bind_double $::STMT 1 NaN0 sqlite3_step $::STMT sqlite3_reset $::STMT db eval {SELECT x, typeof(x) FROM t1} } {{} null inf real -inf real {} null {} null} - do_test nan-1.1.6 { + do_realnum_test nan-1.1.6 { sqlite3_bind_double $::STMT 1 -NaN0 sqlite3_step $::STMT sqlite3_reset $::STMT @@ -231,12 +231,12 @@ if {$tcl_platform(platform) != "symbian"} { # Do not run these tests on Symbian, as the Tcl port doesn't like to # convert from floating point value "-inf" to a string. # - do_test nan-4.7 { + do_realnum_test nan-4.7 { db eval {DELETE FROM t1} db eval "INSERT INTO t1 VALUES([string repeat 9 309].0)" db eval {SELECT x, typeof(x) FROM t1} } {inf real} - do_test nan-4.8 { + do_realnum_test nan-4.8 { db eval {DELETE FROM t1} db eval "INSERT INTO t1 VALUES(-[string repeat 9 309].0)" db eval {SELECT x, typeof(x) FROM t1} @@ -313,7 +313,7 @@ do_test nan-4.18 { db eval {SELECT CAST(x AS text), typeof(x) FROM t1} } {-9.88131291682493e-324 real} -do_test nan-4.20 { +do_realnum_test nan-4.20 { db eval {DELETE FROM t1} set big [string repeat 9 10000].0e-9000 db eval "INSERT INTO t1 VALUES($big)" diff --git a/test/oserror.test b/test/oserror.test index 2d3a24a8..bdeb8684 100644 --- a/test/oserror.test +++ b/test/oserror.test @@ -51,6 +51,7 @@ proc do_re_test {tn script expression} { # a call to getcwd() may fail if there are no free file descriptors. So # an error may be reported for either open() or getcwd() here. # +puts "Possible valgrind error about invalid file descriptor follows:" do_test 1.1.1 { set ::log [list] list [catch { diff --git a/test/pager1.test b/test/pager1.test index 8c31e153..136ca05f 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -441,6 +441,8 @@ do_test pager1.4.2.1 { db close tstvfs delete } {} + +if {$::tcl_platform(platform)!="windows"} { do_test pager1.4.2.2 { faultsim_restore_and_reopen execsql { @@ -473,6 +475,7 @@ do_test pager1.4.2.5 { PRAGMA integrity_check; } } {4 ok} +} do_test pager1.4.3.1 { testvfs tstvfs -default 1 @@ -1548,6 +1551,7 @@ do_execsql_test pager1-13.1.1 { UPDATE t1 SET b = a_string(400); } {persist} +if {$::tcl_platform(platform)!="windows"} { # Run transactions of increasing sizes. Eventually, one (or more than one) # of these will write just enough content that one of the old headers created # by the transaction in the block above lies immediately after the content @@ -1570,7 +1574,9 @@ for {set nUp 1} {$nUp<64} {incr nUp} { } {ok} db2 close } +} +if {$::tcl_platform(platform)!="windows"} { # Same test as above. But this time with an index on the table. # do_execsql_test pager1-13.2.1 { @@ -1591,6 +1597,7 @@ for {set nUp 1} {$nUp<64} {incr nUp} { } {ok} db2 close } +} db close tv delete @@ -1653,7 +1660,7 @@ for {set i 0} {$i<513} {incr i 3} { testvfs tv -default 1 tv script xOpenCb tv filter xOpen -proc xOpenCb {method filename} { +proc xOpenCb {method filename args} { set ::file_len [string length $filename] } sqlite3 db test.db @@ -2381,6 +2388,7 @@ do_test pager1-30.1 { # file can still be rolled back. This is required for backward compatibility - # versions of SQLite prior to 3.5.8 always set this field to zero. # +if {$tcl_platform(platform)=="unix"} { do_test pager1-31.1 { faultsim_delete_and_reopen execsql { @@ -2408,7 +2416,7 @@ do_test pager1-31.1 { sqlite3 db2 test.db2 execsql { PRAGMA integrity_check } db2 } {ok} - +} finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index c5c7d3b3..ced6da3b 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -20,6 +20,11 @@ if {[permutation] == "inmemory_journal"} { return } +if {$::tcl_platform(platform)=="windows"} { + finish_test + return +} + set a_string_counter 1 proc a_string {n} { global a_string_counter diff --git a/test/permutations.test b/test/permutations.test index 9c48d9aa..22e2189d 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -179,10 +179,11 @@ test_suite "fts3" -prefix "" -description { fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test + fts3sort.test fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test + fts3aux1.test fts3comp1.test fts3auto.test } @@ -528,7 +529,7 @@ test_suite "inmemory_journal" -description { stmt.test # WAL mode is different. - wal* + wal* tkt-2d1a5c67d.test }] ifcapable mem3 { diff --git a/test/quota.test b/test/quota.test index 2cf76280..d7601e0e 100644 --- a/test/quota.test +++ b/test/quota.test @@ -202,6 +202,9 @@ do_test quota-3.2.9 { set ::quota [list] proc quota_callback {file limitvar size} { upvar $limitvar limit + if {$::tcl_platform(platform)=="windows"} { + set file [ lindex [string map {\\ \/} $file] 0 ] + } lappend ::quota $file $size set limit 0 } diff --git a/test/sync.test b/test/sync.test index 98aa7731..4b02ad72 100644 --- a/test/sync.test +++ b/test/sync.test @@ -19,19 +19,27 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl # -# These tests are only applicable on unix when pager pragma are +# These tests are only applicable when pager pragma are # enabled. Also, since every test uses an ATTACHed database, they # are only run when ATTACH is enabled. # -if {$::tcl_platform(platform)!="unix"} { - finish_test - return -} ifcapable !pager_pragmas||!attach { finish_test return } +set sqlite_sync_count 0 +proc cond_incr_sync_count {adj} { + global sqlite_sync_count + if {$::tcl_platform(platform) == "windows"} { + incr sqlite_sync_count $adj + } { + ifcapable !dirsync { + incr sqlite_sync_count $adj + } + } +} + do_test sync-1.1 { set sqlite_sync_count 0 file delete -force test2.db @@ -42,9 +50,7 @@ do_test sync-1.1 { ATTACH DATABASE 'test2.db' AS db2; CREATE TABLE db2.t2(x,y); } - ifcapable !dirsync { - incr sqlite_sync_count 2 - } + cond_incr_sync_count 2 set sqlite_sync_count } 8 ifcapable pager_pragmas { @@ -58,9 +64,7 @@ ifcapable pager_pragmas { INSERT INTO t2 VALUES(3,4); COMMIT; } - ifcapable !dirsync { - incr sqlite_sync_count 3 - } + cond_incr_sync_count 3 set sqlite_sync_count } 8 } @@ -74,9 +78,7 @@ do_test sync-1.3 { INSERT INTO t2 VALUES(5,6); COMMIT; } - ifcapable !dirsync { - incr sqlite_sync_count 3 - } + cond_incr_sync_count 3 set sqlite_sync_count } 10 ifcapable pager_pragmas { diff --git a/test/tester.tcl b/test/tester.tcl index bae10530..8624821a 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -354,6 +354,31 @@ proc do_test {name cmd expected} { flush stdout } +proc filepath_normalize {p} { + # test cases should be written to assume "unix"-like file paths + if {$::tcl_platform(platform)!="unix"} { + # lreverse*2 as a hack to remove any unneeded {} after the string map + lreverse [lreverse [string map {\\ /} [regsub -nocase -all {[a-z]:[/\\]+} $p {/}]]] + } { + set p + } +} +proc do_filepath_test {name cmd expected} { + uplevel [list do_test $name [ + subst -nocommands { filepath_normalize [ $cmd ] } + ] [filepath_normalize $expected]] +} + +proc realnum_normalize {r} { + # different TCL versions display floating point values differently. + string map {1.#INF inf Inf inf .0e e} [regsub -all {(e[+-])0+} $r {\1}] +} +proc do_realnum_test {name cmd expected} { + uplevel [list do_test $name [ + subst -nocommands { realnum_normalize [ $cmd ] } + ] [realnum_normalize $expected]] +} + proc fix_testname {varname} { upvar $varname testname if {[info exists ::testprefix] @@ -365,11 +390,11 @@ proc fix_testname {varname} { proc do_execsql_test {testname sql {result {}}} { fix_testname testname - uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]] + uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] } proc do_catchsql_test {testname sql result} { fix_testname testname - uplevel do_test $testname [list "catchsql {$sql}"] [list $result] + uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result] } proc do_eqp_test {name sql res} { uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res] @@ -747,6 +772,17 @@ proc integrity_check {name {db db}} { } } + +# Return true if the SQL statement passed as the second argument uses a +# statement transaction. +# +proc sql_uses_stmt {db sql} { + set stmt [sqlite3_prepare $db $sql -1 dummy] + set uses [uses_stmt_journal $stmt] + sqlite3_finalize $stmt + return $uses +} + proc fix_ifcapable_expr {expr} { set ret "" set state 0 diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test new file mode 100644 index 00000000..278e1d38 --- /dev/null +++ b/test/tkt-2d1a5c67d.test @@ -0,0 +1,127 @@ +# 2011 May 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. Specifically, +# it tests that ticket [2d1a5c67dfc2363e44f29d9bbd57f7331851390a] has +# been resolved. +# +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-2d1a5c67d + +ifcapable {!wal || !vtab} {finish_test; return} + +for {set ii 1} {$ii<=10} {incr ii} { + do_test tkt-2d1a5c67d.1.$ii { + db close + forcedelete test.db test.db-wal + sqlite3 db test.db + db eval "PRAGMA cache_size=$::ii" + db eval { + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a,b); + CREATE INDEX t1b ON t1(b); + CREATE TABLE t2(x,y UNIQUE); + INSERT INTO t2 VALUES(3,4); + BEGIN; + INSERT INTO t1(a,b) VALUES(1,2); + SELECT 'A', * FROM t2 WHERE y=4; + SELECT 'B', * FROM t1; + COMMIT; + SELECT 'C', * FROM t1; + } + } {wal A 3 4 B 1 2 C 1 2} +} + +db close +forcedelete test.db test.db-wal +sqlite3 db test.db +register_wholenumber_module db +db eval { + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a,b); + CREATE INDEX t1b ON t1(b); + CREATE TABLE t2(x,y); + CREATE VIRTUAL TABLE nums USING wholenumber; + INSERT INTO t2 SELECT value, randomblob(1000) FROM nums + WHERE value BETWEEN 1 AND 1000; +} + +for {set ii 1} {$ii<=10} {incr ii} { + do_test tkt-2d1a5c67d.2.$ii { + db eval "PRAGMA cache_size=$::ii" + db eval { + DELETE FROM t1; + BEGIN; + INSERT INTO t1(a,b) VALUES(1,2); + SELECT sum(length(y)) FROM t2; + COMMIT; + SELECT * FROM t1; + } + } {1000000 1 2} +} + +db close +sqlite3 db test.db + + +do_execsql_test 3.1 { + PRAGMA cache_size = 10; + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a); +} + +do_execsql_test 3.2 { + INSERT INTO t3 VALUES(NULL, randomblob(500)); + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 2 + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 4 + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 8 + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 16 + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 32 + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 64 + INSERT INTO t3 SELECT NULL, b||b FROM t3; -- 128 +} + +do_execsql_test 3.3 { + BEGIN; + INSERT INTO t4 VALUES('xyz'); +} + +do_test 3.4 { + set blobs [list] + for {set i 1} {$i<100} {incr i} { + set b [db incrblob -readonly t3 b $i] + read $b + lappend blobs $b + } + + execsql COMMIT + execsql { SELECT * FROM t4 WHERE a = 'xyz' } +} {xyz} + +do_test 3.5 { + foreach b $blobs { close $b } + execsql { SELECT * FROM t4 WHERE a = 'xyz' } +} {xyz} + +# Check that recovery works on the WAL file. +# +forcedelete test.db2-wal test.db2 +do_test 3.6 { + file copy test.db-wal test.db2-wal + file copy test.db test.db2 + sqlite3 db2 test.db2 + execsql { SELECT * FROM t4 WHERE a = 'xyz' } db2 +} {xyz} + +finish_test diff --git a/test/tkt-91e2e8ba6f.test b/test/tkt-91e2e8ba6f.test new file mode 100644 index 00000000..57f13740 --- /dev/null +++ b/test/tkt-91e2e8ba6f.test @@ -0,0 +1,52 @@ +# 2011 June 23 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for SQLite. Specifically, it tests that SQLite +# does not crash and an error is returned if localhost() fails. This +# is the problem reported by ticket 91e2e8ba6f. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix tkt-91e2e8ba6f + + +do_execsql_test 1.1 { + CREATE TABLE t1(x INTEGER, y REAL); + INSERT INTO t1 VALUES(11, 11); +} {} + +do_execsql_test 1.2 { + SELECT x/10, y/10 FROM t1; +} {1 1.1} + +do_execsql_test 1.3 { + SELECT x/10, y/10 FROM (SELECT * FROM t1); +} {1 1.1} + +do_execsql_test 1.4 { + SELECT x/10, y/10 FROM (SELECT * FROM t1 LIMIT 5 OFFSET 0); +} {1 1.1} + +do_execsql_test 1.5 { + SELECT x/10, y/10 FROM (SELECT * FROM t1 LIMIT 5 OFFSET 0) LIMIT 5 OFFSET 0; +} {1 1.1} + +do_execsql_test 1.6 { + SELECT a.x/10, a.y/10 FROM + (SELECT * FROM t1 LIMIT 5 OFFSET 0) AS a, t1 AS b WHERE a.x = b.x + LIMIT 5 OFFSET 0; +} {1 1.1} + +do_execsql_test 1.7 { + CREATE VIEW v1 AS SELECT * FROM t1 LIMIT 5; + SELECT a.x/10, a.y/10 FROM v1 AS a, t1 AS b WHERE a.x = b.x LIMIT 5 OFFSET 0; +} {1 1.1} + +finish_test diff --git a/test/tkt-bd484a090c.test b/test/tkt-bd484a090c.test new file mode 100644 index 00000000..3d2b5999 --- /dev/null +++ b/test/tkt-bd484a090c.test @@ -0,0 +1,38 @@ +# 2011 June 21 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for SQLite. Specifically, it tests that SQLite +# does not crash and an error is returned if localhost() fails. This +# is the problem reported by ticket bd484a090c. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix tkt-bd484a090c + + +do_test 1.1 { + lindex [catchsql { SELECT datetime('now', 'localtime') }] 0 +} {0} +do_test 1.2 { + lindex [catchsql { SELECT datetime('now', 'utc') }] 0 +} {0} + +sqlite3_test_control SQLITE_TESTCTRL_LOCALTIME_FAULT 1 + +do_test 2.1 { + catchsql { SELECT datetime('now', 'localtime') } +} {1 {local time unavailable}} +do_test 2.2 { + catchsql { SELECT datetime('now', 'utc') } +} {1 {local time unavailable}} + +sqlite3_test_control SQLITE_TESTCTRL_LOCALTIME_FAULT 0 + +finish_test diff --git a/test/tkt3838.test b/test/tkt3838.test index 4fb5c55a..5dfc2b8a 100644 --- a/test/tkt3838.test +++ b/test/tkt3838.test @@ -25,7 +25,7 @@ ifcapable !altertable { return } -do_test tkt3838-1.1 { +do_realnum_test tkt3838-1.1 { db eval { PRAGMA encoding=UTF16; CREATE TABLE t1(x); diff --git a/test/tkt3922.test b/test/tkt3922.test index 6506dced..86bf5fec 100644 --- a/test/tkt3922.test +++ b/test/tkt3922.test @@ -36,21 +36,21 @@ if {[working_64bit_int]} { } } {-1 integer} } -do_test tkt3922.2 { +do_realnum_test tkt3922.2 { execsql { DELETE FROM t1; INSERT INTO t1 VALUES('-9223372036854775809'); SELECT a, typeof(a) FROM t1; } } {-9.22337203685478e+18 real} -do_test tkt3922.3 { +do_realnum_test tkt3922.3 { execsql { DELETE FROM t1; INSERT INTO t1 VALUES('-9223372036854776832'); SELECT a, typeof(a) FROM t1; } } {-9.22337203685478e+18 real} -do_test tkt3922.4 { +do_realnum_test tkt3922.4 { execsql { DELETE FROM t1; INSERT INTO t1 VALUES('-9223372036854776833'); @@ -78,7 +78,7 @@ if {[working_64bit_int]} { } } {1 integer} } -do_test tkt3922.6 { +do_realnum_test tkt3922.6 { execsql { DELETE FROM t1; INSERT INTO t1 VALUES('9223372036854775808'); diff --git a/test/trace2.test b/test/trace2.test index a70dd89d..42738db3 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -141,10 +141,10 @@ ifcapable fts3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" - "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" - "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" + "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC" + "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" - "-- DELETE FROM 'main'.'x1_segdir'" + "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } diff --git a/test/triggerC.test b/test/triggerC.test index 694d069d..e5a492a4 100644 --- a/test/triggerC.test +++ b/test/triggerC.test @@ -228,25 +228,25 @@ foreach {n tdefn rc} { } do_test triggerC-2.2 { - execsql { + execsql " CREATE TABLE t22(x); CREATE TRIGGER t22a AFTER INSERT ON t22 BEGIN INSERT INTO t22 SELECT x + (SELECT max(x) FROM t22) FROM t22; END; CREATE TRIGGER t22b BEFORE INSERT ON t22 BEGIN - SELECT CASE WHEN (SELECT count(*) FROM t22) >= 100 + SELECT CASE WHEN (SELECT count(*) FROM t22) >= [expr $SQLITE_MAX_TRIGGER_DEPTH / 2] THEN RAISE(IGNORE) ELSE NULL END; END; INSERT INTO t22 VALUES(1); SELECT count(*) FROM t22; - } -} {100} + " +} [list [expr $SQLITE_MAX_TRIGGER_DEPTH / 2]] do_test triggerC-2.3 { - execsql { + execsql " CREATE TABLE t23(x PRIMARY KEY); CREATE TRIGGER t23a AFTER INSERT ON t23 BEGIN @@ -254,15 +254,15 @@ do_test triggerC-2.3 { END; CREATE TRIGGER t23b BEFORE INSERT ON t23 BEGIN - SELECT CASE WHEN new.x>500 + SELECT CASE WHEN new.x>[expr $SQLITE_MAX_TRIGGER_DEPTH / 2] THEN RAISE(IGNORE) ELSE NULL END; END; INSERT INTO t23 VALUES(1); SELECT count(*) FROM t23; - } -} {500} + " +} [list [expr $SQLITE_MAX_TRIGGER_DEPTH / 2]] #----------------------------------------------------------------------- @@ -288,12 +288,12 @@ do_test triggerC-3.1.3 { } {} do_test triggerC-3.2.1 { - execsql { + execsql " CREATE TABLE t3b(x); - CREATE TRIGGER t3bi AFTER INSERT ON t3b WHEN new.x<2000 BEGIN + CREATE TRIGGER t3bi AFTER INSERT ON t3b WHEN new.x<[expr $SQLITE_MAX_TRIGGER_DEPTH * 2] BEGIN INSERT INTO t3b VALUES(new.x+1); END; - } + " catchsql { INSERT INTO t3b VALUES(1); } @@ -303,39 +303,39 @@ do_test triggerC-3.2.2 { } {} do_test triggerC-3.3.1 { - catchsql { - INSERT INTO t3b VALUES(1001); - } + catchsql " + INSERT INTO t3b VALUES([expr $SQLITE_MAX_TRIGGER_DEPTH + 1]); + " } {0 {}} do_test triggerC-3.3.2 { db eval {SELECT count(*), max(x), min(x) FROM t3b} -} {1000 2000 1001} +} [list $SQLITE_MAX_TRIGGER_DEPTH [expr $SQLITE_MAX_TRIGGER_DEPTH * 2] [expr $SQLITE_MAX_TRIGGER_DEPTH + 1]] do_test triggerC-3.4.1 { - catchsql { + catchsql " DELETE FROM t3b; - INSERT INTO t3b VALUES(999); - } + INSERT INTO t3b VALUES([expr $SQLITE_MAX_TRIGGER_DEPTH - 1]); + " } {1 {too many levels of trigger recursion}} do_test triggerC-3.4.2 { db eval {SELECT count(*), max(x), min(x) FROM t3b} } {0 {} {}} do_test triggerC-3.5.1 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 100 - catchsql { - INSERT INTO t3b VALUES(1901); - } + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH [expr $SQLITE_MAX_TRIGGER_DEPTH / 10] + catchsql " + INSERT INTO t3b VALUES([expr ($SQLITE_MAX_TRIGGER_DEPTH * 2) - ($SQLITE_MAX_TRIGGER_DEPTH / 10) + 1]); + " } {0 {}} do_test triggerC-3.5.2 { db eval {SELECT count(*), max(x), min(x) FROM t3b} -} {100 2000 1901} +} [list [expr $SQLITE_MAX_TRIGGER_DEPTH / 10] [expr $SQLITE_MAX_TRIGGER_DEPTH * 2] [expr ($SQLITE_MAX_TRIGGER_DEPTH * 2) - ($SQLITE_MAX_TRIGGER_DEPTH / 10) + 1]] do_test triggerC-3.5.3 { - catchsql { + catchsql " DELETE FROM t3b; - INSERT INTO t3b VALUES(1900); - } + INSERT INTO t3b VALUES([expr ($SQLITE_MAX_TRIGGER_DEPTH * 2) - ($SQLITE_MAX_TRIGGER_DEPTH / 10)]); + " } {1 {too many levels of trigger recursion}} do_test triggerC-3.5.4 { db eval {SELECT count(*), max(x), min(x) FROM t3b} @@ -343,25 +343,25 @@ do_test triggerC-3.5.4 { do_test triggerC-3.6.1 { sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1 - catchsql { - INSERT INTO t3b VALUES(2000); - } + catchsql " + INSERT INTO t3b VALUES([expr $SQLITE_MAX_TRIGGER_DEPTH * 2]); + " } {0 {}} do_test triggerC-3.6.2 { db eval {SELECT count(*), max(x), min(x) FROM t3b} -} {1 2000 2000} +} [list 1 [expr $SQLITE_MAX_TRIGGER_DEPTH * 2] [expr $SQLITE_MAX_TRIGGER_DEPTH * 2]] do_test triggerC-3.6.3 { - catchsql { + catchsql " DELETE FROM t3b; - INSERT INTO t3b VALUES(1999); - } + INSERT INTO t3b VALUES([expr ($SQLITE_MAX_TRIGGER_DEPTH * 2) - 1]); + " } {1 {too many levels of trigger recursion}} do_test triggerC-3.6.4 { db eval {SELECT count(*), max(x), min(x) FROM t3b} } {0 {} {}} -sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000 - +sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH $SQLITE_MAX_TRIGGER_DEPTH + #----------------------------------------------------------------------- # This next block of tests, triggerC-4.*, checks that affinity diff --git a/test/types3.test b/test/types3.test index 33f2595c..807ae84f 100644 --- a/test/types3.test +++ b/test/types3.test @@ -21,7 +21,7 @@ source $testdir/tester.tcl # A variable with only a string representation comes in as TEXT do_test types3-1.1 { set V {} - append V {} + append V x concat [tcl_variable_type V] [execsql {SELECT typeof(:V)}] } {string text} diff --git a/test/unordered.test b/test/unordered.test index ca4beaca..a9c62534 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -57,6 +57,9 @@ foreach idxmode {ordered unordered} { 6 "SELECT * FROM t1 WHERE a = ?" {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~1 rows)}} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?) (~1 rows)}} + 7 "SELECT count(*) FROM t1" + {0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1(~128 rows)}} + {0 0 0 {SCAN TABLE t1 (~128 rows)}} } { do_eqp_test 1.$idxmode.$tn $sql $r($idxmode) } diff --git a/test/uri.test b/test/uri.test new file mode 100644 index 00000000..f361e929 --- /dev/null +++ b/test/uri.test @@ -0,0 +1,310 @@ +# 2011 April 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Test organization: +# +# 1.*: That file names are correctly extracted from URIs. +# 2.*: That URI options (query parameters) are correctly extracted from URIs. +# 3.*: That specifying an unknown VFS causes an error. +# 4.*: Tests for specifying other options (other than "vfs"). +# 5.*: Test using a different VFS with an attached database. +# 6.*: Test that authorities other than "" and localhost cause errors. +# 7.*: Test that a read-write db can be attached to a read-only connection. +# + +set testprefix uri +db close +sqlite3_shutdown +sqlite3_config_uri 1 + +#------------------------------------------------------------------------- +# Test that file names are correctly extracted from URIs. +# +foreach {tn uri file} { + 1 test.db test.db + 2 file:test.db test.db + 3 file://PWD/test.db test.db + 4 file:PWD/test.db test.db + 5 file:test.db?mork=1 test.db + 6 file:test.db?mork=1&tonglor=2 test.db + 7 file:test.db?mork=1#boris test.db + 8 file:test.db#boris test.db + 9 test.db#boris test.db#boris + 10 file:test%2Edb test.db + 11 file file + 12 http:test.db http:test.db + 13 file:test.db%00extra test.db + 14 file:testdb%00.db%00extra testdb + + 15 test.db?mork=1#boris test.db?mork=1#boris + 16 file://localhostPWD/test.db%3Fhello test.db?hello +} { + + if {$tcl_platform(platform)=="windows"} { + if {$tn>14} break + set uri [string map [list PWD /[pwd]] $uri] + } else { + set uri [string map [list PWD [pwd]] $uri] + } + + if {[file isdir $file]} {error "$file is a directory"} + forcedelete $file + do_test 1.$tn.1 { file exists $file } 0 + set DB [sqlite3_open $uri] + do_test 1.$tn.2 { file exists $file } 1 + sqlite3_close $DB + forcedelete $file + + do_test 1.$tn.3 { file exists $file } 0 + sqlite3 db xxx.db + catchsql { ATTACH $uri AS aux } + do_test 1.$tn.4 { file exists $file } 1 + db close +} + +#------------------------------------------------------------------------- +# Test that URI query parameters are passed through to the VFS layer +# correctly. +# +testvfs tvfs2 +testvfs tvfs -default 1 +tvfs filter xOpen +tvfs script open_method +proc open_method {method file arglist} { + set ::arglist $arglist +} +foreach {tn uri kvlist} { + 1 file:test.db?hello=world {hello world} + 2 file:test.db?hello&world {hello {} world {}} + 3 file:test.db?hello=1&world=2&vfs=tvfs {hello 1 world 2 vfs tvfs} + 4 file:test.db?hello=1&world=2&vfs=tvfs2 {} + 5 file:test.db?%68%65%6C%6C%6F=%77%6F%72%6C%64 {hello world} + 6 file:testdb%00.db?hello%00extra=world%00ex {hello world} + 7 file:testdb%00.db?hello%00=world%00 {hello world} + 8 file:testdb%00.db?=world&xyz=abc {xyz abc} + 9 file:test.db?%00hello=world&xyz=abc {xyz abc} + 10 file:test.db?hello=%00world&xyz= {hello {} xyz {}} + 11 file:test.db?=#ravada {} + 12 file:test.db?&&&&&&&&hello=world&&&&&&& {hello world} + + 13 test.db?&&&&&&&&hello=world&&&&&&& {} + 14 http:test.db?hello&world {} +} { + + if {$tcl_platform(platform) == "windows" && $tn>12} { + continue + } + + set ::arglist "" + set DB [sqlite3_open $uri] + do_test 2.$tn.1 { set ::arglist } $kvlist + sqlite3_close $DB + + sqlite3 db xxx.db + set ::arglist "" + execsql { ATTACH $uri AS aux } + do_test 2.$tn.2 { set ::arglist } $kvlist + db close +} +tvfs delete +tvfs2 delete + +#------------------------------------------------------------------------- +# Test that specifying a non-existent VFS raises an error. +# +do_test 3.1 { + list [catch { sqlite3 db "file:test.db?vfs=nosuchvfs" } msg] $msg +} {1 {no such vfs: nosuchvfs}} + +#------------------------------------------------------------------------- +# Test some of the other options (other than "vfs"). +# +foreach {tn mode create_ok write_ok readonly_ok} { + 1 ro 0 0 1 + 2 rw 0 1 0 + 3 rwc 1 1 0 +} { + catch { db close } + forcedelete test.db + + set A(1) {0 {}} + set A(0) {1 {unable to open database file}} + do_test 4.1.$tn.1 { + list [catch {sqlite3 db "file:test.db?mode=$mode"} msg] $msg + } $A($create_ok) + + catch { db close } + forcedelete test.db + sqlite3 db test.db + db eval { CREATE TABLE t1(a, b) } + db close + + set A(1) {0 {}} + set A(0) {1 {attempt to write a readonly database}} + do_test 4.1.$tn.2 { + sqlite3 db "file:test.db?mode=$mode" + catchsql { INSERT INTO t1 VALUES(1, 2) } + } $A($write_ok) + + set A(1) {0 {}} + set A(0) [list 1 "access mode not allowed: $mode"] + do_test 4.1.$tn.3 { + list [catch {sqlite3 db "file:test.db?mode=$mode" -readonly 1} msg] $msg + } $A($readonly_ok) +} + +set orig [sqlite3_enable_shared_cache] +foreach {tn options sc_default is_shared} { + 1 "" 1 1 + 2 "cache=private" 1 0 + 3 "cache=shared" 1 1 + 4 "" 0 0 + 5 "cache=private" 0 0 + 6 "cache=shared" 0 1 +} { + catch { db close } + forcedelete test.db + + sqlite3_enable_shared_cache 1 + sqlite3 db2 test.db + db2 eval {CREATE TABLE t1(a, b)} + + sqlite3_enable_shared_cache $sc_default + sqlite3 db "file:test.db?$options" + db eval {SELECT * FROM t1} + + set A(1) {1 {database table is locked: t1}} + set A(0) {0 {}} + do_test 4.2.$tn { + db2 eval {BEGIN; INSERT INTO t1 VALUES(1, 2);} + catchsql { SELECT * FROM t1 } + } $A($is_shared) + + db2 close +} + +do_test 4.3.1 { + list [catch {sqlite3 db "file:test.db?mode=rc"} msg] $msg +} {1 {no such access mode: rc}} +do_test 4.3.2 { + list [catch {sqlite3 db "file:test.db?cache=public"} msg] $msg +} {1 {no such cache mode: public}} + +#------------------------------------------------------------------------- +# Test that things work if an ATTACHed database uses a different VFS than +# the main database. The important point is that for all operations +# involving the ATTACHed database, the correct versions of the following +# VFS are used for all operations involving the attached database. +# +# xOpen +# xDelete +# xAccess +# xFullPathname +# + +# This block of code creates two VFS - "tvfs1" and "tvfs2". Each time one +# of the above methods is called using "tvfs1", global variable ::T1(X) is +# set, where X is the file-name the method is called on. Calls to the above +# methods using "tvfs2" set entries in the global T2 array. +# +testvfs tvfs1 +tvfs1 filter {xOpen xDelete xAccess xFullPathname} +tvfs1 script tvfs1_callback +proc tvfs1_callback {method filename args} { + set ::T1([file tail $filename]) 1 +} +testvfs tvfs2 +tvfs2 filter {xOpen xDelete xAccess xFullPathname} +tvfs2 script tvfs2_callback +proc tvfs2_callback {method filename args} { + set ::T2([file tail $filename]) 1 +} + +catch {db close} +eval forcedelete [glob test.db*] +do_test 5.1.1 { + sqlite3 db file:test.db1?vfs=tvfs1 + execsql { + ATTACH 'file:test.db2?vfs=tvfs2' AS aux; + PRAGMA main.journal_mode = PERSIST; + PRAGMA aux.journal_mode = PERSIST; + CREATE TABLE t1(a, b); + CREATE TABLE aux.t2(a, b); + PRAGMA main.journal_mode = WAL; + PRAGMA aux.journal_mode = WAL; + INSERT INTO t1 VALUES('x', 'y'); + INSERT INTO t2 VALUES('x', 'y'); + } + lsort [array names ::T1] +} {test.db1 test.db1-journal test.db1-wal} + +do_test 5.1.2 { + lsort [array names ::T2] +} {test.db2 test.db2-journal test.db2-wal} + +db close +tvfs1 delete +tvfs2 delete + +#------------------------------------------------------------------------- +# Check that only "" and "localhost" are acceptable as authorities. +# +catch {db close} +foreach {tn uri res} { + 1 "file://localhost/PWD/test.db" {not an error} + 2 "file:///PWD/test.db" {not an error} + 3 "file:/PWD/test.db" {not an error} + 4 "file://l%6Fcalhost/PWD/test.db" {invalid uri authority: l%6Fcalhost} + 5 "file://lbcalhost/PWD/test.db" {invalid uri authority: lbcalhost} + 6 "file://x/PWD/test.db" {invalid uri authority: x} +} { + + if {$tcl_platform(platform)=="windows"} { + set uri [string map [list PWD [string range [pwd] 3 end]] $uri] + } else { + set uri [string map [list PWD [string range [pwd] 1 end]] $uri] + } + + do_test 6.$tn { + set DB [sqlite3_open $uri] + sqlite3_errmsg $DB + } $res + catch { sqlite3_close $DB } +} + +forcedelete test.db test.db2 +do_test 7.1 { + sqlite3 db test.db + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t2(a, b); + INSERT INTO t1 VALUES('a', 'b'); + } + db close +} {} +do_test 7.2 { + sqlite3 db file:test.db?mode=ro + execsql { ATTACH 'file:test.db2?mode=rw' AS aux } +} {} +do_execsql_test 7.3 { + INSERT INTO t2 VALUES('c', 'd') +} {} +do_catchsql_test 7.4 { + INSERT INTO t1 VALUES(3, 4) +} {1 {attempt to write a readonly database}} + +finish_test diff --git a/test/wal2.test b/test/wal2.test index 82234330..97966cd9 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -23,6 +23,18 @@ set testprefix wal2 ifcapable !wal {finish_test ; return } +set sqlite_sync_count 0 +proc cond_incr_sync_count {adj} { + global sqlite_sync_count + if {$::tcl_platform(platform) == "windows"} { + incr sqlite_sync_count $adj + } { + ifcapable !dirsync { + incr sqlite_sync_count $adj + } + } +} + proc set_tvfs_hdr {file args} { # Set $nHdr to the number of bytes in the wal-index header: @@ -1189,6 +1201,7 @@ foreach {tn sql reslist} { } {10 0 5 5 0 2 2} do_test wal2-14.$tn.3 { + cond_incr_sync_count 1 list $sqlite_sync_count $sqlite_fullsync_count } [lrange $reslist 0 1] diff --git a/test/wal5.test b/test/wal5.test index ad6bcfc7..0c700dfc 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,7 +235,14 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 7 5} - do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} + if {$tcl_platform(platform) == "windows"} { + # on unix, the size_hint is a no-op if no chunk size is set. + # the windows implementation does not have a similar check, + # and because of this, the db file size has an extra page. + do_test 2.3.$tn.8 { file_page_counts } {2 7 2 7} + } { + do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} + } } # Check that checkpoints block on the correct locks. And respond correctly diff --git a/test/wal7.test b/test/wal7.test new file mode 100644 index 00000000..cacfaeda --- /dev/null +++ b/test/wal7.test @@ -0,0 +1,118 @@ +# 2011 May 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the PRAGMA journal_size_limit when +# in WAL mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !wal {finish_test ; return } + +# Case 1: No size limit. Journal can get large. +# +do_test wal7-1.0 { + db close + forcedelete test.db + sqlite3 db test.db + db eval { + PRAGMA page_size=1024; + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=50; -- 50 pages + CREATE TABLE t1(x, y UNIQUE); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(zeroblob(200000),4); + CREATE TABLE t2(z); + DELETE FROM t1; + INSERT INTO t2 SELECT x FROM t1; + } + expr {[file size test.db-wal]>50*1100} +} 1 +do_test wal7-1.1 { + db eval {PRAGMA wal_checkpoint} + expr {[file size test.db-wal]>50*1100} +} 1 +do_test wal7-1.2 { + db eval {INSERT INTO t2 VALUES('hi');} + expr {[file size test.db-wal]>50*1100} +} 1 + +# Case 2: Size limit at half the autocheckpoint size. +# +do_test wal7-2.0 { + db close + forcedelete test.db + sqlite3 db test.db + db eval { + PRAGMA page_size=1024; + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=50; -- 50 pages + PRAGMA journal_size_limit=25000; + CREATE TABLE t1(x, y UNIQUE); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(zeroblob(200000),4); + CREATE TABLE t2(z); + DELETE FROM t1; + INSERT INTO t2 VALUES(1); + } + file size test.db-wal +} 25000 + + +# Case 3: Size limit of zero. +# +do_test wal7-3.0 { + db close + forcedelete test.db + sqlite3 db test.db + db eval { + PRAGMA page_size=1024; + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=50; -- 50 pages + PRAGMA journal_size_limit=0; + CREATE TABLE t1(x, y UNIQUE); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(zeroblob(200000),4); + CREATE TABLE t2(z); + DELETE FROM t1; + INSERT INTO t2 VALUES(1); + } + set sz [file size test.db-wal] + expr {$sz>0 && $sz<13700} +} 1 + + +# Case 4: Size limit set before going WAL +# +do_test wal7-4.0 { + db close + forcedelete test.db + sqlite3 db test.db + db eval { + PRAGMA page_size=1024; + PRAGMA journal_size_limit=25000; + PRAGMA journal_mode=WAL; + PRAGMA wal_autocheckpoint=50; -- 50 pages + CREATE TABLE t1(x, y UNIQUE); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(zeroblob(200000),4); + CREATE TABLE t2(z); + DELETE FROM t1; + INSERT INTO t2 VALUES(1); + } + set sz [file size test.db-wal] +} 25000 + + + + + +finish_test diff --git a/test/walro.test b/test/walro.test new file mode 100644 index 00000000..60bdce9e --- /dev/null +++ b/test/walro.test @@ -0,0 +1,161 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walro + +# These tests are only going to work on unix. +# +if {$::tcl_platform(platform) != "unix"} { + finish_test + return +} + +do_multiclient_test tn { + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + forcedelete walro + + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + + file mkdir walro + + do_test 1.1.1 { + code2 { sqlite3 db2 test.db } + sql2 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('a', 'b'); + } + file exists test.db-shm + } {1} + + do_test 1.1.2 { + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db file:test.db?readonly_shm=1 } + } {} + + do_test 1.1.3 { sql1 "SELECT * FROM t1" } {a b} + do_test 1.1.4 { sql2 "INSERT INTO t1 VALUES('c', 'd')" } {} + do_test 1.1.5 { sql1 "SELECT * FROM t1" } {a b c d} + + # Check that the read-only connection cannot write or checkpoint the db. + # + do_test 1.1.6 { + csql1 "INSERT INTO t1 VALUES('e', 'f')" + } {1 {attempt to write a readonly database}} + do_test 1.1.7 { + csql1 "PRAGMA wal_checkpoint" + } {1 {attempt to write a readonly database}} + + do_test 1.1.9 { sql2 "INSERT INTO t1 VALUES('e', 'f')" } {} + do_test 1.1.10 { sql1 "SELECT * FROM t1" } {a b c d e f} + + do_test 1.1.11 { + sql2 { + INSERT INTO t1 VALUES('g', 'h'); + PRAGMA wal_checkpoint; + } + set {} {} + } {} + do_test 1.1.12 { sql1 "SELECT * FROM t1" } {a b c d e f g h} + do_test 1.1.13 { sql2 "INSERT INTO t1 VALUES('i', 'j')" } {} + + do_test 1.2.1 { + code2 { db2 close } + code1 { db close } + list [file exists test.db-wal] [file exists test.db-shm] + } {1 1} + do_test 1.2.2 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j} + + do_test 1.2.3 { + code1 { db close } + file attributes test.db-shm -permissions rw-r--r-- + hexio_write test.db-shm 0 01020304 + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db file:test.db?readonly_shm=1 } + csql1 { SELECT * FROM t1 } + } {1 {attempt to write a readonly database}} + do_test 1.2.4 { + code1 { sqlite3_extended_errcode db } + } {SQLITE_READONLY_RECOVERY} + + do_test 1.2.5 { + file attributes test.db-shm -permissions rw-r--r-- + code2 { sqlite3 db2 test.db } + sql2 "SELECT * FROM t1" + } {a b c d e f g h i j} + file attributes test.db-shm -permissions r--r--r-- + do_test 1.2.6 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j} + + do_test 1.2.7 { + sql2 { + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES('k', 'l'); + } + set {} {} + } {} + do_test 1.2.8 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j k l} + + # Now check that if the readonly_shm option is not supplied, or if it + # is set to zero, it is not possible to connect to the database without + # read-write access to the shm. + do_test 1.3.1 { + code1 { db close } + code1 { sqlite3 db test.db } + csql1 { SELECT * FROM t1 } + } {1 {unable to open database file}} + + # Also test that if the -shm file can be opened for read/write access, + # it is, even if readonly_shm=1 is present in the URI. + do_test 1.3.2.1 { + code1 { db close } + code2 { db2 close } + file exists test.db-shm + } {0} + do_test 1.3.2.2 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j k l} + do_test 1.3.2.3 { + code1 { db close } + close [open test.db-shm w] + file attributes test.db-shm -permissions r--r--r-- + code1 { sqlite3 db file:test.db?readonly_shm=1 } + csql1 { SELECT * FROM t1 } + } {1 {attempt to write a readonly database}} + do_test 1.3.2.4 { + code1 { sqlite3_extended_errcode db } + } {SQLITE_READONLY_RECOVERY} +} + +finish_test diff --git a/tool/build-shell.sh b/tool/build-shell.sh new file mode 100644 index 00000000..54e83080 --- /dev/null +++ b/tool/build-shell.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# +# This script demonstrates how to do a full-featured build of the sqlite3 +# command-line shell on Linux. +# +# SQLite source code should be in a sibling directory named "sqlite". For +# example, put SQLite sources in ~/sqlite/sqlite and run this script from +# ~/sqlite/bld. There should be an appropriate Makefile in the current +# directory as well. +# +make sqlite3.c +gcc -o sqlite3 -g -Os -I. \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_ENABLE_VFSTRACE \ + -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_RTREE \ + -DHAVE_READLINE \ + -DHAVE_USLEEP=1 \ + ../sqlite/src/shell.c ../sqlite/src/test_vfstrace.c \ + sqlite3.c -ldl -lreadline -lncurses diff --git a/tool/getlock.c b/tool/getlock.c new file mode 100644 index 00000000..7eff04d7 --- /dev/null +++ b/tool/getlock.c @@ -0,0 +1,134 @@ +/* +** This utility program looks at an SQLite database and determines whether +** or not it is locked, the kind of lock, and who is holding this lock. +** +** This only works on unix when the posix advisory locking method is used +** (which is the default on unix) and when the PENDING_BYTE is in its +** usual place. +*/ +#include +#include +#include +#include +#include +#include +#include +#include + +static void usage(const char *argv0){ + fprintf(stderr, "Usage: %s database\n", argv0); + exit(1); +} + +/* Check for a conflicting lock. If one is found, print an this +** on standard output using the format string given and return 1. +** If there are no conflicting locks, return 0. +*/ +static int isLocked( + int h, /* File descriptor to check */ + int type, /* F_RDLCK or F_WRLCK */ + unsigned int iOfst, /* First byte of the lock */ + unsigned int iCnt, /* Number of bytes in the lock range */ + const char *zType /* Type of lock */ +){ + struct flock lk; + + memset(&lk, 0, sizeof(lk)); + lk.l_type = type; + lk.l_whence = SEEK_SET; + lk.l_start = iOfst; + lk.l_len = iCnt; + if( fcntl(h, F_GETLK, &lk)==(-1) ){ + fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); + exit(1); + } + if( lk.l_type==F_UNLCK ) return 0; + printf("%s lock held by %d\n", zType, (int)lk.l_pid); + return 1; +} + +/* +** Location of locking bytes in the database file +*/ +#define PENDING_BYTE (0x40000000) +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Lock locations for shared-memory locks used by WAL mode. +*/ +#define SHM_BASE 120 +#define SHM_WRITE SHM_BASE +#define SHM_CHECKPOINT (SHM_BASE+1) +#define SHM_RECOVER (SHM_BASE+2) +#define SHM_READ_FIRST (SHM_BASE+3) +#define SHM_READ_SIZE 5 + + +int main(int argc, char **argv){ + int hDb; /* File descriptor for the open database file */ + int hShm; /* File descriptor for WAL shared-memory file */ + char *zShm; /* Name of the shared-memory file for WAL mode */ + ssize_t got; /* Bytes read from header */ + int isWal; /* True if in WAL mode */ + int nName; /* Length of filename */ + unsigned char aHdr[100]; /* Database header */ + int nLock = 0; /* Number of locks held */ + int i; /* Loop counter */ + + if( argc!=2 ) usage(argv[0]); + hDb = open(argv[1], O_RDONLY, 0); + if( hDb<0 ){ + fprintf(stderr, "cannot open %s\n", argv[1]); + return 1; + } + + /* Make sure we are dealing with an database file */ + got = read(hDb, aHdr, 100); + if( got!=100 || memcmp(aHdr, "SQLite format 3",16)!=0 ){ + fprintf(stderr, "not an SQLite database: %s\n", argv[1]); + exit(1); + } + + /* First check for an exclusive lock */ + if( isLocked(hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, "EXCLUSIVE") ){ + return 0; + } + isWal = aHdr[18]==2; + if( isWal==0 ){ + /* Rollback mode */ + if( isLocked(hDb, F_RDLCK, PENDING_BYTE, 1, "PENDING") ) return 0; + if( isLocked(hDb, F_RDLCK, RESERVED_BYTE, 1, "RESERVED") ) return 0; + if( isLocked(hDb, F_WRLCK, SHARED_FIRST, SHARED_SIZE, "SHARED") ){ + return 0; + } + }else{ + /* WAL mode */ + nName = (int)strlen(argv[1]); + zShm = malloc( nName + 100 ); + if( zShm==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + memcpy(zShm, argv[1], nName); + memcpy(&zShm[nName], "-shm", 5); + hShm = open(zShm, O_RDONLY, 0); + if( hShm<0 ){ + fprintf(stderr, "cannot open %s\n", zShm); + return 1; + } + if( isLocked(hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") ){ + return 0; + } + nLock += isLocked(hShm, F_RDLCK, SHM_CHECKPOINT, 1, "WAL-CHECKPOINT"); + nLock += isLocked(hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE"); + for(i=0; i 0) || (lem.nconflict > 0)) ? 1 : 0; - successful_exit = (exitcode == 0); exit(exitcode); return (exitcode); } @@ -2761,23 +2741,6 @@ PRIVATE FILE *file_open( lemp->errorcnt++; return 0; } - - /* Add files we create to a list, so we can delete them if we fail. This - ** is to keep makefiles from getting confused. We don't include .out files, - ** though: this is debug information, and you don't want it deleted if there - ** was an error you need to track down. - */ - if(( *mode=='w' ) && (strcmp(suffix, ".out") != 0)){ - const char **ptr = (const char **) - realloc(made_files, sizeof (const char **) * (made_files_count + 1)); - const char *fname = Strsafe(lemp->outname); - if ((ptr == NULL) || (fname == NULL)) { - free(ptr); - memory_error(); - } - made_files = ptr; - made_files[made_files_count++] = fname; - } return fp; } @@ -3434,6 +3397,10 @@ void print_stack_union( /* Allocate and initialize types[] and allocate stddt[] */ arraysize = lemp->nsymbol * 2; types = (char**)calloc( arraysize, sizeof(char*) ); + if( types==0 ){ + fprintf(stderr,"Out of memory.\n"); + exit(1); + } for(i=0; ivartype ){ @@ -3447,7 +3414,7 @@ void print_stack_union( if( len>maxdtlength ) maxdtlength = len; } stddt = (char*)malloc( maxdtlength*2 + 1 ); - if( types==0 || stddt==0 ){ + if( stddt==0 ){ fprintf(stderr,"Out of memory.\n"); exit(1); } diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index fa99f2df..a834cab2 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -46,6 +46,8 @@ close $in # of the file. # set out [open sqlite3.c w] +# Force the output to use unix line endings, even on Windows. +fconfigure $out -translation lf set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1] puts $out [subst \ {/****************************************************************************** diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 554069c3..f68f61a3 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -65,6 +65,9 @@ close $in set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(} +# Force the output to use unix line endings, even on Windows. +fconfigure stdout -translation lf + # Process the src/sqlite.h.in ext/rtree/sqlite3rtree.h files. # foreach file [list $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h] { diff --git a/tool/omittest.tcl b/tool/omittest.tcl index f1963ff1..33a71cee 100644 --- a/tool/omittest.tcl +++ b/tool/omittest.tcl @@ -31,8 +31,8 @@ should work. The following properties are required: More precisely, the following two invocations must be supported: - make -f $::MAKEFILE testfixture OPTS="-DSQLITE_OMIT_ALTERTABLE=1" - make -f $::MAKEFILE test + $::MAKEBIN -f $::MAKEFILE testfixture OPTS="-DSQLITE_OMIT_ALTERTABLE=1" + $::MAKEBIN -f $::MAKEFILE test Makefiles generated by the sqlite configure program cannot be used as they do not respect the OPTS variable. @@ -73,7 +73,7 @@ catch { file copy -force ./libtool $dir } set rc [catch { - exec make -C $dir -f $::MAKEFILE $target OPTS=$opts >& $dir/build.log + exec $::MAKEBIN -C $dir -f $::MAKEFILE clean $target OPTS=$opts >& $dir/build.log }] if {$rc} { puts "No good. See $dir/build.log." @@ -102,7 +102,7 @@ catch { puts -nonewline "Testing $dir..." flush stdout set rc [catch { - exec make -C $dir -f $::MAKEFILE test OPTS=$opts >& $dir/test.log + exec $::MAKEBIN -C $dir -f $::MAKEFILE test OPTS=$opts >& $dir/test.log }] if {$rc} { puts "No good. See $dir/test.log." @@ -119,8 +119,9 @@ catch { # option. # proc process_options {argv} { + set ::MAKEBIN make ;# Default value if {$::tcl_platform(platform)=="windows" || $::tcl_platform(platform)=="os2"} { - set ::MAKEFILE ./Makefile ;# Default value + set ::MAKEFILE ./Makefile ;# Default value on Windows and OS2 } else { set ::MAKEFILE ./Makefile.linux-gcc ;# Default value } @@ -133,6 +134,11 @@ proc process_options {argv} { set ::MAKEFILE [lindex $argv $i] } + -nmake { + set ::MAKEBIN nmake + set ::MAKEFILE ./Makefile.msc + } + -skip_run { set ::SKIP_RUN 1 } @@ -251,7 +257,7 @@ proc main {argv} { exit -1 } - set dirname "test_[string range $sym 7 end]" + set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]" run_quick_test $dirname $sym } else { # First try a test with all OMIT symbols except SQLITE_OMIT_FLOATING_POINT @@ -270,14 +276,14 @@ proc main {argv} { # are the OMIT_FLOATING_POINT and OMIT_PRAGMA symbols, even though we # know they will fail. It's good to be reminded of this from time to time. foreach sym $::OMIT_SYMBOLS { - set dirname "test_[string range $sym 7 end]" + set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]" run_quick_test $dirname $sym } # Try the ENABLE/DISABLE symbols one at a time. # We don't do them all at once since some are conflicting. foreach sym $::ENABLE_SYMBOLS { - set dirname "test_[string range $sym 7 end]" + set dirname "test_[regsub -nocase {^x*SQLITE_} $sym {}]" run_quick_test $dirname $sym } } diff --git a/tool/shell1.test b/tool/shell1.test index a344ee62..9dd9df55 100644 --- a/tool/shell1.test +++ b/tool/shell1.test @@ -199,8 +199,9 @@ do_test shell1-1.15.3 { # -version show SQLite version do_test shell1-1.16.1 { - catchcmd "-version test.db" "" -} {0 3.7.6.2} + set x [catchcmd "-version test.db" ""] + regexp {0 \{3.\d.\d+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+\}} $x +} 1 #---------------------------------------------------------------------------- # Test cases shell1-2.*: Basic "dot" command token parsing. @@ -711,4 +712,9 @@ do_test shell1-3.27.4 { catchcmd "test.db" ".timer OFF BAD" } {1 {Error: unknown command or invalid arguments: "timer". Enter ".help" for help}} +do_test shell1-3-28.1 { + catchcmd test.db \ + ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" +} "0 {(123) hello\n456}" + puts "CLI tests completed successfully" diff --git a/tool/showdb.c b/tool/showdb.c index c954153c..057abd32 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -464,7 +464,7 @@ static void usage(const char *argv0){ " NNNbc Decode btree page NNN and show content\n" " NNNbm Decode btree page NNN and show a layout map\n" " NNNt Decode freelist trunk page NNN\n" - " NNNtd Show leave freelist pages on the decode\n" + " NNNtd Show leaf freelist pages on the decode\n" " NNNtr Recurisvely decode freelist starting at NNN\n" ); } diff --git a/tool/symbols.sh b/tool/symbols.sh new file mode 100644 index 00000000..8aec0056 --- /dev/null +++ b/tool/symbols.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# Run this script in a directory that contains a valid SQLite makefile in +# order to verify that unintentionally exported symbols. +# +make sqlite3.c + +echo '****** Exported symbols from a build including RTREE, FTS4 & ICU ******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_ENABLE_ICU \ + sqlite3.c +nm sqlite3.o | grep ' T ' | sort -k 3 + +echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' +nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_' + +echo '****** Dependencies of the core. No extensions. No OS interface *******' +gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + -DSQLITE_OS_OTHER -DSQLITE_THREADSAFE=0 \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 + +echo '****** Dependencies including RTREE & FTS4 *******' +gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ + -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ + sqlite3.c +nm sqlite3.o | grep ' U ' | sort -k 3 diff --git a/tool/tostr.awk b/tool/tostr.awk new file mode 100644 index 00000000..83c6cc1a --- /dev/null +++ b/tool/tostr.awk @@ -0,0 +1,9 @@ +#!/usr/bin/awk +# +# Convert input text into a C string +# +{ + gsub(/\\/,"\\\\"); + gsub(/\"/,"\\\""); + print "\"" $0 "\\n\""; +} diff --git a/tool/warnings.sh b/tool/warnings.sh new file mode 100644 index 00000000..e1fa2b2d --- /dev/null +++ b/tool/warnings.sh @@ -0,0 +1,14 @@ +#/bin/sh +# +# Run this script in a directory with a working makefile to check for +# compiler warnings in SQLite. +# +make sqlite3.c +echo '********** No optimizations. Includes FTS4 and RTREE *********' +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4 and RTREE *********' +gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ + sqlite3.c