signalapp/sqlcipher
signalapp/sqlcipher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge sqlite-release(3.11.0) into prerelease-integration

Browse Source
This commit is contained in:
Nick Parker 2016-02-22 11:20:06 -06:00
commit 3550fb400a
526 changed files with 94431 additions and 30321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
Makefile.arm-wince-mingw32ce-gcc
View File

@ -1,138 +0,0 @@
#!/usr/make
#
# Makefile for SQLITE
#
# This is a template makefile for SQLite. Most people prefer to
# use the autoconf generated "configure" script to generate the
# makefile automatically. But that does not work for everybody
# and in every situation. If you are having problems with the
# "configure" script, you might want to try this makefile as an
# alternative. Create a copy of this file, edit the parameters
# below and type "make".
#
#### The directory where to find the mingw32ce tools
MINGW32CE = /opt/mingw32ce/bin
#### The target prefix of the mingw32ce tools
TARGET = arm-wince-mingw32ce
#### The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = ../sqlite
#### C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = gcc -g -O2
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
USLEEP =
#USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
THREADSAFE = -DTHREADSAFE=1
#THREADSAFE = -DTHREADSAFE=0
#### Specify any extra linker options needed to make the library
# thread safe
#
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
#TLIBS = -lrt # fdatasync on Solaris 8
TLIBS =
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
#
# SQLite uses some expensive assert() statements in the inner loop.
# You can make the library go almost twice as fast if you compile
# with -DNDEBUG=1
#
#OPTS = -DSQLITE_DEBUG=2
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1 -DSQLITE_OS_WIN=1 -D_WIN32_WCE=1
#OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
#
EXE = .exe
#EXE =
#### C Compile and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
#TCC = gcc -O6
#TCC = gcc -g -O0 -Wall
#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
TCC = $(MINGW32CE)/$(TARGET)-gcc -O2
#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
#### Tools used to build a static library.
#
#AR = ar cr
#AR = /opt/mingw/bin/i386-mingw32-ar cr
AR = $(MINGW32CE)/$(TARGET)-ar cr
#RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
RANLIB = $(MINGW32CE)/$(TARGET)-ranlib
#MKSHLIB = gcc -shared
#SO = so
#SHPREFIX = lib
MKSHLIB = $(MINGW32CE)/$(TARGET)-gcc -shared
SO = dll
SHPREFIX =
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
TCLOBJ =
#TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
#### Linker options needed by programs using readline() must link against.
#
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
# You should not have to change anything below this line
###############################################################################
include $(TOP)/main.mk

228
Makefile.in
View File

@ -55,6 +55,7 @@ LIBTCL = @TCL_LIB_SPEC@
# Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
# The library that programs using readline() must link against.
#
@ -66,7 +67,7 @@ TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@
# Any target libraries which libsqlite must be linked against
#
TLIBS = @LIBS@
TLIBS = @LIBS@ $(LIBS)
# Flags controlling use of the in memory btree implementation
#
@ -174,9 +175,6 @@ LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS)
LTLINK = $(LIBTOOL) --mode=link $(TCC) $(LTCOMPILE_EXTRAS) @LDFLAGS@ $(LTLINK_EXTRAS)
LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL)
# nawk compatible awk.
NAWK = @AWK@
# You should not have to change anything below this line
###############################################################################
@ -192,18 +190,20 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
fts3_tokenize_vtab.lo \
fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \
fts5.lo \
func.lo global.lo hash.lo \
icu.lo insert.lo journal.lo legacy.lo loadext.lo \
icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \
main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
memjournal.lo \
mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \
table.lo threads.lo tokenize.lo trigger.lo \
random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \
table.lo threads.lo tokenize.lo treeview.lo trigger.lo \
update.lo util.lo vacuum.lo \
vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \
vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo $(CRYPTOLIBOBJ)
vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \
utf.lo vtab.lo $(CRYPTOLIBOBJ)
# Object files for the amalgamation.
#
@ -290,9 +290,10 @@ SRC = \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/table.c \
$(TOP)/src/threads.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/src/threads.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/treeview.c \
$(TOP)/src/trigger.c \
$(TOP)/src/utf.c \
$(TOP)/src/update.c \
@ -313,6 +314,8 @@ SRC = \
$(TOP)/src/wal.h \
$(TOP)/src/walker.c \
$(TOP)/src/where.c \
$(TOP)/src/wherecode.c \
$(TOP)/src/whereexpr.c \
$(TOP)/src/whereInt.h
# Source code for extensions
@ -359,6 +362,12 @@ SRC += \
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
SRC += \
$(TOP)/ext/rbu/sqlite3rbu.h \
$(TOP)/ext/rbu/sqlite3rbu.c
SRC += \
$(TOP)/ext/misc/json1.c
# Generated source code files
@ -413,9 +422,11 @@ TESTSRC = \
$(TOP)/src/test_tclvar.c \
$(TOP)/src/test_thread.c \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_windirent.c \
$(TOP)/src/test_wsd.c \
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_test.c
$(TOP)/ext/fts3/fts3_test.c \
$(TOP)/ext/rbu/test_rbu.c
# Statically linked extensions
#
@ -425,10 +436,14 @@ TESTSRC += \
$(TOP)/ext/misc/eval.c \
$(TOP)/ext/misc/fileio.c \
$(TOP)/ext/misc/fuzzer.c \
$(TOP)/ext/fts5/fts5_tcl.c \
$(TOP)/ext/fts5/fts5_test_mi.c \
$(TOP)/ext/fts5/fts5_test_tok.c \
$(TOP)/ext/misc/ieee754.c \
$(TOP)/ext/misc/nextchar.c \
$(TOP)/ext/misc/percentile.c \
$(TOP)/ext/misc/regexp.c \
$(TOP)/ext/misc/series.c \
$(TOP)/ext/misc/spellfix.c \
$(TOP)/ext/misc/totype.c \
$(TOP)/ext/misc/wholenumber.c
@ -470,6 +485,8 @@ TESTSRC2 = \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbetrace.c \
$(TOP)/src/where.c \
$(TOP)/src/wherecode.c \
$(TOP)/src/whereexpr.c \
parse.c \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3_aux.c \
@ -530,6 +547,33 @@ EXTHDR += \
EXTHDR += \
$(TOP)/ext/rtree/sqlite3rtree.h
# executables needed for testing
#
TESTPROGS = \
testfixture$(TEXE) \
sqlite3$(TEXE) \
sqlite3_analyzer$(TEXE) \
sqldiff$(TEXE)
# Databases containing fuzzer test cases
#
FUZZDATA = \
$(TOP)/test/fuzzdata1.db \
$(TOP)/test/fuzzdata2.db \
$(TOP)/test/fuzzdata3.db \
$(TOP)/test/fuzzdata4.db
# Standard options to testfixture
#
TESTOPTS = --verbose=file --output=test-out.txt
# Extra compiler options for various shell tools
#
SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
@ -553,15 +597,25 @@ libtclsqlite3.la: tclsqlite.lo libsqlcipher.la
-avoid-version
sqlcipher$(TEXE): $(TOP)/src/shell.c libsqlcipher.la sqlite3.h
$(LTLINK) $(READLINE_FLAGS) \
$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) \
-o $@ $(TOP)/src/shell.c libsqlcipher.la \
$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS)
srcck1$(BEXE): $(TOP)/tool/srcck1.c
$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c
sourcetest: srcck1$(BEXE) sqlite3.c
./srcck1 sqlite3.c
fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
$(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h
$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS)
mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c
$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \
@ -587,19 +641,23 @@ mptest: mptester$(TEXE)
# files are automatically generated. This target takes care of
# all that automatic generation.
#
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl
.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
rm tsrc/sqlite.h.in tsrc/parse.y
$(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl $(OPTS) <tsrc/vdbe.c >vdbe.new
mv vdbe.new tsrc/vdbe.c
cp fts5.c fts5.h tsrc
touch .target_source
sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl
$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl
cp tsrc/shell.c tsrc/sqlite3ext.h .
sqlite3ext.h: .target_source
cp tsrc/sqlite3ext.h .
tclsqlite3.c: sqlite3.c
echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c
cat sqlite3.c >>tclsqlite3.c
@ -616,9 +674,9 @@ sqlite3.lo: sqlite3.c
# Rules to build the LEMON compiler generator
#
lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/src/lempar.c
lemon$(BEXE): $(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
$(BCC) -o $@ $(TOP)/tool/lemon.c
cp $(TOP)/src/lempar.c .
cp $(TOP)/tool/lempar.c .
# Rules to build individual *.o files from generated *.c files. This
# applies to:
@ -812,6 +870,9 @@ threads.lo: $(TOP)/src/threads.c $(HDR)
tokenize.lo: $(TOP)/src/tokenize.c keywordhash.h $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c
treeview.lo: $(TOP)/src/treeview.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/treeview.c
trigger.lo: $(TOP)/src/trigger.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/trigger.c
@ -860,6 +921,12 @@ walker.lo: $(TOP)/src/walker.c $(HDR)
where.lo: $(TOP)/src/where.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/where.c
wherecode.lo: $(TOP)/src/wherecode.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/wherecode.c
whereexpr.lo: $(TOP)/src/whereexpr.c $(HDR)
$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/whereexpr.c
tclsqlite.lo: $(TOP)/src/tclsqlite.c $(HDR)
$(LTCOMPILE) -DUSE_TCL_STUBS=1 -c $(TOP)/src/tclsqlite.c
@ -875,22 +942,22 @@ tclsqlcipher$(TEXE): tclsqlite-shell.lo libsqlcipher.la
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
$(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c
opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl
$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
#
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk
parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl
cp $(TOP)/src/parse.y .
rm -f parse.h
./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y
mv parse.h parse.h.temp
$(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
$(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h
sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION
$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
@ -966,6 +1033,45 @@ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
json1.lo: $(TOP)/ext/misc/json1.c
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c
# FTS5 things
#
FTS5_SRC = \
$(TOP)/ext/fts5/fts5.h \
$(TOP)/ext/fts5/fts5Int.h \
$(TOP)/ext/fts5/fts5_aux.c \
$(TOP)/ext/fts5/fts5_buffer.c \
$(TOP)/ext/fts5/fts5_main.c \
$(TOP)/ext/fts5/fts5_config.c \
$(TOP)/ext/fts5/fts5_expr.c \
$(TOP)/ext/fts5/fts5_hash.c \
$(TOP)/ext/fts5/fts5_index.c \
fts5parse.c fts5parse.h \
$(TOP)/ext/fts5/fts5_storage.c \
$(TOP)/ext/fts5/fts5_tokenize.c \
$(TOP)/ext/fts5/fts5_unicode2.c \
$(TOP)/ext/fts5/fts5_varint.c \
$(TOP)/ext/fts5/fts5_vocab.c \
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
./lemon $(OPTS) fts5parse.y
fts5parse.h: fts5parse.c
fts5.c: $(FTS5_SRC)
$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
cp $(TOP)/ext/fts5/fts5.h .
fts5.lo: fts5.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c fts5.c
sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c
# Rules to build the 'testfixture' application.
#
@ -988,51 +1094,58 @@ testfixture$(TEXE): $(TESTFIXTURE_SRC)
-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
# A very detailed test running most or all test cases
fulltest: testfixture$(TEXE) sqlcipher$(TEXE) fuzztest
./testfixture$(TEXE) $(TOP)/test/all.test
fulltest: $(TESTPROGS) fuzztest
./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
# Really really long testing
soaktest: testfixture$(TEXE) sqlcipher$(TEXE) fuzzoomtest
./testfixture$(TEXE) $(TOP)/test/all.test -soak=1
soaktest: $(TESTPROGS)
./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 $(TESTOPTS)
# Do extra testing but not aeverything.
fulltestonly: testfixture$(TEXE) sqlcipher$(TEXE)
# Do extra testing but not everything.
fulltestonly: $(TESTPROGS) fuzztest
./testfixture$(TEXE) $(TOP)/test/full.test
# Fuzz testing
fuzztest: fuzzershell$(TEXE)
./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt
fuzztest: fuzzcheck$(TEXE) $(FUZZDATA)
./fuzzcheck$(TEXE) $(FUZZDATA)
fuzzoomtest: fuzzershell$(TEXE)
./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom
fastfuzztest: fuzzcheck$(TEXE) $(FUZZDATA)
./fuzzcheck$(TEXE) --limit-mem 100M $(FUZZDATA)
# This is the common case. Run many tests but not those that take
# a really long time.
valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA)
valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA)
# Minimal testing that runs in less than 3 minutes
#
test: testfixture$(TEXE) sqlcipher$(TEXE) fuzztest
./testfixture$(TEXE) $(TOP)/test/veryquick.test
quicktest: ./testfixture$(TEXE)
./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)
# This is the common case. Run many tests that do not take too long,
# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
#
test: $(TESTPROGS) sourcetest fastfuzztest
./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
# Run a test using valgrind. This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest: testfixture$(TEXE) sqlite3$(TEXE) fuzzershell$(TEXE)
valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt
OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind
valgrindtest: $(TESTPROGS) valgrindfuzz
OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind $(TESTOPTS)
# A very fast test that checks basic sanity. The name comes from
# the 60s-era electronics testing: "Turn it on and see if smoke
# comes out."
#
smoketest: testfixture$(TEXE) fuzzershell$(TEXE)
./testfixture$(TEXE) $(TOP)/test/main.test
smoketest: $(TESTPROGS) fuzzcheck$(TEXE)
./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)
sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
echo "#define TCLSH 2" > $@
echo "#define SQLITE_ENABLE_DBSTAT_VTAB" >> $@
echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
echo "static const char *tclsh_main_loop(void){" >> $@
echo "static const char *zMainloop = " >> $@
$(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@
$(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@
echo "; return zMainloop; }" >> $@
sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
@ -1062,6 +1175,12 @@ wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c
speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo
$(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS)
rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
$(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS)
# This target will fail if the SQLite amalgamation contains any exported
# symbols that do not begin with "sqlite3_". It is run as part of the
# releasetest.tcl script.
@ -1070,10 +1189,15 @@ checksymbols: sqlite3.lo
nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
echo '0 errors out of 1 tests'
# Build the amalgamation-autoconf package.
# Build the amalgamation-autoconf package. The amalamgation-tarball target builds
# a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz.
# The snapshot-tarball target builds a tarball named by the SHA1 hash
#
amalgamation-tarball: sqlite3.c
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --normal
snapshot-tarball: sqlite3.c
TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
@ -1087,7 +1211,7 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c \
$(TOP)/test/tt3_lookaside1.c
threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
$(LTLINK) $(TOP)/test/threadtest3.c sqlite3.lo -o $@ $(TLIBS)
$(LTLINK) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.lo -o $@ $(TLIBS)
threadtest: threadtest3$(TEXE)
./threadtest3$(TEXE)
@ -1101,9 +1225,9 @@ lib_install: libsqlcipher.la
$(INSTALL) -d $(DESTDIR)$(libdir)
$(LTINSTALL) libsqlcipher.la $(DESTDIR)$(libdir)
install: sqlcipher$(BEXE) lib_install sqlite3.h sqlcipher.pc ${HAVE_TCL:1=tcl_install}
install: sqlcipher$(TEXE) lib_install sqlite3.h sqlcipher.pc ${HAVE_TCL:1=tcl_install}
$(INSTALL) -d $(DESTDIR)$(bindir)
$(LTINSTALL) sqlcipher$(BEXE) $(DESTDIR)$(bindir)
$(LTINSTALL) sqlcipher$(TEXE) $(DESTDIR)$(bindir)
$(INSTALL) -d $(DESTDIR)$(includedir)
$(INSTALL) -m 0644 sqlite3.h $(DESTDIR)$(includedir)
$(INSTALL) -m 0644 $(TOP)/src/sqlite3ext.h $(DESTDIR)$(includedir)
@ -1139,9 +1263,15 @@ clean:
rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
rm -f sqlite-*-output.vsix
rm -f mptester mptester.exe
rm -f rbu rbu.exe
rm -f srcck1 srcck1.exe
rm -f fuzzershell fuzzershell.exe
rm -f fuzzcheck fuzzcheck.exe
rm -f sqldiff sqldiff.exe
rm -f fts5.* fts5parse.*
distclean: clean
rm -f config.log config.status libtool Makefile sqlcipher.pc
rm -f config.h config.log config.status libtool Makefile sqlcipher.pc
#
# Windows section

5
Makefile.linux-gcc
View File

@ -118,11 +118,6 @@ READLINE_FLAGS =
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
# You should not have to change anything below this line
###############################################################################
include $(TOP)/main.mk

797
Makefile.msc
View File

File diff suppressed because it is too large Load Diff

673
Makefile.vxworks
View File

@ -1,673 +0,0 @@
#!/usr/make
#
# Makefile for SQLITE on VxWorks
ifeq ($(FORCPU),)
FORCPU = SH32gnule
endif
TOOL_FAMILY = gnu
include $(WIND_USR)/tool/gnu/make.$(FORCPU)
#### The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.in" and the "configure.in" script.
#
TOP = .
#### C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
BCC = gcc -g -O2
#BCC = /opt/ancic/bin/c89 -0
#### If the target operating system supports the "usleep()" system
# call, then define the HAVE_USLEEP macro for all C modules.
#
USLEEP =
#USLEEP = -DHAVE_USLEEP=1
#### If you want the SQLite library to be safe for use within a
# multi-threaded program, then define the following macro
# appropriately:
#
THREADSAFE = -DSQLITE_THREADSAFE=1
#THREADSAFE = -DSQLITE_THREADSAFE=0
#### Specify any extra linker options needed to make the library
# thread safe
#
#THREADLIB = -lpthread
THREADLIB =
#### Specify any extra libraries needed to access required functions.
#
ifeq ($(CPU),SH32)
# for SH4 shared library
TLIBS_SHARED += -L$(WIND_USR)/lib/sh/SH32/commonle/PIC
else
# for all other CPUs shared library
TLIBS_SHARED += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
endif
# for static library
TLIBS += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS)
#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1
# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all
# malloc()s and free()s in order to track down memory leaks.
#
# SQLite uses some expensive assert() statements in the inner loop.
# You can make the library go almost twice as fast if you compile
# with -DNDEBUG=1
#
#OPTS = -DSQLITE_DEBUG=2
#OPTS = -DSQLITE_DEBUG=1
#OPTS =
OPTS = -DNDEBUG=1 -DSQLITE_OS_UNIX=1 $(THREADSAFE)
OPTS += -DSQLITE_OMIT_LOAD_EXTENSION=1
OPTS += -DSQLITE_ENABLE_LOCKING_STYLE=1
OPTS += -DSQLITE_THREAD_OVERRIDE_LOCK=0
OPTS += -DSQLITE_ENABLE_COLUMN_METADATA=1
OPTS += -DHAVE_FDATASYNC=1
#### The suffix to add to executable files. ".exe" for windows.
# Nothing for unix.
#
EXE = .vxe
#EXE =
#### C Compile and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
#TCC = gcc -O6
#TCC = gcc -g -O0 -Wall
#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage
#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6
TCC = $(CC) $(DEFINE_CC) -O2 -g -mrtp $(CC_ARCH_SPEC) -D_REENTRANT=1 -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
TCC += -I$(WIND_USR)/h -I$(WIND_USR)/h/wrn/coreip
#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive
#TCC_SHARED = $(TCC) -fPIC
TCC_SHARED = $(TCC)
#### Tools used to build a static library.
#
#ARX = ar cr
#ARX = /opt/mingw/bin/i386-mingw32-ar cr
AR += cr
#RANLIB = ranlib
#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib
#MKSHLIB = gcc -shared
#SO = so
#SHPREFIX = lib
MKSHLIB = $(CC) $(DEFINE_CC) -mrtp -shared $(CC_ARCH_SPEC) -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL)
SO = so
SHPREFIX = lib
#### Extra compiler options needed for programs that use the TCL library.
#
#TCL_FLAGS =
#TCL_FLAGS = -DSTATIC_BUILD=1
TCL_FLAGS = -I/home/drh/tcltk/8.5linux
#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1
#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux
#### Linker options needed to link against the TCL library.
#
#LIBTCL = -ltcl -lm -ldl
LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl
#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt
#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc
#### Additional objects for SQLite library when TCL support is enabled.
TCLOBJ =
#TCLOBJ = tclsqlite.o
#### Compiler options needed for programs that use the readline() library.
#
READLINE_FLAGS =
#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline
#### Linker options needed by programs using readline() must link against.
#
LIBREADLINE =
#LIBREADLINE = -static -lreadline -ltermcap
#### Which "awk" program provides nawk compatibilty
#
# NAWK = nawk
NAWK = awk
#### Pasted and adapted main.mk file
###############################################################################
# The following macros should be defined before this script is
# invoked:
#
# TOP The toplevel directory of the source tree. This is the
# directory that contains this "Makefile.in" and the
# "configure.in" script.
#
# BCC C Compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
# THREADLIB Specify any extra linker options needed to make the library
# thread safe
#
# OPTS Extra compiler command-line options.
#
# EXE The suffix to add to executable files. ".exe" for windows
# and "" for Unix.
#
# TCC C Compiler and options for use in building executables that
# will run on the target platform. This is usually the same
# as BCC, unless you are cross-compiling.
#
# AR Tools used to build a static library.
# RANLIB
#
# TCL_FLAGS Extra compiler options needed for programs that use the
# TCL library.
#
# LIBTCL Linker options needed to link against the TCL library.
#
# READLINE_FLAGS Compiler options needed for programs that use the
# readline() library.
#
# LIBREADLINE Linker options needed by programs using readline() must
# link against.
#
# NAWK Nawk compatible awk program. Older (obsolete?) solaris
# systems need this to avoid using the original AT&T AWK.
#
# Once the macros above are defined, the rest of this make script will
# build the SQLite library and testing tools.
################################################################################
# This is how we compile
#
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX_SHARED = $(TCC_SHARED) $(OPTS) -I. -I$(TOP)/src -I$(TOP) \
-I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 \
-I$(TOP)/ext/async
# Object files for the SQLite library.
#
LIBOBJ+= alter.o analyze.o attach.o auth.o \
backup.o bitvec.o btmutex.o btree.o build.o \
callback.o complete.o date.o delete.o expr.o fault.o \
fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
fts3_tokenizer.o fts3_tokenizer1.o \
func.o global.o hash.o \
icu.o insert.o journal.o legacy.o loadext.o \
main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
memjournal.o \
mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \
notify.o opcodes.o os.o os_unix.o os_win.o \
pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
random.o resolve.o rowset.o rtree.o select.o status.o \
table.o tokenize.o trigger.o \
update.o util.o vacuum.o \
vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \
walker.o where.o utf.o vtab.o
# All of the source code files.
#
SRC = \
$(TOP)/src/alter.c \
$(TOP)/src/analyze.c \
$(TOP)/src/attach.c \
$(TOP)/src/auth.c \
$(TOP)/src/backup.c \
$(TOP)/src/bitvec.c \
$(TOP)/src/btmutex.c \
$(TOP)/src/btree.c \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/build.c \
$(TOP)/src/callback.c \
$(TOP)/src/complete.c \
$(TOP)/src/ctime.c \
$(TOP)/src/date.c \
$(TOP)/src/delete.c \
$(TOP)/src/expr.c \
$(TOP)/src/fault.c \
$(TOP)/src/func.c \
$(TOP)/src/global.c \
$(TOP)/src/hash.c \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
$(TOP)/src/insert.c \
$(TOP)/src/journal.c \
$(TOP)/src/legacy.c \
$(TOP)/src/loadext.c \
$(TOP)/src/main.c \
$(TOP)/src/malloc.c \
$(TOP)/src/mem0.c \
$(TOP)/src/mem1.c \
$(TOP)/src/mem2.c \
$(TOP)/src/mem3.c \
$(TOP)/src/mem5.c \
$(TOP)/src/memjournal.c \
$(TOP)/src/msvc.h \
$(TOP)/src/mutex.c \
$(TOP)/src/mutex.h \
$(TOP)/src/mutex_noop.c \
$(TOP)/src/mutex_unix.c \
$(TOP)/src/mutex_w32.c \
$(TOP)/src/notify.c \
$(TOP)/src/os.c \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_setup.h \
$(TOP)/src/os_unix.c \
$(TOP)/src/os_win.c \
$(TOP)/src/os_win.h \
$(TOP)/src/pager.c \
$(TOP)/src/pager.h \
$(TOP)/src/parse.y \
$(TOP)/src/pcache.c \
$(TOP)/src/pcache.h \
$(TOP)/src/pcache1.c \
$(TOP)/src/pragma.c \
$(TOP)/src/prepare.c \
$(TOP)/src/printf.c \
$(TOP)/src/random.c \
$(TOP)/src/resolve.c \
$(TOP)/src/rowset.c \
$(TOP)/src/select.c \
$(TOP)/src/status.c \
$(TOP)/src/shell.c \
$(TOP)/src/sqlite.h.in \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/table.c \
$(TOP)/src/tclsqlite.c \
$(TOP)/src/tokenize.c \
$(TOP)/src/trigger.c \
$(TOP)/src/utf.c \
$(TOP)/src/update.c \
$(TOP)/src/util.c \
$(TOP)/src/vacuum.c \
$(TOP)/src/vdbe.c \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeapi.c \
$(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbeblob.c \
$(TOP)/src/vdbemem.c \
$(TOP)/src/vdbeInt.h \
$(TOP)/src/vtab.c \
$(TOP)/src/walker.c \
$(TOP)/src/where.c
# Source code for extensions
#
SRC += \
$(TOP)/ext/fts1/fts1.c \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.c \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_porter.c \
$(TOP)/ext/fts1/fts1_tokenizer.h \
$(TOP)/ext/fts1/fts1_tokenizer1.c
SRC += \
$(TOP)/ext/fts2/fts2.c \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.c \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_icu.c \
$(TOP)/ext/fts2/fts2_porter.c \
$(TOP)/ext/fts2/fts2_tokenizer.h \
$(TOP)/ext/fts2/fts2_tokenizer.c \
$(TOP)/ext/fts2/fts2_tokenizer1.c
SRC += \
$(TOP)/ext/fts3/fts3.c \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_expr.h \
$(TOP)/ext/fts3/fts3_hash.c \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_icu.c \
$(TOP)/ext/fts3/fts3_porter.c \
$(TOP)/ext/fts3/fts3_tokenizer.h \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/fts3/fts3_tokenizer1.c
SRC += \
$(TOP)/ext/icu/sqliteicu.h \
$(TOP)/ext/icu/icu.c
SRC += \
$(TOP)/ext/rtree/rtree.h \
$(TOP)/ext/rtree/rtree.c
# Generated source code files
#
SRC += \
keywordhash.h \
opcodes.c \
opcodes.h \
parse.c \
parse.h \
sqlite3.h
# Source code to the test files.
#
TESTSRC = \
$(TOP)/src/test1.c \
$(TOP)/src/test2.c \
$(TOP)/src/test3.c \
$(TOP)/src/test4.c \
$(TOP)/src/test5.c \
$(TOP)/src/test6.c \
$(TOP)/src/test7.c \
$(TOP)/src/test8.c \
$(TOP)/src/test9.c \
$(TOP)/src/test_autoext.c \
$(TOP)/src/test_async.c \
$(TOP)/src/test_backup.c \
$(TOP)/src/test_btree.c \
$(TOP)/src/test_config.c \
$(TOP)/src/test_devsym.c \
$(TOP)/src/test_func.c \
$(TOP)/src/test_hexio.c \
$(TOP)/src/test_journal.c \
$(TOP)/src/test_malloc.c \
$(TOP)/src/test_md5.c \
$(TOP)/src/test_mutex.c \
$(TOP)/src/test_onefile.c \
$(TOP)/src/test_osinst.c \
$(TOP)/src/test_pcache.c \
$(TOP)/src/test_schema.c \
$(TOP)/src/test_server.c \
$(TOP)/src/test_tclvar.c \
$(TOP)/src/test_thread.c \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_wsd.c \
#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c
#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c
TESTSRC2 = \
$(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \
$(TOP)/src/build.c $(TOP)/src/ctime.c $(TOP)/src/date.c \
$(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \
$(TOP)/src/os_unix.c $(TOP)/src/os_win.c \
$(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \
$(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c \
$(TOP)/src/pcache1.c $(TOP)/src/select.c $(TOP)/src/tokenize.c \
$(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \
$(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c \
$(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_expr.c \
$(TOP)/ext/fts3/fts3_tokenizer.c \
$(TOP)/ext/async/sqlite3async.c
# Header files used by all library source files.
#
HDR = \
$(TOP)/src/btree.h \
$(TOP)/src/btreeInt.h \
$(TOP)/src/hash.h \
$(TOP)/src/hwtime.h \
keywordhash.h \
$(TOP)/src/msvc.h \
$(TOP)/src/mutex.h \
opcodes.h \
$(TOP)/src/os.h \
$(TOP)/src/os_common.h \
$(TOP)/src/os_setup.h \
$(TOP)/src/os_win.h \
$(TOP)/src/pager.h \
$(TOP)/src/pcache.h \
parse.h \
sqlite3.h \
$(TOP)/src/sqlite3ext.h \
$(TOP)/src/sqliteInt.h \
$(TOP)/src/sqliteLimit.h \
$(TOP)/src/vdbe.h \
$(TOP)/src/vdbeInt.h
# Header files used by extensions
#
EXTHDR += \
$(TOP)/ext/fts1/fts1.h \
$(TOP)/ext/fts1/fts1_hash.h \
$(TOP)/ext/fts1/fts1_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts2/fts2.h \
$(TOP)/ext/fts2/fts2_hash.h \
$(TOP)/ext/fts2/fts2_tokenizer.h
EXTHDR += \
$(TOP)/ext/fts3/fts3.h \
$(TOP)/ext/fts3/fts3_expr.h \
$(TOP)/ext/fts3/fts3_hash.h \
$(TOP)/ext/fts3/fts3_tokenizer.h
EXTHDR += \
$(TOP)/ext/rtree/rtree.h
EXTHDR += \
$(TOP)/ext/icu/sqliteicu.h
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: sqlite3.h libsqlite3.a sqlite3$(EXE)
libsqlite3.a: $(LIBOBJ)
$(AR) libsqlite3.a $(LIBOBJ)
$(RANLIB) libsqlite3.a
$(SHPREFIX)sqlite3.$(SO): $(LIBOBJ)
$(MKSHLIB) -o $(SHPREFIX)sqlite3.$(SO) $(LIBOBJ) $(TLIBS_SHARED)
sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h
$(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \
$(TOP)/src/shell.c \
$(LIBREADLINE) $(TLIBS) $(THREADLIB) -L. -lsqlite3
# This target creates a directory named "tsrc" and fills it with
# copies of all of the C source code and header files needed to
# build on the target system. Some of the C source code and header
# files are automatically generated. This target takes care of
# all that automatic generation.
#
target_source: $(SRC)
rm -rf tsrc
mkdir tsrc
cp -f $(SRC) tsrc
rm tsrc/sqlite.h.in tsrc/parse.y
touch target_source
sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl
tclsh $(TOP)/tool/mksqlite3c.tcl
cp sqlite3.c tclsqlite3.c
cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl
tclsh $(TOP)/ext/fts2/mkfts2amal.tcl
fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl
tclsh $(TOP)/ext/fts3/mkfts3amal.tcl
# Rules to build the LEMON compiler generator
#
lemon: $(TOP)/tool/lemon.c $(TOP)/src/lempar.c
$(BCC) -o lemon $(TOP)/tool/lemon.c
cp $(TOP)/src/lempar.c .
# Rules to build individual *.o files from generated *.c files. This
# applies to:
#
# parse.o
# opcodes.o
#
%.o: %.c $(HDR)
$(TCCX_SHARED) -c $<
# Rules to build individual *.o files from files in the src directory.
#
%.o: $(TOP)/src/%.c $(HDR)
$(TCCX_SHARED) -c $<
tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR)
$(TCCX_SHARED) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c
# Rules to build opcodes.c and opcodes.h
#
opcodes.c: opcodes.h $(TOP)/mkopcodec.awk
$(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c
opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk
cat parse.h $(TOP)/src/vdbe.c | \
$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h
# Rules to build parse.c and parse.h - the outputs of lemon.
#
parse.h: parse.c
parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
cp $(TOP)/src/parse.y .
rm -f parse.h
./lemon $(OPTS) parse.y
mv parse.h parse.h.temp
awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
sqlite3.h: $(TOP)/src/sqlite.h.in
sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \
-e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | $(NAWK) '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \
$(TOP)/src/sqlite.h.in >sqlite3.h
keywordhash.h: $(TOP)/tool/mkkeywordhash.c
$(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c
./mkkeywordhash >keywordhash.h
# Rules to build the extension objects.
#
icu.o: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c
fts2.o: $(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
fts2_hash.o: $(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
fts2_icu.o: $(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
fts2_porter.o: $(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
fts2_tokenizer.o: $(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c
fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c
fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c
fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c
fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c
fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c
fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
# Rules for building test programs and for running tests
#
tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a
$(TCCX_SHARED) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \
$(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB)
# Rules to build the 'testfixture' application.
#
TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a
amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
-DSQLITE_ENABLE_FTS3=1 \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \
-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
fulltest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test
soaktest: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/all.test -soak=1
fulltestonly: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/full.test
test: testfixture$(EXE) sqlite3$(EXE)
./testfixture$(EXE) $(TOP)/test/veryquick.test
sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \
$(TOP)/tool/spaceanal.tcl
sed \
-e '/^#/d' \
-e 's,\\,\\\\,g' \
-e 's,",\\",g' \
-e 's,^,",' \
-e 's,$$,\\n",' \
$(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h
$(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \
-DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_DEBUG=1 -DSQLITE_PRIVATE="" \
$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \
-o sqlite3_analyzer$(EXE) \
$(LIBTCL) $(THREADLIB)
TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO)
$(TEST_EXTENSION): $(TOP)/src/test_loadext.c
$(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION)
extensiontest: testfixture$(EXE) $(TEST_EXTENSION)
./testfixture$(EXE) $(TOP)/test/loadext.test
clean:
rm -f *.o sqlite3$(EXE) libsqlite3.a sqlite3.h opcodes.*
rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h
rm -f $(PUBLISH)
rm -f *.da *.bb *.bbg gmon.out
rm -rf quota2a quota2b quota2c
rm -rf tsrc target_source
rm -f testloadext.dll libtestloadext.so
rm -f sqlite3.c fts?amal.c tclsqlite3.c
rm -f sqlite3rc.h
rm -f shell.c sqlite3ext.h
rm -f $(SHPREFIX)sqlite3.$(SO)

19
README.md
View File

@ -24,6 +24,10 @@ an iPhone data vault and password manager (http://getstrip.com).
We welcome contributions, to contribute to SQLCipher, a [contributor agreement](https://www.zetetic.net/contributions/) needs to be submitted. All submissions should be based on the `prerelease` branch.
If you are reading this on a Git mirror someplace, you are doing it wrong.
The [official repository](https://www.sqlite.org/src/) is better. Go there
now.
## Compiling
Building SQLCipher is almost the same as compiling a regular version of
@ -305,7 +309,7 @@ complex code. So there is a lot of complexity in the SQLite implementation.
Key files:
* **sqlite3.h** - This file defines the public interface to the SQLite
* **sqlite.h.in** - This file defines the public interface to the SQLite
library. Readers will need to be familiar with this interface before
trying to understand how the library works internally.
@ -313,7 +317,7 @@ Key files:
used internally by SQLite.
* **parse.y** - This file describes the LALR(1) grammer that SQLite uses
to parse SQL statements, and the actions that are taken at each stop
to parse SQL statements, and the actions that are taken at each step
in the parsing process.
* **vdbe.c** - This file implements the virtual machine that runs
@ -337,6 +341,17 @@ Key files:
between SQLite and the underlying operating system using the run-time
pluggable VFS interface.
* **shell.c** - This file is not part of the core SQLite library. This
is the file that, when linked against sqlite3.a, generates the
"sqlite3.exe" command-line shell.
* **tclsqlite.c** - This file implements the Tcl bindings for SQLite. It
is not part of the core SQLite library. But as most of the tests in this
repository are written in Tcl, the Tcl language bindings are important.
There are many other source files. Each has a suscinct header comment that
describes its purpose and role within the larger system.
## Contacts

2
VERSION
View File

@ -1 +1 @@
3.8.10.2
3.11.0

34
addopcodes.awk
View File

@ -1,34 +0,0 @@
#!/usr/bin/awk
#
# This script appends additional token codes to the end of the
# parse.h file that lemon generates. These extra token codes are
# not used by the parser. But they are used by the tokenizer and/or
# the code generator.
#
#
BEGIN {
max = 0
}
/^#define TK_/ {
print $0
if( max<$3 ) max = $3
}
END {
printf "#define TK_%-29s %4d\n", "TO_TEXT", ++max
printf "#define TK_%-29s %4d\n", "TO_BLOB", ++max
printf "#define TK_%-29s %4d\n", "TO_NUMERIC", ++max
printf "#define TK_%-29s %4d\n", "TO_INT", ++max
printf "#define TK_%-29s %4d\n", "TO_REAL", ++max
printf "#define TK_%-29s %4d\n", "ISNOT", ++max
printf "#define TK_%-29s %4d\n", "END_OF_FILE", ++max
printf "#define TK_%-29s %4d\n", "ILLEGAL", ++max
printf "#define TK_%-29s %4d\n", "SPACE", ++max
printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", ++max
printf "#define TK_%-29s %4d\n", "FUNCTION", ++max
printf "#define TK_%-29s %4d\n", "COLUMN", ++max
printf "#define TK_%-29s %4d\n", "AGG_FUNCTION", ++max
printf "#define TK_%-29s %4d\n", "AGG_COLUMN", ++max
printf "#define TK_%-29s %4d\n", "UMINUS", ++max
printf "#define TK_%-29s %4d\n", "UPLUS", ++max
printf "#define TK_%-29s %4d\n", "REGISTER", ++max
}

10
autoconf/Makefile.am
View File

@ -1,17 +1,19 @@
AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE
lib_LTLIBRARIES = libsqlite3.la
libsqlite3_la_SOURCES = sqlite3.c
libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8
bin_PROGRAMS = sqlite3
sqlite3_SOURCES = shell.c sqlite3.h
sqlite3_LDADD = sqlite3.$(OBJEXT) @READLINE_LIBS@
sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h
sqlite3_LDADD = @READLINE_LIBS@
sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
include_HEADERS = sqlite3.h sqlite3ext.h
EXTRA_DIST = sqlite3.1 tea
EXTRA_DIST = sqlite3.1 tea Makefile.msc sqlite3.rc README.txt
pkgconfigdir = ${libdir}/pkgconfig
pkgconfig_DATA = sqlite3.pc

921
autoconf/Makefile.msc Normal file
View File

@ -0,0 +1,921 @@
#### DO NOT EDIT ####
# This makefile is automatically generated from the Makefile.msc at
# the root of the canonical SQLite source tree (not the
# amalgamation tarball) using the tool/mkmsvcmin.tcl
# script.
#
#
# nmake Makefile for SQLite
#
###############################################################################
############################## START OF OPTIONS ###############################
###############################################################################
# The toplevel directory of the source tree. This is the directory
# that contains this "Makefile.msc".
#
TOP = .
# Set this non-0 to enable full warnings (-W4, etc) when compiling.
#
!IFNDEF USE_FULLWARN
USE_FULLWARN = 0
!ENDIF
# Set this non-0 to use "stdcall" calling convention for the core library
# and shell executable.
#
!IFNDEF USE_STDCALL
USE_STDCALL = 0
!ENDIF
# Set this non-0 to have the shell executable link against the core dynamic
# link library.
#
!IFNDEF DYNAMIC_SHELL
DYNAMIC_SHELL = 0
!ENDIF
# Set this non-0 to enable extra code that attempts to detect misuse of the
# SQLite API.
#
!IFNDEF API_ARMOR
API_ARMOR = 0
!ENDIF
# If necessary, create a list of harmless compiler warnings to disable when
# compiling the various tools. For the SQLite source code itself, warnings,
# if any, will be disabled from within it.
#
!IFNDEF NO_WARN
!IF $(USE_FULLWARN)!=0
NO_WARN = -wd4054 -wd4055 -wd4100 -wd4127 -wd4130 -wd4152 -wd4189 -wd4206
NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4305 -wd4306 -wd4702 -wd4706
!ENDIF
!ENDIF
# Set this non-0 to use the library paths and other options necessary for
# Windows Phone 8.1.
#
!IFNDEF USE_WP81_OPTS
USE_WP81_OPTS = 0
!ENDIF
# Set this non-0 to split the SQLite amalgamation file into chunks to
# be used for debugging with Visual Studio.
#
!IFNDEF SPLIT_AMALGAMATION
SPLIT_AMALGAMATION = 0
!ENDIF
# Set this non-0 to dynamically link to the MSVC runtime library.
#
!IFNDEF USE_CRT_DLL
USE_CRT_DLL = 0
!ENDIF
# Set this non-0 to link to the RPCRT4 library.
#
!IFNDEF USE_RPCRT4_LIB
USE_RPCRT4_LIB = 0
!ENDIF
# Set this non-0 to generate assembly code listings for the source code
# files.
#
!IFNDEF USE_LISTINGS
USE_LISTINGS = 0
!ENDIF
# Set this non-0 to attempt setting the native compiler automatically
# for cross-compiling the command line tools needed during the compilation
# process.
#
!IFNDEF XCOMPILE
XCOMPILE = 0
!ENDIF
# Set this non-0 to use the native libraries paths for cross-compiling
# the command line tools needed during the compilation process.
#
!IFNDEF USE_NATIVE_LIBPATHS
USE_NATIVE_LIBPATHS = 0
!ENDIF
# Set this 0 to skip the compiling and embedding of version resources.
#
!IFNDEF USE_RC
USE_RC = 1
!ENDIF
# Set this non-0 to compile binaries suitable for the WinRT environment.
# This setting does not apply to any binaries that require Tcl to operate
# properly (i.e. the text fixture, etc).
#
!IFNDEF FOR_WINRT
FOR_WINRT = 0
!ENDIF
# Set this non-0 to compile binaries suitable for the UWP environment.
# This setting does not apply to any binaries that require Tcl to operate
# properly (i.e. the text fixture, etc).
#
!IFNDEF FOR_UWP
FOR_UWP = 0
!ENDIF
# Set this non-0 to compile binaries suitable for the Windows 10 platform.
#
!IFNDEF FOR_WIN10
FOR_WIN10 = 0
!ENDIF
# Set this to non-0 to create and use PDBs.
#
!IFNDEF SYMBOLS
SYMBOLS = 1
!ENDIF
# Set this to non-0 to use the SQLite debugging heap subsystem.
#
!IFNDEF MEMDEBUG
MEMDEBUG = 0
!ENDIF
# Set this to non-0 to use the Win32 native heap subsystem.
#
!IFNDEF WIN32HEAP
WIN32HEAP = 0
!ENDIF
# Set this to non-0 to enable OSTRACE() macros, which can be useful when
# debugging.
#
!IFNDEF OSTRACE
OSTRACE = 0
!ENDIF
# Set this to one of the following values to enable various debugging
# features. Each level includes the debugging options from the previous
# levels. Currently, the recognized values for DEBUG are:
#
# 0 == NDEBUG: Disables assert() and other runtime diagnostics.
# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API.
# 2 == Disables NDEBUG and all optimizations and then enables PDBs.
# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code.
# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call.
# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros.
# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros.
#
!IFNDEF DEBUG
DEBUG = 0
!ENDIF
# Enable use of available compiler optimizations? Normally, this should be
# non-zero. Setting this to zero, thus disabling all compiler optimizations,
# can be useful for testing.
#
!IFNDEF OPTIMIZATIONS
OPTIMIZATIONS = 2
!ENDIF
# Set the source code file to be used by executables and libraries when
# they need the amalgamation.
#
!IFNDEF SQLITE3C
!IF $(SPLIT_AMALGAMATION)!=0
SQLITE3C = sqlite3-all.c
!ELSE
SQLITE3C = sqlite3.c
!ENDIF
!ENDIF
# Set the include code file to be used by executables and libraries when
# they need SQLite.
#
!IFNDEF SQLITE3H
SQLITE3H = sqlite3.h
!ENDIF
# This is the name to use for the SQLite dynamic link library (DLL).
#
!IFNDEF SQLITE3DLL
SQLITE3DLL = sqlite3.dll
!ENDIF
# This is the name to use for the SQLite import library (LIB).
#
!IFNDEF SQLITE3LIB
SQLITE3LIB = sqlite3.lib
!ENDIF
# This is the name to use for the SQLite shell executable (EXE).
#
!IFNDEF SQLITE3EXE
SQLITE3EXE = sqlite3.exe
!ENDIF
# This is the argument used to set the program database (PDB) file for the
# SQLite shell executable (EXE).
#
!IFNDEF SQLITE3EXEPDB
SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
!ENDIF
# These are the "standard" SQLite compilation options used when compiling for
# the Windows platform.
#
!IFNDEF OPT_FEATURE_FLAGS
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
!ENDIF
# These are the "extended" SQLite compilation options used when compiling for
# the Windows 10 platform.
#
!IFNDEF EXT_FEATURE_FLAGS
!IF $(FOR_WIN10)!=0
EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS4=1
EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_SYSTEM_MALLOC=1
EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) -DSQLITE_OMIT_LOCALTIME=1
!ELSE
EXT_FEATURE_FLAGS =
!ENDIF
!ENDIF
###############################################################################
############################### END OF OPTIONS ################################
###############################################################################
# When compiling for the Windows 10 platform, the PLATFORM macro must be set
# to an appropriate value (e.g. x86, x64, arm, arm64, etc).
#
!IF $(FOR_WIN10)!=0
!IFNDEF PLATFORM
!ERROR Using the FOR_WIN10 option requires a value for PLATFORM.
!ENDIF
!ENDIF
# This assumes that MSVC is always installed in 32-bit Program Files directory
# and sets the variable for use in locating other 32-bit installs accordingly.
#
PROGRAMFILES_X86 = $(VCINSTALLDIR)\..\..
PROGRAMFILES_X86 = $(PROGRAMFILES_X86:\\=\)
# Check for the predefined command macro CC. This should point to the compiler
# binary for the target platform. If it is not defined, simply define it to
# the legacy default value 'cl.exe'.
#
!IFNDEF CC
CC = cl.exe
!ENDIF
# Check for the command macro LD. This should point to the linker binary for
# the target platform. If it is not defined, simply define it to the legacy
# default value 'link.exe'.
#
!IFNDEF LD
LD = link.exe
!ENDIF
# Check for the predefined command macro RC. This should point to the resource
# compiler binary for the target platform. If it is not defined, simply define
# it to the legacy default value 'rc.exe'.
#
!IFNDEF RC
RC = rc.exe
!ENDIF
# Check for the MSVC runtime library path macro. Otherwise, this value will
# default to the 'lib' directory underneath the MSVC installation directory.
#
!IFNDEF CRTLIBPATH
CRTLIBPATH = $(VCINSTALLDIR)\lib
!ENDIF
CRTLIBPATH = $(CRTLIBPATH:\\=\)
# Check for the command macro NCC. This should point to the compiler binary
# for the platform the compilation process is taking place on. If it is not
# defined, simply define it to have the same value as the CC macro. When
# cross-compiling, it is suggested that this macro be modified via the command
# line (since nmake itself does not provide a built-in method to guess it).
# For example, to use the x86 compiler when cross-compiling for x64, a command
# line similar to the following could be used (all on one line):
#
# nmake /f Makefile.msc sqlite3.dll
# XCOMPILE=1 USE_NATIVE_LIBPATHS=1
#
# Alternatively, the full path and file name to the compiler binary for the
# platform the compilation process is taking place may be specified (all on
# one line):
#
# nmake /f Makefile.msc sqlite3.dll
# "NCC=""%VCINSTALLDIR%\bin\cl.exe"""
# USE_NATIVE_LIBPATHS=1
#
!IFDEF NCC
NCC = $(NCC:\\=\)
!ELSEIF $(XCOMPILE)!=0
NCC = "$(VCINSTALLDIR)\bin\$(CC)"
NCC = $(NCC:\\=\)
!ELSE
NCC = $(CC)
!ENDIF
# Check for the MSVC native runtime library path macro. Otherwise,
# this value will default to the 'lib' directory underneath the MSVC
# installation directory.
#
!IFNDEF NCRTLIBPATH
NCRTLIBPATH = $(VCINSTALLDIR)\lib
!ENDIF
NCRTLIBPATH = $(NCRTLIBPATH:\\=\)
# Check for the Platform SDK library path macro. Otherwise, this
# value will default to the 'lib' directory underneath the Windows
# SDK installation directory (the environment variable used appears
# to be available when using Visual C++ 2008 or later via the
# command line).
#
!IFNDEF NSDKLIBPATH
NSDKLIBPATH = $(WINDOWSSDKDIR)\lib
!ENDIF
NSDKLIBPATH = $(NSDKLIBPATH:\\=\)
# Check for the UCRT library path macro. Otherwise, this value will
# default to the version-specific, platform-specific 'lib' directory
# underneath the Windows SDK installation directory.
#
!IFNDEF UCRTLIBPATH
UCRTLIBPATH = $(WINDOWSSDKDIR)\lib\$(WINDOWSSDKLIBVERSION)\ucrt\$(PLATFORM)
!ENDIF
UCRTLIBPATH = $(UCRTLIBPATH:\\=\)
# C compiler and options for use in building executables that
# will run on the platform that is doing the build.
#
!IF $(USE_FULLWARN)!=0
BCC = $(NCC) -nologo -W4 $(CCOPTS) $(BCCOPTS)
!ELSE
BCC = $(NCC) -nologo -W3 $(CCOPTS) $(BCCOPTS)
!ENDIF
# Check if assembly code listings should be generated for the source
# code files to be compiled.
#
!IF $(USE_LISTINGS)!=0
BCC = $(BCC) -FAcs
!ENDIF
# Check if the native library paths should be used when compiling
# the command line tools used during the compilation process. If
# so, set the necessary macro now.
#
!IF $(USE_NATIVE_LIBPATHS)!=0
NLTLIBPATHS = "/LIBPATH:$(NCRTLIBPATH)" "/LIBPATH:$(NSDKLIBPATH)"
!IFDEF NUCRTLIBPATH
NUCRTLIBPATH = $(NUCRTLIBPATH:\\=\)
NLTLIBPATHS = $(NLTLIBPATHS) "/LIBPATH:$(NUCRTLIBPATH)"
!ENDIF
!ENDIF
# C compiler and options for use in building executables that
# will run on the target platform. (BCC and TCC are usually the
# same unless your are cross-compiling.)
#
!IF $(USE_FULLWARN)!=0
TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS)
!ELSE
TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS)
!ENDIF
TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise
RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS)
# Adjust the names of the primary targets for use with Windows 10.
#
!IF $(FOR_WIN10)!=0
SQLITE3DLL = winsqlite3.dll
SQLITE3LIB = winsqlite3.lib
SQLITE3EXE = winsqlite3shell.exe
SQLITE3EXEPDB =
!ENDIF
# Check if we want to use the "stdcall" calling convention when compiling.
# This is not supported by the compilers for non-x86 platforms. It should
# also be noted here that building any target with these "stdcall" options
# will most likely fail if the Tcl library is also required. This is due
# to how the Tcl library functions are declared and exported (i.e. without
# an explicit calling convention, which results in "cdecl").
#
!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0
!IF "$(PLATFORM)"=="x86"
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
!ELSE
!IFNDEF PLATFORM
CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
!ENDIF
!ENDIF
!ELSE
CORE_CCONV_OPTS =
SHELL_CCONV_OPTS =
!ENDIF
# These are additional compiler options used for the core library.
#
!IFNDEF CORE_COMPILE_OPTS
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS) -DSQLITE_API=__declspec(dllexport)
!ELSE
CORE_COMPILE_OPTS = $(CORE_CCONV_OPTS)
!ENDIF
!ENDIF
# These are the additional targets that the core library should depend on
# when linking.
#
!IFNDEF CORE_LINK_DEP
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
CORE_LINK_DEP =
!ELSE
CORE_LINK_DEP =
!ENDIF
!ENDIF
# These are additional linker options used for the core library.
#
!IFNDEF CORE_LINK_OPTS
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
CORE_LINK_OPTS =
!ELSE
CORE_LINK_OPTS =
!ENDIF
!ENDIF
# These are additional compiler options used for the shell executable.
#
!IFNDEF SHELL_COMPILE_OPTS
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport)
!ELSE
SHELL_COMPILE_OPTS = $(SHELL_CCONV_OPTS)
!ENDIF
!ENDIF
# This is the source code that the shell executable should be compiled
# with.
#
!IFNDEF SHELL_CORE_SRC
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
SHELL_CORE_SRC =
!ELSE
SHELL_CORE_SRC = $(SQLITE3C)
!ENDIF
!ENDIF
# This is the core library that the shell executable should depend on.
#
!IFNDEF SHELL_CORE_DEP
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
SHELL_CORE_DEP = $(SQLITE3DLL)
!ELSE
SHELL_CORE_DEP =
!ENDIF
!ENDIF
# This is the core library that the shell executable should link with.
#
!IFNDEF SHELL_CORE_LIB
!IF $(DYNAMIC_SHELL)!=0 || $(FOR_WIN10)!=0
SHELL_CORE_LIB = $(SQLITE3LIB)
!ELSE
SHELL_CORE_LIB =
!ENDIF
!ENDIF
# These are additional linker options used for the shell executable.
#
!IFNDEF SHELL_LINK_OPTS
SHELL_LINK_OPTS = $(SHELL_CORE_LIB)
!ENDIF
# Check if assembly code listings should be generated for the source
# code files to be compiled.
#
!IF $(USE_LISTINGS)!=0
TCC = $(TCC) -FAcs
!ENDIF
# When compiling the library for use in the WinRT environment,
# the following compile-time options must be used as well to
# disable use of Win32 APIs that are not available and to enable
# use of Win32 APIs that are specific to Windows 8 and/or WinRT.
#
!IF $(FOR_WINRT)!=0
TCC = $(TCC) -DSQLITE_OS_WINRT=1
RCC = $(RCC) -DSQLITE_OS_WINRT=1
TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP
!ENDIF
# C compiler options for the Windows 10 platform (needs MSVC 2015).
#
!IF $(FOR_WIN10)!=0
TCC = $(TCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE
!ENDIF
# Also, we need to dynamically link to the correct MSVC runtime
# when compiling for WinRT (e.g. debug or release) OR if the
# USE_CRT_DLL option is set to force dynamically linking to the
# MSVC runtime library.
#
!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0
!IF $(DEBUG)>1
TCC = $(TCC) -MDd
BCC = $(BCC) -MDd
!ELSE
TCC = $(TCC) -MD
BCC = $(BCC) -MD
!ENDIF
!ELSE
!IF $(DEBUG)>1
TCC = $(TCC) -MTd
BCC = $(BCC) -MTd
!ELSE
TCC = $(TCC) -MT
BCC = $(BCC) -MT
!ENDIF
!ENDIF
# Define -DNDEBUG to compile without debugging (i.e., for production usage)
# Omitting the define will cause extra debugging code to be inserted and
# includes extra comments when "EXPLAIN stmt" is used.
#
!IF $(DEBUG)==0
TCC = $(TCC) -DNDEBUG
BCC = $(BCC) -DNDEBUG
RCC = $(RCC) -DNDEBUG
!ENDIF
!IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0
TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1
RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1
!ENDIF
!IF $(DEBUG)>2
TCC = $(TCC) -DSQLITE_DEBUG=1
RCC = $(RCC) -DSQLITE_DEBUG=1
!ENDIF
!IF $(DEBUG)>4 || $(OSTRACE)!=0
TCC = $(TCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
RCC = $(RCC) -DSQLITE_FORCE_OS_TRACE=1 -DSQLITE_DEBUG_OS_TRACE=1
!ENDIF
!IF $(DEBUG)>5
TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE=1
RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE=1
!ENDIF
# Prevent warnings about "insecure" MSVC runtime library functions
# being used.
#
TCC = $(TCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
BCC = $(BCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
RCC = $(RCC) -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS
# Prevent warnings about "deprecated" POSIX functions being used.
#
TCC = $(TCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
BCC = $(BCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
RCC = $(RCC) -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS
# Use the SQLite debugging heap subsystem?
#
!IF $(MEMDEBUG)!=0
TCC = $(TCC) -DSQLITE_MEMDEBUG=1
RCC = $(RCC) -DSQLITE_MEMDEBUG=1
# Use native Win32 heap subsystem instead of malloc/free?
#
!ELSEIF $(WIN32HEAP)!=0
TCC = $(TCC) -DSQLITE_WIN32_MALLOC=1
RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1
# Validate the heap on every call into the native Win32 heap subsystem?
#
!IF $(DEBUG)>3
TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
!ENDIF
!ENDIF
# Compiler options needed for programs that use the readline() library.
#
!IFNDEF READLINE_FLAGS
READLINE_FLAGS = -DHAVE_READLINE=0
!ENDIF
# The library that programs using readline() must link against.
#
!IFNDEF LIBREADLINE
LIBREADLINE =
!ENDIF
# Should the database engine be compiled threadsafe
#
TCC = $(TCC) -DSQLITE_THREADSAFE=1
RCC = $(RCC) -DSQLITE_THREADSAFE=1
# Do threads override each others locks by default (1), or do we test (-1)
#
TCC = $(TCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
RCC = $(RCC) -DSQLITE_THREAD_OVERRIDE_LOCK=-1
# Any target libraries which libsqlite must be linked against
#
!IFNDEF TLIBS
TLIBS =
!ENDIF
# Flags controlling use of the in memory btree implementation
#
# SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to
# default to file, 2 to default to memory, and 3 to force temporary
# tables to always be in memory.
#
TCC = $(TCC) -DSQLITE_TEMP_STORE=1
RCC = $(RCC) -DSQLITE_TEMP_STORE=1
# Enable/disable loadable extensions, and other optional features
# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
# The same set of OMIT and ENABLE flags should be passed to the
# LEMON parser generator and the mkkeywordhash tool as well.
# These are the required SQLite compilation options used when compiling for
# the Windows platform.
#
REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_MAX_TRIGGER_DEPTH=100
# If we are linking to the RPCRT4 library, enable features that need it.
#
!IF $(USE_RPCRT4_LIB)!=0
REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) -DSQLITE_WIN32_USE_UUID=1
!ENDIF
# Add the required and optional SQLite compilation options into the command
# lines used to invoke the MSVC code and resource compilers.
#
TCC = $(TCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
RCC = $(RCC) $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS)
# Add in any optional parameters specified on the commane line, e.g.
# nmake /f Makefile.msc all "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1"
#
TCC = $(TCC) $(OPTS)
RCC = $(RCC) $(OPTS)
# If compiling for debugging, add some defines.
#
!IF $(DEBUG)>1
TCC = $(TCC) -D_DEBUG
BCC = $(BCC) -D_DEBUG
RCC = $(RCC) -D_DEBUG
!ENDIF
# If optimizations are enabled or disabled (either implicitly or
# explicitly), add the necessary flags.
#
!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0
TCC = $(TCC) -Od
BCC = $(BCC) -Od
!ELSEIF $(OPTIMIZATIONS)>=3
TCC = $(TCC) -Ox
BCC = $(BCC) -Ox
!ELSEIF $(OPTIMIZATIONS)==2
TCC = $(TCC) -O2
BCC = $(BCC) -O2
!ELSEIF $(OPTIMIZATIONS)==1
TCC = $(TCC) -O1
BCC = $(BCC) -O1
!ENDIF
# If symbols are enabled (or compiling for debugging), enable PDBs.
#
!IF $(DEBUG)>1 || $(SYMBOLS)!=0
TCC = $(TCC) -Zi
BCC = $(BCC) -Zi
!ENDIF
# Command line prefixes for compiling code, compiling resources,
# linking, etc.
#
LTCOMPILE = $(TCC) -Fo$@
LTRCOMPILE = $(RCC) -r
LTLIB = lib.exe
LTLINK = $(TCC) -Fe$@
# If requested, link to the RPCRT4 library.
#
!IF $(USE_RPCRT4_LIB)!=0
LTLINK = $(LTLINK) rpcrt4.lib
!ENDIF
# If a platform was set, force the linker to target that.
# Note that the vcvars*.bat family of batch files typically
# set this for you. Otherwise, the linker will attempt
# to deduce the binary type based on the object files.
!IFDEF PLATFORM
LTLINKOPTS = /NOLOGO /MACHINE:$(PLATFORM)
LTLIBOPTS = /NOLOGO /MACHINE:$(PLATFORM)
!ELSE
LTLINKOPTS = /NOLOGO
LTLIBOPTS = /NOLOGO
!ENDIF
# When compiling for use in the WinRT environment, the following
# linker option must be used to mark the executable as runnable
# only in the context of an application container.
#
!IF $(FOR_WINRT)!=0
LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER
!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0"
!IFNDEF STORELIBPATH
!IF "$(PLATFORM)"=="x86"
STORELIBPATH = $(CRTLIBPATH)\store
!ELSEIF "$(PLATFORM)"=="x64"
STORELIBPATH = $(CRTLIBPATH)\store\amd64
!ELSEIF "$(PLATFORM)"=="ARM"
STORELIBPATH = $(CRTLIBPATH)\store\arm
!ELSE
STORELIBPATH = $(CRTLIBPATH)\store
!ENDIF
!ENDIF
STORELIBPATH = $(STORELIBPATH:\\=\)
LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)"
!ENDIF
!ENDIF
# When compiling for Windows Phone 8.1, an extra library path is
# required.
#
!IF $(USE_WP81_OPTS)!=0
!IFNDEF WP81LIBPATH
!IF "$(PLATFORM)"=="x86"
WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
!ELSEIF "$(PLATFORM)"=="ARM"
WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM
!ELSE
WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86
!ENDIF
!ENDIF
!ENDIF
# When compiling for Windows Phone 8.1, some extra linker options
# are also required.
#
!IF $(USE_WP81_OPTS)!=0
!IFDEF WP81LIBPATH
LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)"
!ENDIF
LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE
LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib
LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib
!ENDIF
# When compiling for UWP or the Windows 10 platform, some extra linker
# options are also required.
#
!IF $(FOR_UWP)!=0 || $(FOR_WIN10)!=0
LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE /NODEFAULTLIB:kernel32.lib
LTLINKOPTS = $(LTLINKOPTS) mincore.lib
!IFDEF PSDKLIBPATH
LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(PSDKLIBPATH)"
!ENDIF
!ENDIF
!IF $(FOR_WIN10)!=0
LTLINKOPTS = $(LTLINKOPTS) /guard:cf "/LIBPATH:$(UCRTLIBPATH)"
!IF $(DEBUG)>1
LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrtd.lib /DEFAULTLIB:ucrtd.lib
!ELSE
LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib
!ENDIF
!ENDIF
# If either debugging or symbols are enabled, enable PDBs.
#
!IF $(DEBUG)>1 || $(SYMBOLS)!=0
LDFLAGS = /DEBUG $(LDOPTS)
!ELSE
LDFLAGS = $(LDOPTS)
!ENDIF
# You should not have to change anything below this line
###############################################################################
# Object files for the amalgamation.
#
LIBOBJS1 = sqlite3.lo
# Determine the real value of LIBOBJ based on the 'configure' script
#
LIBOBJ = $(LIBOBJS1)
# Determine if embedded resource compilation and usage are enabled.
#
!IF $(USE_RC)!=0
LIBRESOBJS = sqlite3res.lo
!ELSE
LIBRESOBJS =
!ENDIF
# Additional compiler options for the shell. These are only effective
# when the shell is not being dynamically linked.
#
!IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_EXPLAIN_COMMENTS
!ENDIF
# This is the default Makefile target. The objects listed here
# are what get build when you type just "make" with no arguments.
#
all: dll libsqlite3.lib shell
# Dynamic link library section.
#
dll: $(SQLITE3DLL)
# Shell executable.
#
shell: $(SQLITE3EXE)
libsqlite3.lib: $(LIBOBJ)
$(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS)
$(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
$(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\shell.c $(SHELL_CORE_SRC) \
/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
# Rule to build the amalgamation
#
sqlite3.lo: $(SQLITE3C)
$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(SQLITE3C)
# Rule to build the Win32 resources object file.
#
!IF $(USE_RC)!=0
_HASHCHAR=^#
!IF ![echo !IFNDEF VERSION > rcver.vc] && \
![for /F "delims=" %V in ('type "$(SQLITE3H)" ^| find "$(_HASHCHAR)define SQLITE_VERSION "') do (echo VERSION = ^^%V >> rcver.vc)] && \
![echo !ENDIF >> rcver.vc]
!INCLUDE rcver.vc
!ENDIF
RESOURCE_VERSION = $(VERSION:^#=)
RESOURCE_VERSION = $(RESOURCE_VERSION:define=)
RESOURCE_VERSION = $(RESOURCE_VERSION:SQLITE_VERSION=)
RESOURCE_VERSION = $(RESOURCE_VERSION:"=)
RESOURCE_VERSION = $(RESOURCE_VERSION:.=,)
$(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H)
echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h
echo #define SQLITE_RESOURCE_VERSION $(RESOURCE_VERSION) >> sqlite3rc.h
echo #endif >> sqlite3rc.h
$(LTRCOMPILE) -fo $(LIBRESOBJS) -DRC_VERONLY $(TOP)\sqlite3.rc
!ENDIF
clean:
del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL
del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL

32
autoconf/README
View File

@ -1,32 +0,0 @@
This package contains:
* the SQLite library amalgamation (single file) source code distribution,
* the shell.c file used to build the sqlite3 shell too, and
* the sqlite3.h and sqlite3ext.h header files required to link programs
and sqlite extensions against the installed libary.
* autoconf/automake installation infrastucture.
The generic installation instructions for autoconf/automake are found
in the INSTALL file.
The following SQLite specific boolean options are supported:
--enable-readline use readline in shell tool [default=yes]
--enable-threadsafe build a thread-safe library [default=yes]
--enable-dynamic-extensions support loadable extensions [default=yes]
The default value for the CFLAGS variable (options passed to the C
compiler) includes debugging symbols in the build, resulting in larger
binaries than are necessary. Override it on the configure command
line like this:
$ CFLAGS="-Os" ./configure
to produce a smaller installation footprint.
Other SQLite compilation parameters can also be set using CFLAGS. For
example:
$ CFLAGS="-Os -DSQLITE_OMIT_TRIGGERS" ./configure

113
autoconf/README.txt Normal file
View File

@ -0,0 +1,113 @@
This package contains:
* the SQLite library amalgamation source code file: sqlite3.c
* the sqlite3.h and sqlite3ext.h header files that define the C-language
interface to the sqlite3.c library file
* the shell.c file used to build the sqlite3 command-line shell program
* autoconf/automake installation infrastucture for building on POSIX
compliant systems
* a Makefile.msc and sqlite3.rc for building with Microsoft Visual C++ on
Windows
SUMMARY OF HOW TO BUILD
=======================
Unix: ./configure; make
Windows: nmake /f Makefile.msc
BUILDING ON POSIX
=================
The generic installation instructions for autoconf/automake are found
in the INSTALL file.
The following SQLite specific boolean options are supported:
--enable-readline use readline in shell tool [default=yes]
--enable-threadsafe build a thread-safe library [default=yes]
--enable-dynamic-extensions support loadable extensions [default=yes]
The default value for the CFLAGS variable (options passed to the C
compiler) includes debugging symbols in the build, resulting in larger
binaries than are necessary. Override it on the configure command
line like this:
$ CFLAGS="-Os" ./configure
to produce a smaller installation footprint.
Other SQLite compilation parameters can also be set using CFLAGS. For
example:
$ CFLAGS="-Os -DSQLITE_THREADSAFE=0" ./configure
BUILDING WITH MICROSOFT VISUAL C++
==================================
To compile for Windows using Microsoft Visual C++:
$ nmake /f Makefile.msc
Using Microsoft Visual C++ 2005 (or later) is recommended. Several Windows
platform variants may be built by adding additional macros to the NMAKE
command line.
Building for WinRT 8.0
----------------------
FOR_WINRT=1
Using Microsoft Visual C++ 2012 (or later) is required. When using the
above, something like the following macro will need to be added to the
NMAKE command line as well:
"NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86"
Building for WinRT 8.1
----------------------
FOR_WINRT=1
Using Microsoft Visual C++ 2013 (or later) is required. When using the
above, something like the following macro will need to be added to the
NMAKE command line as well:
"NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"
Building for UWP 10.0
---------------------
FOR_WINRT=1 FOR_UWP=1
Using Microsoft Visual C++ 2015 (or later) is required. When using the
above, something like the following macros will need to be added to the
NMAKE command line as well:
"NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
"PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
"NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86"
Building for the Windows 10 SDK
-------------------------------
FOR_WIN10=1
Using Microsoft Visual C++ 2015 (or later) is required. When using the
above, no other macros should be needed on the NMAKE command line.
Other preprocessor defines
--------------------------
Additionally, preprocessor defines may be specified by using the OPTS macro
on the NMAKE command line. However, not all possible preprocessor defines
may be specified in this manner as some require the amalgamation to be built
with them enabled (see http://www.sqlite.org/compile.html). For example, the
following will work:
"OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
However, the following will not compile unless the amalgamation was built
with it enabled:
"OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"

1530
autoconf/config.guess vendored
View File

File diff suppressed because it is too large Load Diff

1773
autoconf/config.sub vendored
View File

File diff suppressed because it is too large Load Diff

67
autoconf/configure.ac
View File

@ -4,11 +4,13 @@
#
# --enable-threadsafe
# --enable-readline
# --enable-editline
# --enable-static-shell
# --enable-dynamic-extensions
#
AC_PREREQ(2.61)
AC_INIT(sqlite, 3.7.5, http://www.sqlite.org)
AC_INIT(sqlite, --SQLITE-VERSION--, http://www.sqlite.org)
AC_CONFIG_SRCDIR([sqlite3.c])
# Use automake.
@ -18,7 +20,6 @@ AC_SYS_LARGEFILE
# Check for required programs.
AC_PROG_CC
AC_PROG_RANLIB
AC_PROG_LIBTOOL
AC_PROG_MKDIR_P
@ -30,12 +31,29 @@ AC_CONFIG_FILES([Makefile sqlite3.pc])
AC_SUBST(BUILD_CFLAGS)
#-----------------------------------------------------------------------
# --enable-editline
# --enable-readline
#
AC_ARG_ENABLE(editline, [AS_HELP_STRING(
[--enable-editline],
[use BSD libedit])],
[], [enable_editline=yes])
AC_ARG_ENABLE(readline, [AS_HELP_STRING(
[--enable-readline],
[use readline in shell tool (yes, no) [default=yes]])],
[], [enable_readline=yes])
[use readline])],
[], [enable_readline=no])
if test x"$enable_editline" != xno ; then
sLIBS=$LIBS
LIBS=""
AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no])
READLINE_LIBS=$LIBS
if test x"$LIBS" != "x"; then
AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline)
else
unset ac_cv_search_readline
fi
LIBS=$sLIBS
fi
if test x"$enable_readline" != xno ; then
sLIBS=$LIBS
LIBS=""
@ -58,6 +76,7 @@ THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0
if test x"$enable_threadsafe" != "xno"; then
THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1"
AC_SEARCH_LIBS(pthread_create, pthread)
AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
fi
AC_SUBST(THREADSAFE_FLAGS)
#-----------------------------------------------------------------------
@ -78,6 +97,46 @@ AC_MSG_RESULT($enable_dynamic_extensions)
AC_SUBST(DYNAMIC_EXTENSION_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-fts5
#
AC_ARG_ENABLE(fts5, [AS_HELP_STRING(
[--enable-fts5], [include fts5 support [default=no]])],
[], [enable_fts5=no])
if test x"$enable_fts5" == "xyes"; then
AC_SEARCH_LIBS(log, m)
FTS5_FLAGS=-DSQLITE_ENABLE_FTS5
fi
AC_SUBST(FTS5_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-json1
#
AC_ARG_ENABLE(json1, [AS_HELP_STRING(
[--enable-json1], [include json1 support [default=no]])],
[], [enable_json1=no])
if test x"$enable_json1" == "xyes"; then
JSON1_FLAGS=-DSQLITE_ENABLE_JSON1
fi
AC_SUBST(JSON1_FLAGS)
#-----------------------------------------------------------------------
#-----------------------------------------------------------------------
# --enable-static-shell
#
AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
[--enable-static-shell],
[statically link libsqlite3 into shell tool [default=yes]])],
[], [enable_static_shell=yes])
if test x"$enable_static_shell" == "xyes"; then
EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
else
EXTRA_SHELL_OBJ=libsqlite3.la
fi
AC_SUBST(EXTRA_SHELL_OBJ)
#-----------------------------------------------------------------------
AC_CHECK_FUNCS(posix_fallocate)
#-----------------------------------------------------------------------

708
autoconf/depcomp
View File

@ -1,708 +0,0 @@
#! /bin/sh
# depcomp - compile a program generating dependencies as side-effects
scriptversion=2012-03-27.16; # UTC
# Copyright (C) 1999, 2000, 2003, 2004, 2005, 2006, 2007, 2009, 2010,
# 2011, 2012 Free Software Foundation, Inc.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# Originally written by Alexandre Oliva <oliva@dcc.unicamp.br>.
case $1 in
'')
echo "$0: No command. Try '$0 --help' for more information." 1>&2
exit 1;
;;
-h | --h*)
cat <<\EOF
Usage: depcomp [--help] [--version] PROGRAM [ARGS]
Run PROGRAMS ARGS to compile a file, generating dependencies
as side-effects.
Environment variables:
depmode Dependency tracking mode.
source Source file read by 'PROGRAMS ARGS'.
object Object file output by 'PROGRAMS ARGS'.
DEPDIR directory where to store dependencies.
depfile Dependency file to output.
tmpdepfile Temporary file to use when outputting dependencies.
libtool Whether libtool is used (yes/no).
Report bugs to <bug-automake@gnu.org>.
EOF
exit $?
;;
-v | --v*)
echo "depcomp $scriptversion"
exit $?
;;
esac
# A tabulation character.
tab=' '
# A newline character.
nl='
'
if test -z "$depmode" || test -z "$source" || test -z "$object"; then
echo "depcomp: Variables source, object and depmode must be set" 1>&2
exit 1
fi
# Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po.
depfile=${depfile-`echo "$object" |
sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`}
tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`}
rm -f "$tmpdepfile"
# Some modes work just like other modes, but use different flags. We
# parameterize here, but still list the modes in the big case below,
# to make depend.m4 easier to write. Note that we *cannot* use a case
# here, because this file can only contain one case statement.
if test "$depmode" = hp; then
# HP compiler uses -M and no extra arg.
gccflag=-M
depmode=gcc
fi
if test "$depmode" = dashXmstdout; then
# This is just like dashmstdout with a different argument.
dashmflag=-xM
depmode=dashmstdout
fi
cygpath_u="cygpath -u -f -"
if test "$depmode" = msvcmsys; then
# This is just like msvisualcpp but w/o cygpath translation.
# Just convert the backslash-escaped backslashes to single forward
# slashes to satisfy depend.m4
cygpath_u='sed s,\\\\,/,g'
depmode=msvisualcpp
fi
if test "$depmode" = msvc7msys; then
# This is just like msvc7 but w/o cygpath translation.
# Just convert the backslash-escaped backslashes to single forward
# slashes to satisfy depend.m4
cygpath_u='sed s,\\\\,/,g'
depmode=msvc7
fi
if test "$depmode" = xlc; then
# IBM C/C++ Compilers xlc/xlC can output gcc-like dependency informations.
gccflag=-qmakedep=gcc,-MF
depmode=gcc
fi
case "$depmode" in
gcc3)
## gcc 3 implements dependency tracking that does exactly what
## we want. Yay! Note: for some reason libtool 1.4 doesn't like
## it if -MD -MP comes after the -MF stuff. Hmm.
## Unfortunately, FreeBSD c89 acceptance of flags depends upon
## the command line argument order; so add the flags where they
## appear in depend2.am. Note that the slowdown incurred here
## affects only configure: in makefiles, %FASTDEP% shortcuts this.
for arg
do
case $arg in
-c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;;
*) set fnord "$@" "$arg" ;;
esac
shift # fnord
shift # $arg
done
"$@"
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
mv "$tmpdepfile" "$depfile"
;;
gcc)
## There are various ways to get dependency output from gcc. Here's
## why we pick this rather obscure method:
## - Don't want to use -MD because we'd like the dependencies to end
## up in a subdir. Having to rename by hand is ugly.
## (We might end up doing this anyway to support other compilers.)
## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like
## -MM, not -M (despite what the docs say).
## - Using -M directly means running the compiler twice (even worse
## than renaming).
if test -z "$gccflag"; then
gccflag=-MD,
fi
"$@" -Wp,"$gccflag$tmpdepfile"
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
echo "$object : \\" > "$depfile"
alpha=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
## The second -e expression handles DOS-style file names with drive letters.
sed -e 's/^[^:]*: / /' \
-e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile"
## This next piece of magic avoids the "deleted header file" problem.
## The problem is that when a header file which appears in a .P file
## is deleted, the dependency causes make to die (because there is
## typically no way to rebuild the header). We avoid this by adding
## dummy dependencies for each header file. Too bad gcc doesn't do
## this for us directly.
tr ' ' "$nl" < "$tmpdepfile" |
## Some versions of gcc put a space before the ':'. On the theory
## that the space means something, we add a space to the output as
## well. hp depmode also adds that space, but also prefixes the VPATH
## to the object. Take care to not repeat it in the output.
## Some versions of the HPUX 10.20 sed can't process this invocation
## correctly. Breaking it into two sed invocations is a workaround.
sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \
| sed -e 's/$/ :/' >> "$depfile"
rm -f "$tmpdepfile"
;;
hp)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
sgi)
if test "$libtool" = yes; then
"$@" "-Wp,-MDupdate,$tmpdepfile"
else
"$@" -MDupdate "$tmpdepfile"
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files
echo "$object : \\" > "$depfile"
# Clip off the initial element (the dependent). Don't try to be
# clever and replace this with sed code, as IRIX sed won't handle
# lines with more than a fixed number of characters (4096 in
# IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines;
# the IRIX cc adds comments like '#:fec' to the end of the
# dependency line.
tr ' ' "$nl" < "$tmpdepfile" \
| sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' | \
tr "$nl" ' ' >> "$depfile"
echo >> "$depfile"
# The second pass generates a dummy entry for each header file.
tr ' ' "$nl" < "$tmpdepfile" \
| sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \
>> "$depfile"
else
# The sourcefile does not contain any dependencies, so just
# store a dummy comment line, to avoid errors with the Makefile
# "include basename.Plo" scheme.
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile"
;;
xlc)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
aix)
# The C for AIX Compiler uses -M and outputs the dependencies
# in a .u file. In older versions, this file always lives in the
# current directory. Also, the AIX compiler puts '$object:' at the
# start of each line; $object doesn't have directory information.
# Version 6 uses the directory in both cases.
dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
test "x$dir" = "x$object" && dir=
base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
if test "$libtool" = yes; then
tmpdepfile1=$dir$base.u
tmpdepfile2=$base.u
tmpdepfile3=$dir.libs/$base.u
"$@" -Wc,-M
else
tmpdepfile1=$dir$base.u
tmpdepfile2=$dir$base.u
tmpdepfile3=$dir$base.u
"$@" -M
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
exit $stat
fi
for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3"
do
test -f "$tmpdepfile" && break
done
if test -f "$tmpdepfile"; then
# Each line is of the form 'foo.o: dependent.h'.
# Do two passes, one to just change these to
# '$object: dependent.h' and one to simply 'dependent.h:'.
sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
else
# The sourcefile does not contain any dependencies, so just
# store a dummy comment line, to avoid errors with the Makefile
# "include basename.Plo" scheme.
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile"
;;
icc)
# Intel's C compiler anf tcc (Tiny C Compiler) understand '-MD -MF file'.
# However on
# $CC -MD -MF foo.d -c -o sub/foo.o sub/foo.c
# ICC 7.0 will fill foo.d with something like
# foo.o: sub/foo.c
# foo.o: sub/foo.h
# which is wrong. We want
# sub/foo.o: sub/foo.c
# sub/foo.o: sub/foo.h
# sub/foo.c:
# sub/foo.h:
# ICC 7.1 will output
# foo.o: sub/foo.c sub/foo.h
# and will wrap long lines using '\':
# foo.o: sub/foo.c ... \
# sub/foo.h ... \
# ...
# tcc 0.9.26 (FIXME still under development at the moment of writing)
# will emit a similar output, but also prepend the continuation lines
# with horizontal tabulation characters.
"$@" -MD -MF "$tmpdepfile"
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
# Each line is of the form 'foo.o: dependent.h',
# or 'foo.o: dep1.h dep2.h \', or ' dep3.h dep4.h \'.
# Do two passes, one to just change these to
# '$object: dependent.h' and one to simply 'dependent.h:'.
sed -e "s/^[ $tab][ $tab]*/ /" -e "s,^[^:]*:,$object :," \
< "$tmpdepfile" > "$depfile"
sed '
s/[ '"$tab"'][ '"$tab"']*/ /g
s/^ *//
s/ *\\*$//
s/^[^:]*: *//
/^$/d
/:$/d
s/$/ :/
' < "$tmpdepfile" >> "$depfile"
rm -f "$tmpdepfile"
;;
hp2)
# The "hp" stanza above does not work with aCC (C++) and HP's ia64
# compilers, which have integrated preprocessors. The correct option
# to use with these is +Maked; it writes dependencies to a file named
# 'foo.d', which lands next to the object file, wherever that
# happens to be.
# Much of this is similar to the tru64 case; see comments there.
dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
test "x$dir" = "x$object" && dir=
base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
if test "$libtool" = yes; then
tmpdepfile1=$dir$base.d
tmpdepfile2=$dir.libs/$base.d
"$@" -Wc,+Maked
else
tmpdepfile1=$dir$base.d
tmpdepfile2=$dir$base.d
"$@" +Maked
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile1" "$tmpdepfile2"
exit $stat
fi
for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2"
do
test -f "$tmpdepfile" && break
done
if test -f "$tmpdepfile"; then
sed -e "s,^.*\.[a-z]*:,$object:," "$tmpdepfile" > "$depfile"
# Add 'dependent.h:' lines.
sed -ne '2,${
s/^ *//
s/ \\*$//
s/$/:/
p
}' "$tmpdepfile" >> "$depfile"
else
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile" "$tmpdepfile2"
;;
tru64)
# The Tru64 compiler uses -MD to generate dependencies as a side
# effect. 'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'.
# At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put
# dependencies in 'foo.d' instead, so we check for that too.
# Subdirectories are respected.
dir=`echo "$object" | sed -e 's|/[^/]*$|/|'`
test "x$dir" = "x$object" && dir=
base=`echo "$object" | sed -e 's|^.*/||' -e 's/\.o$//' -e 's/\.lo$//'`
if test "$libtool" = yes; then
# With Tru64 cc, shared objects can also be used to make a
# static library. This mechanism is used in libtool 1.4 series to
# handle both shared and static libraries in a single compilation.
# With libtool 1.4, dependencies were output in $dir.libs/$base.lo.d.
#
# With libtool 1.5 this exception was removed, and libtool now
# generates 2 separate objects for the 2 libraries. These two
# compilations output dependencies in $dir.libs/$base.o.d and
# in $dir$base.o.d. We have to check for both files, because
# one of the two compilations can be disabled. We should prefer
# $dir$base.o.d over $dir.libs/$base.o.d because the latter is
# automatically cleaned when .libs/ is deleted, while ignoring
# the former would cause a distcleancheck panic.
tmpdepfile1=$dir.libs/$base.lo.d # libtool 1.4
tmpdepfile2=$dir$base.o.d # libtool 1.5
tmpdepfile3=$dir.libs/$base.o.d # libtool 1.5
tmpdepfile4=$dir.libs/$base.d # Compaq CCC V6.2-504
"$@" -Wc,-MD
else
tmpdepfile1=$dir$base.o.d
tmpdepfile2=$dir$base.d
tmpdepfile3=$dir$base.d
tmpdepfile4=$dir$base.d
"$@" -MD
fi
stat=$?
if test $stat -eq 0; then :
else
rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
exit $stat
fi
for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" "$tmpdepfile4"
do
test -f "$tmpdepfile" && break
done
if test -f "$tmpdepfile"; then
sed -e "s,^.*\.[a-z]*:,$object:," < "$tmpdepfile" > "$depfile"
sed -e 's,^.*\.[a-z]*:['"$tab"' ]*,,' -e 's,$,:,' < "$tmpdepfile" >> "$depfile"
else
echo "#dummy" > "$depfile"
fi
rm -f "$tmpdepfile"
;;
msvc7)
if test "$libtool" = yes; then
showIncludes=-Wc,-showIncludes
else
showIncludes=-showIncludes
fi
"$@" $showIncludes > "$tmpdepfile"
stat=$?
grep -v '^Note: including file: ' "$tmpdepfile"
if test "$stat" = 0; then :
else
rm -f "$tmpdepfile"
exit $stat
fi
rm -f "$depfile"
echo "$object : \\" > "$depfile"
# The first sed program below extracts the file names and escapes
# backslashes for cygpath. The second sed program outputs the file
# name when reading, but also accumulates all include files in the
# hold buffer in order to output them again at the end. This only
# works with sed implementations that can handle large buffers.
sed < "$tmpdepfile" -n '
/^Note: including file: *\(.*\)/ {
s//\1/
s/\\/\\\\/g
p
}' | $cygpath_u | sort -u | sed -n '
s/ /\\ /g
s/\(.*\)/'"$tab"'\1 \\/p
s/.\(.*\) \\/\1:/
H
$ {
s/.*/'"$tab"'/
G
p
}' >> "$depfile"
rm -f "$tmpdepfile"
;;
msvc7msys)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
#nosideeffect)
# This comment above is used by automake to tell side-effect
# dependency tracking mechanisms from slower ones.
dashmstdout)
# Important note: in order to support this mode, a compiler *must*
# always write the preprocessed file to stdout, regardless of -o.
"$@" || exit $?
# Remove the call to Libtool.
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
# Remove '-o $object'.
IFS=" "
for arg
do
case $arg in
-o)
shift
;;
$object)
shift
;;
*)
set fnord "$@" "$arg"
shift # fnord
shift # $arg
;;
esac
done
test -z "$dashmflag" && dashmflag=-M
# Require at least two characters before searching for ':'
# in the target name. This is to cope with DOS-style filenames:
# a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise.
"$@" $dashmflag |
sed 's:^['"$tab"' ]*[^:'"$tab"' ][^:][^:]*\:['"$tab"' ]*:'"$object"'\: :' > "$tmpdepfile"
rm -f "$depfile"
cat < "$tmpdepfile" > "$depfile"
tr ' ' "$nl" < "$tmpdepfile" | \
## Some versions of the HPUX 10.20 sed can't process this invocation
## correctly. Breaking it into two sed invocations is a workaround.
sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
rm -f "$tmpdepfile"
;;
dashXmstdout)
# This case only exists to satisfy depend.m4. It is never actually
# run, as this mode is specially recognized in the preamble.
exit 1
;;
makedepend)
"$@" || exit $?
# Remove any Libtool call
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
# X makedepend
shift
cleared=no eat=no
for arg
do
case $cleared in
no)
set ""; shift
cleared=yes ;;
esac
if test $eat = yes; then
eat=no
continue
fi
case "$arg" in
-D*|-I*)
set fnord "$@" "$arg"; shift ;;
# Strip any option that makedepend may not understand. Remove
# the object too, otherwise makedepend will parse it as a source file.
-arch)
eat=yes ;;
-*|$object)
;;
*)
set fnord "$@" "$arg"; shift ;;
esac
done
obj_suffix=`echo "$object" | sed 's/^.*\././'`
touch "$tmpdepfile"
${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@"
rm -f "$depfile"
# makedepend may prepend the VPATH from the source file name to the object.
# No need to regex-escape $object, excess matching of '.' is harmless.
sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile"
sed '1,2d' "$tmpdepfile" | tr ' ' "$nl" | \
## Some versions of the HPUX 10.20 sed can't process this invocation
## correctly. Breaking it into two sed invocations is a workaround.
sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' | sed -e 's/$/ :/' >> "$depfile"
rm -f "$tmpdepfile" "$tmpdepfile".bak
;;
cpp)
# Important note: in order to support this mode, a compiler *must*
# always write the preprocessed file to stdout.
"$@" || exit $?
# Remove the call to Libtool.
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
# Remove '-o $object'.
IFS=" "
for arg
do
case $arg in
-o)
shift
;;
$object)
shift
;;
*)
set fnord "$@" "$arg"
shift # fnord
shift # $arg
;;
esac
done
"$@" -E |
sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \
-e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' |
sed '$ s: \\$::' > "$tmpdepfile"
rm -f "$depfile"
echo "$object : \\" > "$depfile"
cat < "$tmpdepfile" >> "$depfile"
sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile"
rm -f "$tmpdepfile"
;;
msvisualcpp)
# Important note: in order to support this mode, a compiler *must*
# always write the preprocessed file to stdout.
"$@" || exit $?
# Remove the call to Libtool.
if test "$libtool" = yes; then
while test "X$1" != 'X--mode=compile'; do
shift
done
shift
fi
IFS=" "
for arg
do
case "$arg" in
-o)
shift
;;
$object)
shift
;;
"-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI")
set fnord "$@"
shift
shift
;;
*)
set fnord "$@" "$arg"
shift
shift
;;
esac
done
"$@" -E 2>/dev/null |
sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile"
rm -f "$depfile"
echo "$object : \\" > "$depfile"
sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile"
echo "$tab" >> "$depfile"
sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile"
rm -f "$tmpdepfile"
;;
msvcmsys)
# This case exists only to let depend.m4 do its work. It works by
# looking at the text of this script. This case will never be run,
# since it is checked for above.
exit 1
;;
none)
exec "$@"
;;
*)
echo "Unknown depmode $depmode" 1>&2
exit 1
;;
esac
exit 0
# Local Variables:
# mode: shell-script
# sh-indentation: 2
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:

527
autoconf/install-sh
View File

@ -1,527 +0,0 @@
#!/bin/sh
# install - install a program, script, or datafile
scriptversion=2011-01-19.21; # UTC
# This originates from X11R5 (mit/util/scripts/install.sh), which was
# later released in X11R6 (xc/config/util/install.sh) with the
# following copyright and license.
#
# Copyright (C) 1994 X Consortium
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# Except as contained in this notice, the name of the X Consortium shall not
# be used in advertising or otherwise to promote the sale, use or other deal-
# ings in this Software without prior written authorization from the X Consor-
# tium.
#
#
# FSF changes to this file are in the public domain.
#
# Calling this script install-sh is preferred over install.sh, to prevent
# `make' implicit rules from creating a file called install from it
# when there is no Makefile.
#
# This script is compatible with the BSD install script, but was written
# from scratch.
nl='
'
IFS=" "" $nl"
# set DOITPROG to echo to test this script
# Don't use :- since 4.3BSD and earlier shells don't like it.
doit=${DOITPROG-}
if test -z "$doit"; then
doit_exec=exec
else
doit_exec=$doit
fi
# Put in absolute file names if you don't have them in your path;
# or use environment vars.
chgrpprog=${CHGRPPROG-chgrp}
chmodprog=${CHMODPROG-chmod}
chownprog=${CHOWNPROG-chown}
cmpprog=${CMPPROG-cmp}
cpprog=${CPPROG-cp}
mkdirprog=${MKDIRPROG-mkdir}
mvprog=${MVPROG-mv}
rmprog=${RMPROG-rm}
stripprog=${STRIPPROG-strip}
posix_glob='?'
initialize_posix_glob='
test "$posix_glob" != "?" || {
if (set -f) 2>/dev/null; then
posix_glob=
else
posix_glob=:
fi
}
'
posix_mkdir=
# Desired mode of installed file.
mode=0755
chgrpcmd=
chmodcmd=$chmodprog
chowncmd=
mvcmd=$mvprog
rmcmd="$rmprog -f"
stripcmd=
src=
dst=
dir_arg=
dst_arg=
copy_on_change=false
no_target_directory=
usage="\
Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
or: $0 [OPTION]... SRCFILES... DIRECTORY
or: $0 [OPTION]... -t DIRECTORY SRCFILES...
or: $0 [OPTION]... -d DIRECTORIES...
In the 1st form, copy SRCFILE to DSTFILE.
In the 2nd and 3rd, copy all SRCFILES to DIRECTORY.
In the 4th, create DIRECTORIES.
Options:
--help display this help and exit.
--version display version info and exit.
-c (ignored)
-C install only if different (preserve the last data modification time)
-d create directories instead of installing files.
-g GROUP $chgrpprog installed files to GROUP.
-m MODE $chmodprog installed files to MODE.
-o USER $chownprog installed files to USER.
-s $stripprog installed files.
-t DIRECTORY install into DIRECTORY.
-T report an error if DSTFILE is a directory.
Environment variables override the default commands:
CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
RMPROG STRIPPROG
"
while test $# -ne 0; do
case $1 in
-c) ;;
-C) copy_on_change=true;;
-d) dir_arg=true;;
-g) chgrpcmd="$chgrpprog $2"
shift;;
--help) echo "$usage"; exit $?;;
-m) mode=$2
case $mode in
*' '* | *' '* | *'
'* | *'*'* | *'?'* | *'['*)
echo "$0: invalid mode: $mode" >&2
exit 1;;
esac
shift;;
-o) chowncmd="$chownprog $2"
shift;;
-s) stripcmd=$stripprog;;
-t) dst_arg=$2
# Protect names problematic for `test' and other utilities.
case $dst_arg in
-* | [=\(\)!]) dst_arg=./$dst_arg;;
esac
shift;;
-T) no_target_directory=true;;
--version) echo "$0 $scriptversion"; exit $?;;
--) shift
break;;
-*) echo "$0: invalid option: $1" >&2
exit 1;;
*) break;;
esac
shift
done
if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
# When -d is used, all remaining arguments are directories to create.
# When -t is used, the destination is already specified.
# Otherwise, the last argument is the destination. Remove it from $@.
for arg
do
if test -n "$dst_arg"; then
# $@ is not empty: it contains at least $arg.
set fnord "$@" "$dst_arg"
shift # fnord
fi
shift # arg
dst_arg=$arg
# Protect names problematic for `test' and other utilities.
case $dst_arg in
-* | [=\(\)!]) dst_arg=./$dst_arg;;
esac
done
fi
if test $# -eq 0; then
if test -z "$dir_arg"; then
echo "$0: no input file specified." >&2
exit 1
fi
# It's OK to call `install-sh -d' without argument.
# This can happen when creating conditional directories.
exit 0
fi
if test -z "$dir_arg"; then
do_exit='(exit $ret); exit $ret'
trap "ret=129; $do_exit" 1
trap "ret=130; $do_exit" 2
trap "ret=141; $do_exit" 13
trap "ret=143; $do_exit" 15
# Set umask so as not to create temps with too-generous modes.
# However, 'strip' requires both read and write access to temps.
case $mode in
# Optimize common cases.
*644) cp_umask=133;;
*755) cp_umask=22;;
*[0-7])
if test -z "$stripcmd"; then
u_plus_rw=
else
u_plus_rw='% 200'
fi
cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
*)
if test -z "$stripcmd"; then
u_plus_rw=
else
u_plus_rw=,u+rw
fi
cp_umask=$mode$u_plus_rw;;
esac
fi
for src
do
# Protect names problematic for `test' and other utilities.
case $src in
-* | [=\(\)!]) src=./$src;;
esac
if test -n "$dir_arg"; then
dst=$src
dstdir=$dst
test -d "$dstdir"
dstdir_status=$?
else
# Waiting for this to be detected by the "$cpprog $src $dsttmp" command
# might cause directories to be created, which would be especially bad
# if $src (and thus $dsttmp) contains '*'.
if test ! -f "$src" && test ! -d "$src"; then
echo "$0: $src does not exist." >&2
exit 1
fi
if test -z "$dst_arg"; then
echo "$0: no destination specified." >&2
exit 1
fi
dst=$dst_arg
# If destination is a directory, append the input filename; won't work
# if double slashes aren't ignored.
if test -d "$dst"; then
if test -n "$no_target_directory"; then
echo "$0: $dst_arg: Is a directory" >&2
exit 1
fi
dstdir=$dst
dst=$dstdir/`basename "$src"`
dstdir_status=0
else
# Prefer dirname, but fall back on a substitute if dirname fails.
dstdir=`
(dirname "$dst") 2>/dev/null ||
expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
X"$dst" : 'X\(//\)[^/]' \| \
X"$dst" : 'X\(//\)$' \| \
X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
echo X"$dst" |
sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
s//\1/
q
}
/^X\(\/\/\)[^/].*/{
s//\1/
q
}
/^X\(\/\/\)$/{
s//\1/
q
}
/^X\(\/\).*/{
s//\1/
q
}
s/.*/./; q'
`
test -d "$dstdir"
dstdir_status=$?
fi
fi
obsolete_mkdir_used=false
if test $dstdir_status != 0; then
case $posix_mkdir in
'')
# Create intermediate dirs using mode 755 as modified by the umask.
# This is like FreeBSD 'install' as of 1997-10-28.
umask=`umask`
case $stripcmd.$umask in
# Optimize common cases.
*[2367][2367]) mkdir_umask=$umask;;
.*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
*[0-7])
mkdir_umask=`expr $umask + 22 \
- $umask % 100 % 40 + $umask % 20 \
- $umask % 10 % 4 + $umask % 2
`;;
*) mkdir_umask=$umask,go-w;;
esac
# With -d, create the new directory with the user-specified mode.
# Otherwise, rely on $mkdir_umask.
if test -n "$dir_arg"; then
mkdir_mode=-m$mode
else
mkdir_mode=
fi
posix_mkdir=false
case $umask in
*[123567][0-7][0-7])
# POSIX mkdir -p sets u+wx bits regardless of umask, which
# is incompatible with FreeBSD 'install' when (umask & 300) != 0.
;;
*)
tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
if (umask $mkdir_umask &&
exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
then
if test -z "$dir_arg" || {
# Check for POSIX incompatibilities with -m.
# HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
# other-writeable bit of parent directory when it shouldn't.
# FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
ls_ld_tmpdir=`ls -ld "$tmpdir"`
case $ls_ld_tmpdir in
d????-?r-*) different_mode=700;;
d????-?--*) different_mode=755;;
*) false;;
esac &&
$mkdirprog -m$different_mode -p -- "$tmpdir" && {
ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
}
}
then posix_mkdir=:
fi
rmdir "$tmpdir/d" "$tmpdir"
else
# Remove any dirs left behind by ancient mkdir implementations.
rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
fi
trap '' 0;;
esac;;
esac
if
$posix_mkdir && (
umask $mkdir_umask &&
$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
)
then :
else
# The umask is ridiculous, or mkdir does not conform to POSIX,
# or it failed possibly due to a race condition. Create the
# directory the slow way, step by step, checking for races as we go.
case $dstdir in
/*) prefix='/';;
[-=\(\)!]*) prefix='./';;
*) prefix='';;
esac
eval "$initialize_posix_glob"
oIFS=$IFS
IFS=/
$posix_glob set -f
set fnord $dstdir
shift
$posix_glob set +f
IFS=$oIFS
prefixes=
for d
do
test X"$d" = X && continue
prefix=$prefix$d
if test -d "$prefix"; then
prefixes=
else
if $posix_mkdir; then
(umask=$mkdir_umask &&
$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
# Don't fail if two instances are running concurrently.
test -d "$prefix" || exit 1
else
case $prefix in
*\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
*) qprefix=$prefix;;
esac
prefixes="$prefixes '$qprefix'"
fi
fi
prefix=$prefix/
done
if test -n "$prefixes"; then
# Don't fail if two instances are running concurrently.
(umask $mkdir_umask &&
eval "\$doit_exec \$mkdirprog $prefixes") ||
test -d "$dstdir" || exit 1
obsolete_mkdir_used=true
fi
fi
fi
if test -n "$dir_arg"; then
{ test -z "$chowncmd" || $doit $chowncmd "$dst"; } &&
{ test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } &&
{ test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false ||
test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1
else
# Make a couple of temp file names in the proper directory.
dsttmp=$dstdir/_inst.$$_
rmtmp=$dstdir/_rm.$$_
# Trap to clean up those temp files at exit.
trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0
# Copy the file name to the temp name.
(umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
# and set any options; do chmod last to preserve setuid bits.
#
# If any of these fail, we abort the whole thing. If we want to
# ignore errors from any of these, just make sure not to ignore
# errors from the above "$doit $cpprog $src $dsttmp" command.
#
{ test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } &&
{ test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } &&
{ test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } &&
{ test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } &&
# If -C, don't bother to copy if it wouldn't change the file.
if $copy_on_change &&
old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` &&
new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` &&
eval "$initialize_posix_glob" &&
$posix_glob set -f &&
set X $old && old=:$2:$4:$5:$6 &&
set X $new && new=:$2:$4:$5:$6 &&
$posix_glob set +f &&
test "$old" = "$new" &&
$cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
then
rm -f "$dsttmp"
else
# Rename the file to the real destination.
$doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||
# The rename failed, perhaps because mv can't rename something else
# to itself, or perhaps because mv is so ancient that it does not
# support -f.
{
# Now remove or move aside any old file at destination location.
# We try this two ways since rm can't unlink itself on some
# systems and the destination file might be busy for other
# reasons. In this case, the final cleanup might fail but the new
# file should still install successfully.
{
test ! -f "$dst" ||
$doit $rmcmd -f "$dst" 2>/dev/null ||
{ $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
{ $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
} ||
{ echo "$0: cannot unlink or rename $dst" >&2
(exit 1); exit 1
}
} &&
# Now rename the file to the real destination.
$doit $mvcmd "$dsttmp" "$dst"
}
fi || exit 1
trap '' 0
fi
done
# Local variables:
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:

9655
autoconf/ltmain.sh
View File

File diff suppressed because it is too large Load Diff

331
autoconf/missing
View File

@ -1,331 +0,0 @@
#! /bin/sh
# Common stub for a few missing GNU programs while installing.
scriptversion=2012-01-06.13; # UTC
# Copyright (C) 1996, 1997, 1999, 2000, 2002, 2003, 2004, 2005, 2006,
# 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
# Originally by Fran,cois Pinard <pinard@iro.umontreal.ca>, 1996.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
if test $# -eq 0; then
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
fi
run=:
sed_output='s/.* --output[ =]\([^ ]*\).*/\1/p'
sed_minuso='s/.* -o \([^ ]*\).*/\1/p'
# In the cases where this matters, `missing' is being run in the
# srcdir already.
if test -f configure.ac; then
configure_ac=configure.ac
else
configure_ac=configure.in
fi
msg="missing on your system"
case $1 in
--run)
# Try to run requested program, and just exit if it succeeds.
run=
shift
"$@" && exit 0
# Exit code 63 means version mismatch. This often happens
# when the user try to use an ancient version of a tool on
# a file that requires a minimum version. In this case we
# we should proceed has if the program had been absent, or
# if --run hadn't been passed.
if test $? = 63; then
run=:
msg="probably too old"
fi
;;
-h|--h|--he|--hel|--help)
echo "\
$0 [OPTION]... PROGRAM [ARGUMENT]...
Handle \`PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an
error status if there is no known handling for PROGRAM.
Options:
-h, --help display this help and exit
-v, --version output version information and exit
--run try to run the given command, and emulate it if it fails
Supported PROGRAM values:
aclocal touch file \`aclocal.m4'
autoconf touch file \`configure'
autoheader touch file \`config.h.in'
autom4te touch the output file, or create a stub one
automake touch all \`Makefile.in' files
bison create \`y.tab.[ch]', if possible, from existing .[ch]
flex create \`lex.yy.c', if possible, from existing .c
help2man touch the output file
lex create \`lex.yy.c', if possible, from existing .c
makeinfo touch the output file
yacc create \`y.tab.[ch]', if possible, from existing .[ch]
Version suffixes to PROGRAM as well as the prefixes \`gnu-', \`gnu', and
\`g' are ignored when checking the name.
Send bug reports to <bug-automake@gnu.org>."
exit $?
;;
-v|--v|--ve|--ver|--vers|--versi|--versio|--version)
echo "missing $scriptversion (GNU Automake)"
exit $?
;;
-*)
echo 1>&2 "$0: Unknown \`$1' option"
echo 1>&2 "Try \`$0 --help' for more information"
exit 1
;;
esac
# normalize program name to check for.
program=`echo "$1" | sed '
s/^gnu-//; t
s/^gnu//; t
s/^g//; t'`
# Now exit if we have it, but it failed. Also exit now if we
# don't have it and --version was passed (most likely to detect
# the program). This is about non-GNU programs, so use $1 not
# $program.
case $1 in
lex*|yacc*)
# Not GNU programs, they don't have --version.
;;
*)
if test -z "$run" && ($1 --version) > /dev/null 2>&1; then
# We have it, but it failed.
exit 1
elif test "x$2" = "x--version" || test "x$2" = "x--help"; then
# Could not run --version or --help. This is probably someone
# running `$TOOL --version' or `$TOOL --help' to check whether
# $TOOL exists and not knowing $TOOL uses missing.
exit 1
fi
;;
esac
# If it does not exist, or fails to run (possibly an outdated version),
# try to emulate it.
case $program in
aclocal*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`acinclude.m4' or \`${configure_ac}'. You might want
to install the \`Automake' and \`Perl' packages. Grab them from
any GNU archive site."
touch aclocal.m4
;;
autoconf*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`${configure_ac}'. You might want to install the
\`Autoconf' and \`GNU m4' packages. Grab them from any GNU
archive site."
touch configure
;;
autoheader*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`acconfig.h' or \`${configure_ac}'. You might want
to install the \`Autoconf' and \`GNU m4' packages. Grab them
from any GNU archive site."
files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER(\([^)]*\)).*/\1/p' ${configure_ac}`
test -z "$files" && files="config.h"
touch_files=
for f in $files; do
case $f in
*:*) touch_files="$touch_files "`echo "$f" |
sed -e 's/^[^:]*://' -e 's/:.*//'`;;
*) touch_files="$touch_files $f.in";;
esac
done
touch $touch_files
;;
automake*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified \`Makefile.am', \`acinclude.m4' or \`${configure_ac}'.
You might want to install the \`Automake' and \`Perl' packages.
Grab them from any GNU archive site."
find . -type f -name Makefile.am -print |
sed 's/\.am$/.in/' |
while read f; do touch "$f"; done
;;
autom4te*)
echo 1>&2 "\
WARNING: \`$1' is needed, but is $msg.
You might have modified some files without having the
proper tools for further handling them.
You can get \`$1' as part of \`Autoconf' from any GNU
archive site."
file=`echo "$*" | sed -n "$sed_output"`
test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
if test -f "$file"; then
touch $file
else
test -z "$file" || exec >$file
echo "#! /bin/sh"
echo "# Created by GNU Automake missing as a replacement of"
echo "# $ $@"
echo "exit 0"
chmod +x $file
exit 1
fi
;;
bison*|yacc*)
echo 1>&2 "\
WARNING: \`$1' $msg. You should only need it if
you modified a \`.y' file. You may need the \`Bison' package
in order for those modifications to take effect. You can get
\`Bison' from any GNU archive site."
rm -f y.tab.c y.tab.h
if test $# -ne 1; then
eval LASTARG=\${$#}
case $LASTARG in
*.y)
SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'`
if test -f "$SRCFILE"; then
cp "$SRCFILE" y.tab.c
fi
SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'`
if test -f "$SRCFILE"; then
cp "$SRCFILE" y.tab.h
fi
;;
esac
fi
if test ! -f y.tab.h; then
echo >y.tab.h
fi
if test ! -f y.tab.c; then
echo 'main() { return 0; }' >y.tab.c
fi
;;
lex*|flex*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified a \`.l' file. You may need the \`Flex' package
in order for those modifications to take effect. You can get
\`Flex' from any GNU archive site."
rm -f lex.yy.c
if test $# -ne 1; then
eval LASTARG=\${$#}
case $LASTARG in
*.l)
SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'`
if test -f "$SRCFILE"; then
cp "$SRCFILE" lex.yy.c
fi
;;
esac
fi
if test ! -f lex.yy.c; then
echo 'main() { return 0; }' >lex.yy.c
fi
;;
help2man*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified a dependency of a manual page. You may need the
\`Help2man' package in order for those modifications to take
effect. You can get \`Help2man' from any GNU archive site."
file=`echo "$*" | sed -n "$sed_output"`
test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
if test -f "$file"; then
touch $file
else
test -z "$file" || exec >$file
echo ".ab help2man is required to generate this page"
exit $?
fi
;;
makeinfo*)
echo 1>&2 "\
WARNING: \`$1' is $msg. You should only need it if
you modified a \`.texi' or \`.texinfo' file, or any other file
indirectly affecting the aspect of the manual. The spurious
call might also be the consequence of using a buggy \`make' (AIX,
DU, IRIX). You might want to install the \`Texinfo' package or
the \`GNU make' package. Grab either from any GNU archive site."
# The file to touch is that specified with -o ...
file=`echo "$*" | sed -n "$sed_output"`
test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"`
if test -z "$file"; then
# ... or it is the one specified with @setfilename ...
infile=`echo "$*" | sed 's/.* \([^ ]*\) *$/\1/'`
file=`sed -n '
/^@setfilename/{
s/.* \([^ ]*\) *$/\1/
p
q
}' $infile`
# ... or it is derived from the source name (dir/f.texi becomes f.info)
test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info
fi
# If the file does not exist, the user really needs makeinfo;
# let's fail without touching anything.
test -f $file || exit 1
touch $file
;;
*)
echo 1>&2 "\
WARNING: \`$1' is needed, and is $msg.
You might have modified some files without having the
proper tools for further handling them. Check the \`README' file,
it often tells you about the needed prerequisites for installing
this package. You may also peek at any GNU archive site, in case
some other package would contain this missing \`$1' program."
exit 1
;;
esac
exit 0
# Local variables:
# eval: (add-hook 'write-file-hooks 'time-stamp)
# time-stamp-start: "scriptversion="
# time-stamp-format: "%:y-%02m-%02d.%02H"
# time-stamp-time-zone: "UTC"
# time-stamp-end: "; # UTC"
# End:

2
autoconf/tea/Makefile.in
View File

@ -348,7 +348,7 @@ clean:
distclean: clean
-rm -f *.tab.c
-rm -f $(CONFIG_CLEAN_FILES)
-rm -f config.cache config.log config.status
-rm -f config.h config.cache config.log config.status
#========================================================================
# Install binary object libraries. On Windows this includes both .dll and

6
autoconf/tea/win/nmakehlp.c
View File

@ -603,8 +603,8 @@ SubstituteFile(
sp = fopen(substitutions, "rt");
if (sp != NULL) {
while (fgets(szBuffer, cbBuffer, sp) != NULL) {
char *ks, *ke, *vs, *ve;
ks = szBuffer;
unsigned char *ks, *ke, *vs, *ve;
ks = (unsigned char*)szBuffer;
while (ks && *ks && isspace(*ks)) ++ks;
ke = ks;
while (ke && *ke && !isspace(*ke)) ++ke;
@ -613,7 +613,7 @@ SubstituteFile(
ve = vs;
while (ve && *ve && !(*ve == '\r' || *ve == '\n')) ++ve;
*ke = 0, *ve = 0;
list_insert(&substPtr, ks, vs);
list_insert(&substPtr, (char*)ks, (char*)vs);
}
fclose(sp);
}

4482
configure vendored
View File

File diff suppressed because it is too large Load Diff

73
configure.ac
View File

@ -90,7 +90,6 @@ fi
#
AC_PROG_LIBTOOL
AC_PROG_INSTALL
AC_PROG_AWK
#########
# Enable large file support (if special flags are necessary)
@ -195,6 +194,7 @@ AC_SUBST(SQLITE_THREADSAFE)
if test "$SQLITE_THREADSAFE" = "1"; then
AC_SEARCH_LIBS(pthread_create, pthread)
AC_SEARCH_LIBS(pthread_mutexattr_init, pthread)
fi
##########
@ -367,6 +367,20 @@ if test "${use_tcl}" = "yes" ; then
fi
fi
# On ubuntu 14.10, $auto_path on tclsh is not quite correct.
# So try again after applying corrections.
if test x"${ac_cv_c_tclconfig}" = x ; then
if test x"$cross_compiling" = xno; then
for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
do
if test -f "$i/tclConfig.sh" ; then
ac_cv_c_tclconfig="$i"
break
fi
done
fi
fi
# then check for a private Tcl installation
if test x"${ac_cv_c_tclconfig}" = x ; then
for i in \
@ -494,11 +508,24 @@ AC_SUBST(HAVE_TCL)
TARGET_READLINE_LIBS=""
TARGET_READLINE_INC=""
TARGET_HAVE_READLINE=0
TARGET_HAVE_EDITLINE=0
AC_ARG_ENABLE([editline],
[AC_HELP_STRING([--enable-editline],[enable BSD editline support])],
[with_editline=$enableval],
[with_editline=auto])
AC_ARG_ENABLE([readline],
[AC_HELP_STRING([--disable-readline],[disable readline support [default=detect]])],
[AC_HELP_STRING([--disable-readline],[disable readline support])],
[with_readline=$enableval],
[with_readline=auto])
if test x"$with_editline" != xno; then
sLIBS=$LIBS
LIBS=""
TARGET_HAVE_EDITLINE=1
AC_SEARCH_LIBS(readline,edit,[with_readline=no],[TARGET_HAVE_EDITLINE=0])
TARGET_READLINE_LIBS=$LIBS
LIBS=$sLIBS
fi
if test x"$with_readline" != xno; then
found="yes"
@ -553,6 +580,7 @@ fi
AC_SUBST(TARGET_READLINE_LIBS)
AC_SUBST(TARGET_READLINE_INC)
AC_SUBST(TARGET_HAVE_READLINE)
AC_SUBST(TARGET_HAVE_EDITLINE)
##########
# Figure out what C libraries are required to compile programs
@ -593,6 +621,47 @@ else
OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1"
fi
#########
# See whether we should enable Full Text Search extensions
AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
[Enable the FTS3 extension]),
[enable_fts3=yes],[enable_fts3=no])
if test "${enable_fts3}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3"
fi
AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
[Enable the FTS4 extension]),
[enable_fts4=yes],[enable_fts4=no])
if test "${enable_fts4}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4"
AC_SEARCH_LIBS([log],[m])
fi
AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
[Enable the FTS5 extension]),
[enable_fts5=yes],[enable_fts5=no])
if test "${enable_fts5}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5"
AC_SEARCH_LIBS([log],[m])
fi
#########
# See whether we should enable JSON1
AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1],
[Enable the JSON1 extension]),
[enable_json1=yes],[enable_json1=no])
if test "${enable_json1}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1"
fi
#########
# See whether we should enable RTREE
AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
[Enable the RTREE extension]),
[enable_rtree=yes],[enable_rtree=no])
if test "${enable_rtree}" = "yes" ; then
OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE"
fi
#########
# attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter
for option in $CFLAGS $CPPFLAGS

16
ext/async/sqlite3async.c
View File

@ -1636,6 +1636,7 @@ void sqlite3async_run(void){
** Control/configure the asynchronous IO system.
*/
int sqlite3async_control(int op, ...){
int rc = SQLITE_OK;
va_list ap;
va_start(ap, op);
switch( op ){
@ -1645,7 +1646,8 @@ int sqlite3async_control(int op, ...){
&& eWhen!=SQLITEASYNC_HALT_NOW
&& eWhen!=SQLITEASYNC_HALT_IDLE
){
return SQLITE_MISUSE;
rc = SQLITE_MISUSE;
break;
}
async.eHalt = eWhen;
async_mutex_enter(ASYNC_MUTEX_QUEUE);
@ -1657,7 +1659,8 @@ int sqlite3async_control(int op, ...){
case SQLITEASYNC_DELAY: {
int iDelay = va_arg(ap, int);
if( iDelay<0 ){
return SQLITE_MISUSE;
rc = SQLITE_MISUSE;
break;
}
async.ioDelay = iDelay;
break;
@ -1668,7 +1671,8 @@ int sqlite3async_control(int op, ...){
async_mutex_enter(ASYNC_MUTEX_QUEUE);
if( async.nFile || async.pQueueFirst ){
async_mutex_leave(ASYNC_MUTEX_QUEUE);
return SQLITE_MISUSE;
rc = SQLITE_MISUSE;
break;
}
async.bLockFiles = bLock;
async_mutex_leave(ASYNC_MUTEX_QUEUE);
@ -1692,9 +1696,11 @@ int sqlite3async_control(int op, ...){
}
default:
return SQLITE_ERROR;
rc = SQLITE_ERROR;
break;
}
return SQLITE_OK;
va_end(ap);
return rc;
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ASYNCIO) */

6
ext/fts1/fts1.c
View File

@ -205,13 +205,13 @@ static int getVarint32(const char *p, int *pi){
*/
/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
static int safe_isspace(char c){
return (c&0x80)==0 ? isspace(c) : 0;
return (c&0x80)==0 ? isspace((unsigned char)c) : 0;
}
static int safe_tolower(char c){
return (c&0x80)==0 ? tolower(c) : c;
return (c&0x80)==0 ? tolower((unsigned char)c) : c;
}
static int safe_isalnum(char c){
return (c&0x80)==0 ? isalnum(c) : 0;
return (c&0x80)==0 ? isalnum((unsigned char)c) : 0;
}
typedef enum DocListType {

2
ext/fts1/simple_tokenizer.c
View File

@ -138,7 +138,7 @@ static int simpleNext(
** case-insensitivity.
*/
char ch = c->pCurrent[ii];
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch;
}
c->zToken[n] = '\0';
*ppToken = c->zToken;

99
ext/fts3/fts3.c
View File

@ -1517,6 +1517,19 @@ static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
#endif
}
/*
** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
** extension is currently being used by a version of SQLite too old to
** support index-info flags. In that case this function is a no-op.
*/
static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){
#if SQLITE_VERSION_NUMBER>=3008012
if( sqlite3_libversion_number()>=3008012 ){
pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
}
#endif
}
/*
** Implementation of the xBestIndex method for FTS3 tables. There
** are three possible strategies, in order of preference:
@ -1607,6 +1620,9 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
}
}
/* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */
if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo);
iIdx = 1;
if( iCons>=0 ){
pInfo->aConstraintUsage[iCons].argvIndex = iIdx++;
@ -1675,7 +1691,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
sqlite3Fts3ExprFree(pCsr->pExpr);
sqlite3Fts3FreeDeferredTokens(pCsr);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_free(pCsr);
return SQLITE_OK;
@ -3176,7 +3192,7 @@ static int fts3FilterMethod(
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr->aDoclist);
sqlite3_free(pCsr->aMatchinfo);
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
sqlite3Fts3ExprFree(pCsr->pExpr);
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
@ -4231,7 +4247,6 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
int bIncrOk = (bOptOk
&& pCsr->bDesc==pTab->bDescIdx
&& p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
&& p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
#ifdef SQLITE_TEST
&& pTab->bNoIncrDoclist==0
#endif
@ -4351,6 +4366,7 @@ void sqlite3Fts3DoclistNext(
p += sqlite3Fts3GetVarint(p, piDocid);
}else{
fts3PoslistCopy(0, &p);
while( p<&aDoclist[nDoclist] && *p==0 ) p++;
if( p>=&aDoclist[nDoclist] ){
*pbEof = 1;
}else{
@ -5074,7 +5090,7 @@ static int fts3EvalNearTrim(
** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is
** advanced to point to the next row that matches "x AND y".
**
** See fts3EvalTestDeferredAndNear() for details on testing if a row is
** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is
** really a match, taking into account deferred tokens and NEAR operators.
*/
static void fts3EvalNextRow(
@ -5294,7 +5310,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
}
/*
** This function is a helper function for fts3EvalTestDeferredAndNear().
** This function is a helper function for sqlite3Fts3EvalTestDeferred().
** Assuming no error occurs or has occurred, It returns non-zero if the
** expression passed as the second argument matches the row that pCsr
** currently points to, or zero if it does not.
@ -5415,7 +5431,7 @@ static int fts3EvalTestExpr(
** Or, if no error occurs and it seems the current row does match the FTS
** query, return 0.
*/
static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){
int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){
int rc = *pRc;
int bMiss = 0;
if( rc==SQLITE_OK ){
@ -5462,7 +5478,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){
pCsr->isRequireSeek = 1;
pCsr->isMatchinfoNeeded = 1;
pCsr->iPrevId = pExpr->iDocid;
}while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
}while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) );
}
/* Check if the cursor is past the end of the docid range specified
@ -5623,7 +5639,7 @@ static int fts3EvalGatherStats(
pCsr->iPrevId = pRoot->iDocid;
}while( pCsr->isEof==0
&& pRoot->eType==FTSQUERY_NEAR
&& fts3EvalTestDeferredAndNear(pCsr, &rc)
&& sqlite3Fts3EvalTestDeferred(pCsr, &rc)
);
if( rc==SQLITE_OK && pCsr->isEof==0 ){
@ -5648,7 +5664,6 @@ static int fts3EvalGatherStats(
fts3EvalNextRow(pCsr, pRoot, &rc);
assert( pRoot->bEof==0 );
}while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
fts3EvalTestDeferredAndNear(pCsr, &rc);
}
}
return rc;
@ -5758,10 +5773,10 @@ int sqlite3Fts3EvalPhrasePoslist(
int rc = SQLITE_OK;
int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
int bOr = 0;
u8 bEof = 0;
u8 bTreeEof = 0;
Fts3Expr *p; /* Used to iterate from pExpr to root */
Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
int bMatch;
/* Check if this phrase descends from an OR expression node. If not,
** return NULL. Otherwise, the entry that corresponds to docid
@ -5795,31 +5810,47 @@ int sqlite3Fts3EvalPhrasePoslist(
}
if( rc!=SQLITE_OK ) return rc;
pIter = pPhrase->pOrPoslist;
iDocid = pPhrase->iOrDocid;
if( pCsr->bDesc==bDescDoclist ){
bEof = !pPhrase->doclist.nAll ||
(pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
}else{
bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
int dummy;
sqlite3Fts3DoclistPrev(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}
pPhrase->pOrPoslist = pIter;
pPhrase->iOrDocid = iDocid;
bMatch = 1;
for(p=pNear; p; p=p->pLeft){
u8 bEof = 0;
Fts3Expr *pTest = p;
Fts3Phrase *pPh;
assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
assert( pTest->eType==FTSQUERY_PHRASE );
pPh = pTest->pPhrase;
if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
pIter = pPh->pOrPoslist;
iDocid = pPh->iOrDocid;
if( pCsr->bDesc==bDescDoclist ){
bEof = !pPh->doclist.nAll ||
(pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
}else{
bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
int dummy;
sqlite3Fts3DoclistPrev(
bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}
pPh->pOrPoslist = pIter;
pPh->iOrDocid = iDocid;
if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
}
if( bMatch ){
pIter = pPhrase->pOrPoslist;
}else{
pIter = 0;
}
}
if( pIter==0 ) return SQLITE_OK;

19
ext/fts3/fts3Int.h
View File

@ -18,6 +18,12 @@
# define NDEBUG 1
#endif
/* FTS3/FTS4 require virtual tables */
#ifdef SQLITE_OMIT_VIRTUALTABLE
# undef SQLITE_ENABLE_FTS3
# undef SQLITE_ENABLE_FTS4
#endif
/*
** FTS4 is really an extension for FTS3. It is enabled using the
** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all
@ -197,6 +203,8 @@ typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3MultiSegReader Fts3MultiSegReader;
typedef struct MatchinfoBuffer MatchinfoBuffer;
/*
** A connection to a fulltext index is an instance of the following
** structure. The xCreate and xConnect methods create an instance
@ -262,6 +270,7 @@ struct Fts3Table {
int nPendingData; /* Current bytes of pending data */
sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
int iPrevLangid; /* Langid of recently inserted document */
int bPrevDelete; /* True if last operation was a delete */
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
/* State variables used for validating that the transaction control
@ -306,9 +315,7 @@ struct Fts3Cursor {
i64 iMinDocid; /* Minimum docid to return */
i64 iMaxDocid; /* Maximum docid to return */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
int nMatchinfo; /* Number of elements in aMatchinfo[] */
char *zMatchinfo; /* Matchinfo specification */
MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */
};
#define FTS3_EVAL_FILTER 0
@ -428,7 +435,9 @@ struct Fts3Expr {
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
u32 *aMI;
/* The following are used by the fts3_snippet.c module. */
int iPhrase; /* Index of this phrase in matchinfo() results */
u32 *aMI; /* See above */
};
/*
@ -549,6 +558,7 @@ void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
@ -564,6 +574,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,

226
ext/fts3/fts3_expr.c
View File

@ -793,125 +793,151 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){
rc = SQLITE_ERROR;
}
if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
Fts3Expr **apLeaf;
apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
if( 0==apLeaf ){
rc = SQLITE_NOMEM;
}else{
memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
}
if( rc==SQLITE_OK ){
int i;
Fts3Expr *p;
/* Set $p to point to the left-most leaf in the tree of eType nodes. */
for(p=pRoot; p->eType==eType; p=p->pLeft){
assert( p->pParent==0 || p->pParent->pLeft==p );
assert( p->pLeft && p->pRight );
}
/* This loop runs once for each leaf in the tree of eType nodes. */
while( 1 ){
int iLvl;
Fts3Expr *pParent = p->pParent; /* Current parent of p */
assert( pParent==0 || pParent->pLeft==p );
p->pParent = 0;
if( pParent ){
pParent->pLeft = 0;
}else{
pRoot = 0;
}
rc = fts3ExprBalance(&p, nMaxDepth-1);
if( rc!=SQLITE_OK ) break;
for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
if( apLeaf[iLvl]==0 ){
apLeaf[iLvl] = p;
p = 0;
}else{
assert( pFree );
pFree->pLeft = apLeaf[iLvl];
pFree->pRight = p;
pFree->pLeft->pParent = pFree;
pFree->pRight->pParent = pFree;
p = pFree;
pFree = pFree->pParent;
p->pParent = 0;
apLeaf[iLvl] = 0;
}
}
if( p ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_TOOBIG;
break;
}
/* If that was the last leaf node, break out of the loop */
if( pParent==0 ) break;
/* Set $p to point to the next leaf in the tree of eType nodes */
for(p=pParent->pRight; p->eType==eType; p=p->pLeft);
/* Remove pParent from the original tree. */
assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
pParent->pRight->pParent = pParent->pParent;
if( pParent->pParent ){
pParent->pParent->pLeft = pParent->pRight;
}else{
assert( pParent==pRoot );
pRoot = pParent->pRight;
}
/* Link pParent into the free node list. It will be used as an
** internal node of the new tree. */
pParent->pParent = pFree;
pFree = pParent;
if( rc==SQLITE_OK ){
if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
Fts3Expr **apLeaf;
apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
if( 0==apLeaf ){
rc = SQLITE_NOMEM;
}else{
memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth);
}
if( rc==SQLITE_OK ){
p = 0;
for(i=0; i<nMaxDepth; i++){
if( apLeaf[i] ){
if( p==0 ){
p = apLeaf[i];
p->pParent = 0;
int i;
Fts3Expr *p;
/* Set $p to point to the left-most leaf in the tree of eType nodes. */
for(p=pRoot; p->eType==eType; p=p->pLeft){
assert( p->pParent==0 || p->pParent->pLeft==p );
assert( p->pLeft && p->pRight );
}
/* This loop runs once for each leaf in the tree of eType nodes. */
while( 1 ){
int iLvl;
Fts3Expr *pParent = p->pParent; /* Current parent of p */
assert( pParent==0 || pParent->pLeft==p );
p->pParent = 0;
if( pParent ){
pParent->pLeft = 0;
}else{
pRoot = 0;
}
rc = fts3ExprBalance(&p, nMaxDepth-1);
if( rc!=SQLITE_OK ) break;
for(iLvl=0; p && iLvl<nMaxDepth; iLvl++){
if( apLeaf[iLvl]==0 ){
apLeaf[iLvl] = p;
p = 0;
}else{
assert( pFree!=0 );
assert( pFree );
pFree->pLeft = apLeaf[iLvl];
pFree->pRight = p;
pFree->pLeft = apLeaf[i];
pFree->pLeft->pParent = pFree;
pFree->pRight->pParent = pFree;
p = pFree;
pFree = pFree->pParent;
p->pParent = 0;
apLeaf[iLvl] = 0;
}
}
if( p ){
sqlite3Fts3ExprFree(p);
rc = SQLITE_TOOBIG;
break;
}
/* If that was the last leaf node, break out of the loop */
if( pParent==0 ) break;
/* Set $p to point to the next leaf in the tree of eType nodes */
for(p=pParent->pRight; p->eType==eType; p=p->pLeft);
/* Remove pParent from the original tree. */
assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent );
pParent->pRight->pParent = pParent->pParent;
if( pParent->pParent ){
pParent->pParent->pLeft = pParent->pRight;
}else{
assert( pParent==pRoot );
pRoot = pParent->pRight;
}
/* Link pParent into the free node list. It will be used as an
** internal node of the new tree. */
pParent->pParent = pFree;
pFree = pParent;
}
pRoot = p;
}else{
/* An error occurred. Delete the contents of the apLeaf[] array
** and pFree list. Everything else is cleaned up by the call to
** sqlite3Fts3ExprFree(pRoot) below. */
Fts3Expr *pDel;
for(i=0; i<nMaxDepth; i++){
sqlite3Fts3ExprFree(apLeaf[i]);
}
while( (pDel=pFree)!=0 ){
pFree = pDel->pParent;
sqlite3_free(pDel);
if( rc==SQLITE_OK ){
p = 0;
for(i=0; i<nMaxDepth; i++){
if( apLeaf[i] ){
if( p==0 ){
p = apLeaf[i];
p->pParent = 0;
}else{
assert( pFree!=0 );
pFree->pRight = p;
pFree->pLeft = apLeaf[i];
pFree->pLeft->pParent = pFree;
pFree->pRight->pParent = pFree;
p = pFree;
pFree = pFree->pParent;
p->pParent = 0;
}
}
}
pRoot = p;
}else{
/* An error occurred. Delete the contents of the apLeaf[] array
** and pFree list. Everything else is cleaned up by the call to
** sqlite3Fts3ExprFree(pRoot) below. */
Fts3Expr *pDel;
for(i=0; i<nMaxDepth; i++){
sqlite3Fts3ExprFree(apLeaf[i]);
}
while( (pDel=pFree)!=0 ){
pFree = pDel->pParent;
sqlite3_free(pDel);
}
}
assert( pFree==0 );
sqlite3_free( apLeaf );
}
}else if( eType==FTSQUERY_NOT ){
Fts3Expr *pLeft = pRoot->pLeft;
Fts3Expr *pRight = pRoot->pRight;
pRoot->pLeft = 0;
pRoot->pRight = 0;
pLeft->pParent = 0;
pRight->pParent = 0;
rc = fts3ExprBalance(&pLeft, nMaxDepth-1);
if( rc==SQLITE_OK ){
rc = fts3ExprBalance(&pRight, nMaxDepth-1);
}
assert( pFree==0 );
sqlite3_free( apLeaf );
if( rc!=SQLITE_OK ){
sqlite3Fts3ExprFree(pRight);
sqlite3Fts3ExprFree(pLeft);
}else{
assert( pLeft && pRight );
pRoot->pLeft = pLeft;
pLeft->pParent = pRoot;
pRoot->pRight = pRight;
pRight->pParent = pRoot;
}
}
}
if( rc!=SQLITE_OK ){
sqlite3Fts3ExprFree(pRoot);
pRoot = 0;

13
ext/fts3/fts3_icu.c
View File

@ -240,12 +240,13 @@ static int icuNext(
** The set of routines that implement the simple tokenizer
*/
static const sqlite3_tokenizer_module icuTokenizerModule = {
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
0, /* iVersion */
icuCreate, /* xCreate */
icuDestroy, /* xCreate */
icuOpen, /* xOpen */
icuClose, /* xClose */
icuNext, /* xNext */
0, /* xLanguageid */
};
/*

314
ext/fts3/fts3_snippet.c
View File

@ -28,6 +28,7 @@
#define FTS3_MATCHINFO_LCS 's' /* nCol values */
#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */
#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */
/*
** The default value for the second argument to matchinfo().
@ -89,9 +90,22 @@ struct MatchInfo {
int nCol; /* Number of columns in table */
int nPhrase; /* Number of matchable phrases in query */
sqlite3_int64 nDoc; /* Number of docs in database */
char flag;
u32 *aMatchinfo; /* Pre-allocated buffer */
};
/*
** An instance of this structure is used to manage a pair of buffers, each
** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
** for details.
*/
struct MatchinfoBuffer {
u8 aRef[3];
int nElem;
int bGlobal; /* Set if global data is loaded */
char *zMatchinfo;
u32 aMatchinfo[1];
};
/*
@ -107,6 +121,97 @@ struct StrBuffer {
};
/*************************************************************************
** Start of MatchinfoBuffer code.
*/
/*
** Allocate a two-slot MatchinfoBuffer object.
*/
static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
MatchinfoBuffer *pRet;
int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer);
int nStr = (int)strlen(zMatchinfo);
pRet = sqlite3_malloc(nByte + nStr+1);
if( pRet ){
memset(pRet, 0, nByte);
pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
pRet->nElem = nElem;
pRet->zMatchinfo = ((char*)pRet) + nByte;
memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
pRet->aRef[0] = 1;
}
return pRet;
}
static void fts3MIBufferFree(void *p){
MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
assert( (u32*)p==&pBuf->aMatchinfo[1]
|| (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2]
);
if( (u32*)p==&pBuf->aMatchinfo[1] ){
pBuf->aRef[1] = 0;
}else{
pBuf->aRef[2] = 0;
}
if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
sqlite3_free(pBuf);
}
}
static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
void (*xRet)(void*) = 0;
u32 *aOut = 0;
if( p->aRef[1]==0 ){
p->aRef[1] = 1;
aOut = &p->aMatchinfo[1];
xRet = fts3MIBufferFree;
}
else if( p->aRef[2]==0 ){
p->aRef[2] = 1;
aOut = &p->aMatchinfo[p->nElem+2];
xRet = fts3MIBufferFree;
}else{
aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
if( aOut ){
xRet = sqlite3_free;
if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
}
}
*paOut = aOut;
return xRet;
}
static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
p->bGlobal = 1;
memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32));
}
/*
** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
*/
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
if( p ){
assert( p->aRef[0]==1 );
p->aRef[0] = 0;
if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
sqlite3_free(p);
}
}
}
/*
** End of MatchinfoBuffer code.
*************************************************************************/
/*
** This function is used to help iterate through a position-list. A position
** list is a list of unique integers, sorted from smallest to largest. Each
@ -143,7 +248,7 @@ static int fts3ExprIterate2(
void *pCtx /* Second argument to pass to callback */
){
int rc; /* Return code */
int eType = pExpr->eType; /* Type of expression node pExpr */
int eType = pExpr->eType; /* Type of expression node pExpr */
if( eType!=FTSQUERY_PHRASE ){
assert( pExpr->pLeft && pExpr->pRight );
@ -177,6 +282,7 @@ static int fts3ExprIterate(
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
@ -221,8 +327,7 @@ static int fts3ExprLoadDoclists(
static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
(*(int *)ctx)++;
UNUSED_PARAMETER(pExpr);
UNUSED_PARAMETER(iPhrase);
pExpr->iPhrase = iPhrase;
return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
@ -443,7 +548,7 @@ static int fts3BestSnippet(
sIter.nSnippet = nSnippet;
sIter.nPhrase = nList;
sIter.iCurrent = -1;
rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
if( rc==SQLITE_OK ){
/* Set the *pmSeen output variable. */
@ -744,6 +849,60 @@ static int fts3ColumnlistCount(char **ppCollist){
return nEntry;
}
/*
** This function gathers 'y' or 'b' data for a single phrase.
*/
static void fts3ExprLHits(
Fts3Expr *pExpr, /* Phrase expression node */
MatchInfo *p /* Matchinfo context */
){
Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
int iStart;
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter = pPhrase->doclist.pList;
int iCol = 0;
assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
if( p->flag==FTS3_MATCHINFO_LHITS ){
iStart = pExpr->iPhrase * p->nCol;
}else{
iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
}
while( 1 ){
int nHit = fts3ColumnlistCount(&pIter);
if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
if( p->flag==FTS3_MATCHINFO_LHITS ){
p->aMatchinfo[iStart + iCol] = (u32)nHit;
}else if( nHit ){
p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F));
}
}
assert( *pIter==0x00 || *pIter==0x01 );
if( *pIter!=0x01 ) break;
pIter++;
pIter += fts3GetVarint32(pIter, &iCol);
}
}
/*
** Gather the results for matchinfo directives 'y' and 'b'.
*/
static void fts3ExprLHitGather(
Fts3Expr *pExpr,
MatchInfo *p
){
assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
if( pExpr->pLeft ){
fts3ExprLHitGather(pExpr->pLeft, p);
fts3ExprLHitGather(pExpr->pRight, p);
}else{
fts3ExprLHits(pExpr, p);
}
}
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
@ -810,51 +969,6 @@ static int fts3ExprLocalHitsCb(
return rc;
}
/*
** fts3ExprIterate() callback used to gather information for the matchinfo
** directive 'y'.
*/
static int fts3ExprLHitsCb(
Fts3Expr *pExpr, /* Phrase expression node */
int iPhrase, /* Phrase number */
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
int rc = SQLITE_OK;
int iStart = iPhrase * p->nCol;
Fts3Expr *pEof; /* Ancestor node already at EOF */
/* This must be a phrase */
assert( pExpr->pPhrase );
/* Initialize all output integers to zero. */
memset(&p->aMatchinfo[iStart], 0, sizeof(u32) * p->nCol);
/* Check if this or any parent node is at EOF. If so, then all output
** values are zero. */
for(pEof=pExpr; pEof && pEof->bEof==0; pEof=pEof->pParent);
if( pEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter = pPhrase->doclist.pList;
int iCol = 0;
while( 1 ){
int nHit = fts3ColumnlistCount(&pIter);
if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
p->aMatchinfo[iStart + iCol] = (u32)nHit;
}
assert( *pIter==0x00 || *pIter==0x01 );
if( *pIter!=0x01 ) break;
pIter++;
pIter += fts3GetVarint32(pIter, &iCol);
}
}
return rc;
}
static int fts3MatchinfoCheck(
Fts3Table *pTab,
char cArg,
@ -868,6 +982,7 @@ static int fts3MatchinfoCheck(
|| (cArg==FTS3_MATCHINFO_LCS)
|| (cArg==FTS3_MATCHINFO_HITS)
|| (cArg==FTS3_MATCHINFO_LHITS)
|| (cArg==FTS3_MATCHINFO_LHITS_BM)
){
return SQLITE_OK;
}
@ -895,6 +1010,10 @@ static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
nVal = pInfo->nCol * pInfo->nPhrase;
break;
case FTS3_MATCHINFO_LHITS_BM:
nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32);
break;
default:
assert( cArg==FTS3_MATCHINFO_HITS );
nVal = pInfo->nCol * pInfo->nPhrase * 3;
@ -1089,7 +1208,7 @@ static int fts3MatchinfoValues(
sqlite3_stmt *pSelect = 0;
for(i=0; rc==SQLITE_OK && zArg[i]; i++){
pInfo->flag = zArg[i];
switch( zArg[i] ){
case FTS3_MATCHINFO_NPHRASE:
if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
@ -1149,9 +1268,13 @@ static int fts3MatchinfoValues(
}
break;
case FTS3_MATCHINFO_LHITS:
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
case FTS3_MATCHINFO_LHITS_BM:
case FTS3_MATCHINFO_LHITS: {
int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
memset(pInfo->aMatchinfo, 0, nZero);
fts3ExprLHitGather(pCsr->pExpr, pInfo);
break;
}
default: {
Fts3Expr *pExpr;
@ -1165,6 +1288,7 @@ static int fts3MatchinfoValues(
if( rc!=SQLITE_OK ) break;
}
rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
sqlite3Fts3EvalTestDeferred(pCsr, &rc);
if( rc!=SQLITE_OK ) break;
}
(void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
@ -1184,7 +1308,8 @@ static int fts3MatchinfoValues(
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(
static void fts3GetMatchinfo(
sqlite3_context *pCtx, /* Return results here */
Fts3Cursor *pCsr, /* FTS3 Cursor object */
const char *zArg /* Second argument to matchinfo() function */
){
@ -1193,6 +1318,9 @@ static int fts3GetMatchinfo(
int rc = SQLITE_OK;
int bGlobal = 0; /* Collect 'global' stats as well as local */
u32 *aOut = 0;
void (*xDestroyOut)(void*) = 0;
memset(&sInfo, 0, sizeof(MatchInfo));
sInfo.pCursor = pCsr;
sInfo.nCol = pTab->nColumn;
@ -1200,21 +1328,18 @@ static int fts3GetMatchinfo(
/* If there is cached matchinfo() data, but the format string for the
** cache does not match the format string for this request, discard
** the cached data. */
if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
assert( pCsr->aMatchinfo );
sqlite3_free(pCsr->aMatchinfo);
pCsr->zMatchinfo = 0;
pCsr->aMatchinfo = 0;
if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
pCsr->pMIBuffer = 0;
}
/* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
/* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
** matchinfo function has been called for this query. In this case
** allocate the array used to accumulate the matchinfo data and
** initialize those elements that are constant for every row.
*/
if( pCsr->aMatchinfo==0 ){
if( pCsr->pMIBuffer==0 ){
int nMatchinfo = 0; /* Number of u32 elements in match-info */
int nArg; /* Bytes in zArg */
int i; /* Used to iterate through zArg */
/* Determine the number of phrases in the query */
@ -1223,30 +1348,46 @@ static int fts3GetMatchinfo(
/* Determine the number of integers in the buffer returned by this call. */
for(i=0; zArg[i]; i++){
char *zErr = 0;
if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
sqlite3_result_error(pCtx, zErr, -1);
sqlite3_free(zErr);
return;
}
nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
}
/* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
nArg = (int)strlen(zArg);
pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
pCsr->nMatchinfo = nMatchinfo;
memcpy(pCsr->zMatchinfo, zArg, nArg+1);
memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
pCsr->isMatchinfoNeeded = 1;
bGlobal = 1;
}
sInfo.aMatchinfo = pCsr->aMatchinfo;
sInfo.nPhrase = pCsr->nPhrase;
if( pCsr->isMatchinfoNeeded ){
rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
pCsr->isMatchinfoNeeded = 0;
if( rc==SQLITE_OK ){
xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
if( xDestroyOut==0 ){
rc = SQLITE_NOMEM;
}
}
return rc;
if( rc==SQLITE_OK ){
sInfo.aMatchinfo = aOut;
sInfo.nPhrase = pCsr->nPhrase;
rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
if( bGlobal ){
fts3MIBufferSetGlobal(pCsr->pMIBuffer);
}
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
if( xDestroyOut ) xDestroyOut(aOut);
}else{
int n = pCsr->pMIBuffer->nElem * sizeof(u32);
sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
}
}
/*
@ -1452,7 +1593,7 @@ void sqlite3Fts3Offsets(
*/
sCtx.iCol = iCol;
sCtx.iTerm = 0;
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
/* Retreive the text stored in column iCol. If an SQL NULL is stored
** in column iCol, jump immediately to the next iteration of the loop.
@ -1544,19 +1685,9 @@ void sqlite3Fts3Matchinfo(
const char *zArg /* Second arg to matchinfo() function */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc;
int i;
const char *zFormat;
if( zArg ){
for(i=0; zArg[i]; i++){
char *zErr = 0;
if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
sqlite3_result_error(pContext, zErr, -1);
sqlite3_free(zErr);
return;
}
}
zFormat = zArg;
}else{
zFormat = FTS3_MATCHINFO_DEFAULT;
@ -1565,17 +1696,10 @@ void sqlite3Fts3Matchinfo(
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}
/* Retrieve matchinfo() data. */
rc = fts3GetMatchinfo(pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pContext, rc);
}else{
int n = pCsr->nMatchinfo * sizeof(u32);
sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
/* Retrieve matchinfo() data. */
fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
}
}

15
ext/fts3/fts3_tokenizer.c
View File

@ -67,6 +67,7 @@ static void scalarFunc(
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( zName==0 || n!=sizeof(pPtr) ){
@ -79,7 +80,14 @@ static void scalarFunc(
sqlite3_result_error(context, "out of memory", -1);
return;
}
}else{
#else
sqlite3_result_error(context, "fts3tokenize: "
"disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1
);
return;
#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
}else
{
if( zName ){
pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
}
@ -328,6 +336,7 @@ finish:
Tcl_DecrRefCount(pRet);
}
#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
static
int registerTokenizer(
sqlite3 *db,
@ -349,6 +358,8 @@ int registerTokenizer(
return sqlite3_finalize(pStmt);
}
#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
static
int queryTokenizer(
@ -420,11 +431,13 @@ static void intTestFunc(
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
#endif
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}

36
ext/fts3/fts3_write.c
View File

@ -860,10 +860,12 @@ static int fts3PendingTermsAdd(
*/
static int fts3PendingTermsDocid(
Fts3Table *p, /* Full-text table handle */
int bDelete, /* True if this op is a delete */
int iLangid, /* Language id of row being written */
sqlite_int64 iDocid /* Docid of row being written */
){
assert( iLangid>=0 );
assert( bDelete==1 || bDelete==0 );
/* TODO(shess) Explore whether partially flushing the buffer on
** forced-flush would provide better performance. I suspect that if
@ -871,7 +873,8 @@ static int fts3PendingTermsDocid(
** buffer was half empty, that would let the less frequent terms
** generate longer doclists.
*/
if( iDocid<=p->iPrevDocid
if( iDocid<p->iPrevDocid
|| (iDocid==p->iPrevDocid && p->bPrevDelete==0)
|| p->iPrevLangid!=iLangid
|| p->nPendingData>p->nMaxPendingData
){
@ -880,6 +883,7 @@ static int fts3PendingTermsDocid(
}
p->iPrevDocid = iDocid;
p->iPrevLangid = iLangid;
p->bPrevDelete = bDelete;
return SQLITE_OK;
}
@ -1069,7 +1073,8 @@ static void fts3DeleteTerms(
if( SQLITE_ROW==sqlite3_step(pSelect) ){
int i;
int iLangid = langidFromSelect(p, pSelect);
rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
i64 iDocid = sqlite3_column_int64(pSelect, 0);
rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid);
for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
int iCol = i-1;
if( p->abNotindexed[iCol]==0 ){
@ -1317,14 +1322,19 @@ static int fts3SegReaderNext(
if( fts3SegReaderIsPending(pReader) ){
Fts3HashElem *pElem = *(pReader->ppNextElem);
if( pElem==0 ){
pReader->aNode = 0;
}else{
sqlite3_free(pReader->aNode);
pReader->aNode = 0;
if( pElem ){
char *aCopy;
PendingList *pList = (PendingList *)fts3HashData(pElem);
int nCopy = pList->nData+1;
pReader->zTerm = (char *)fts3HashKey(pElem);
pReader->nTerm = fts3HashKeysize(pElem);
pReader->nNode = pReader->nDoclist = pList->nData + 1;
pReader->aNode = pReader->aDoclist = pList->aData;
aCopy = (char*)sqlite3_malloc(nCopy);
if( !aCopy ) return SQLITE_NOMEM;
memcpy(aCopy, pList->aData, nCopy);
pReader->nNode = pReader->nDoclist = nCopy;
pReader->aNode = pReader->aDoclist = aCopy;
pReader->ppNextElem++;
assert( pReader->aNode );
}
@ -1564,12 +1574,14 @@ int sqlite3Fts3MsrOvfl(
** second argument.
*/
void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
if( pReader && !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
if( pReader ){
if( !fts3SegReaderIsPending(pReader) ){
sqlite3_free(pReader->zTerm);
}
if( !fts3SegReaderIsRootOnly(pReader) ){
sqlite3_free(pReader->aNode);
sqlite3_blob_close(pReader->pBlob);
}
sqlite3_blob_close(pReader->pBlob);
}
sqlite3_free(pReader);
}
@ -3512,7 +3524,7 @@ static int fts3DoRebuild(Fts3Table *p){
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int iCol;
int iLangid = langidFromSelect(p, pStmt);
rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0));
memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1));
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
if( p->abNotindexed[iCol]==0 ){
@ -5617,7 +5629,7 @@ int sqlite3Fts3UpdateMethod(
}
}
if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
}
if( rc==SQLITE_OK ){
assert( p->iPrevDocid==*pRowid );

2
ext/fts3/tool/fts3view.c
View File

@ -398,7 +398,7 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){
if( sqlite3_step(pStmt)==SQLITE_ROW
&& (nLeaf = sqlite3_column_int(pStmt, 0))>0
){
int nIdx = sqlite3_column_int(pStmt, 5);
nIdx = sqlite3_column_int(pStmt, 5);
sqlite3_int64 sz;
printf("For level %d:\n", i);
printf(" Number of indexes...................... %9d\n", nIdx);

223
ext/fts3/unicode/mkunicode.tcl
View File

@ -1,77 +1,5 @@
#
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of mappings required to remove all
# diacritical marks from a unicode string. Each mapping is itself a list
# consisting of two elements - the unicode codepoint and the single ASCII
# character that it should be replaced with, or an empty string if the
# codepoint should simply be removed from the input. Examples:
#
# { 224 a } (replace codepoint 224 to "a")
# { 769 "" } (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.
#
proc rd_load_unicodedata_text {zName} {
global tl_lookup_table
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
if { [llength $character_decomposition_mapping]!=2
|| [string is xdigit [lindex $character_decomposition_mapping 0]]==0
} {
continue
}
set iCode [expr "0x$code"]
set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
if {[info exists tl_lookup_table($iCode)]} continue
if { ($iAscii >= 97 && $iAscii <= 122)
|| ($iAscii >= 65 && $iAscii <= 90)
} {
lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
set dia($iDia) 1
}
}
foreach d [array names dia] {
lappend lRet [list $d ""]
}
set lRet [lsort -integer -index 0 $lRet]
close $fd
set lRet
}
source [file join [file dirname [info script]] parseunicode.tcl]
proc print_rd {map} {
global tl_lookup_table
@ -117,7 +45,7 @@ proc print_rd {map} {
puts "** E\"). The resuls of passing a codepoint that corresponds to an"
puts "** uppercase letter are undefined."
puts "*/"
puts "static int remove_diacritic(int c)\{"
puts "static int ${::remove_diacritic}(int c)\{"
puts " unsigned short aDia\[\] = \{"
puts -nonewline " 0, "
set i 1
@ -204,53 +132,6 @@ proc print_isdiacritic {zFunc map} {
#-------------------------------------------------------------------------
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of codepoints (integers). The list
# contains all codepoints in the UnicodeData.txt assigned to any "General
# Category" that is not a "Letter" or "Number".
#
proc an_load_unicodedata_text {zName} {
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
set iCode [expr "0x$code"]
set bAlnum [expr {
[lsearch {L N} [string range $general_category 0 0]] >= 0
|| $general_category=="Co"
}]
if { !$bAlnum } { lappend lRet $iCode }
}
close $fd
set lRet
}
proc an_load_separator_ranges {} {
global unicodedata.txt
set lSep [an_load_unicodedata_text ${unicodedata.txt}]
@ -345,9 +226,9 @@ proc print_isalnum {zFunc lRange} {
an_print_range_array $lRange
an_print_ascii_bitmap $lRange
puts {
if( c<128 ){
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
}else if( c<(1<<22) ){
}else if( (unsigned int)c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
@ -440,29 +321,6 @@ proc print_test_isalnum {zFunc lRange} {
#-------------------------------------------------------------------------
proc tl_load_casefolding_txt {zName} {
global tl_lookup_table
set fd [open $zName]
while { ![eof $fd] } {
set line [gets $fd]
if {[string range $line 0 0] == "#"} continue
if {$line == ""} continue
foreach x {a b c d} {unset -nocomplain $x}
foreach {a b c d} [split $line ";"] {}
set a2 [list]
set c2 [list]
foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
set b [string trim $b]
set d [string trim $d]
if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
}
}
proc tl_create_records {} {
global tl_lookup_table
@ -626,19 +484,20 @@ proc print_fold {zFunc} {
tl_print_table_footer toggle
tl_print_ioff_table $liOff
puts {
puts [subst -nocommands {
int ret = c;
assert( c>=0 );
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
@ -649,19 +508,17 @@ proc print_fold {zFunc} {
iHi = iTest-1;
}
}
assert( iRes<0 || c>=aEntry[iRes].iCode );
if( iRes>=0 ){
const struct TableEntry *p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
if( bRemoveDiacritic ) ret = remove_diacritic(ret);
}
if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
}
}]
foreach entry $lHigh {
tl_print_if_entry $entry
@ -732,8 +589,12 @@ proc print_fileheader {} {
*/
}]
puts ""
puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
if {$::generate_fts5_code} {
# no-op
} else {
puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
}
puts ""
puts "#include <assert.h>"
puts ""
@ -760,22 +621,40 @@ proc print_test_main {} {
# our liking.
#
proc usage {} {
puts -nonewline stderr "Usage: $::argv0 ?-test? "
puts -nonewline stderr "Usage: $::argv0 ?-test? ?-fts5? "
puts stderr "<CaseFolding.txt file> <UnicodeData.txt file>"
exit 1
}
if {[llength $argv]!=2 && [llength $argv]!=3} usage
if {[llength $argv]==3 && [lindex $argv 0]!="-test"} usage
if {[llength $argv]<2} usage
set unicodedata.txt [lindex $argv end]
set casefolding.txt [lindex $argv end-1]
set generate_test_code [expr {[llength $argv]==3}]
set remove_diacritic remove_diacritic
set generate_test_code 0
set generate_fts5_code 0
set function_prefix "sqlite3Fts"
for {set i 0} {$i < [llength $argv]-2} {incr i} {
switch -- [lindex $argv $i] {
-test {
set generate_test_code 1
}
-fts5 {
set function_prefix sqlite3Fts5
set generate_fts5_code 1
set remove_diacritic fts5_remove_diacritic
}
default {
usage
}
}
}
print_fileheader
# Print the isalnum() function to stdout.
#
set lRange [an_load_separator_ranges]
print_isalnum sqlite3FtsUnicodeIsalnum $lRange
print_isalnum ${function_prefix}UnicodeIsalnum $lRange
# Leave a gap between the two generated C functions.
#
@ -790,22 +669,26 @@ set mappings [rd_load_unicodedata_text ${unicodedata.txt}]
print_rd $mappings
puts ""
puts ""
print_isdiacritic sqlite3FtsUnicodeIsdiacritic $mappings
print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings
puts ""
puts ""
# Print the fold() function to stdout.
#
print_fold sqlite3FtsUnicodeFold
print_fold ${function_prefix}UnicodeFold
# Print the test routines and main() function to stdout, if -test
# was specified.
#
if {$::generate_test_code} {
print_test_isalnum sqlite3FtsUnicodeIsalnum $lRange
print_fold_test sqlite3FtsUnicodeFold $mappings
print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
print_fold_test ${function_prefix}UnicodeFold $mappings
print_test_main
}
puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
if {$generate_fts5_code} {
# no-op
} else {
puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
}

146
ext/fts3/unicode/parseunicode.tcl Normal file
View File

@ -0,0 +1,146 @@
#--------------------------------------------------------------------------
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of mappings required to remove all
# diacritical marks from a unicode string. Each mapping is itself a list
# consisting of two elements - the unicode codepoint and the single ASCII
# character that it should be replaced with, or an empty string if the
# codepoint should simply be removed from the input. Examples:
#
# { 224 a } (replace codepoint 224 to "a")
# { 769 "" } (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.
#
proc rd_load_unicodedata_text {zName} {
global tl_lookup_table
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
if { [llength $character_decomposition_mapping]!=2
|| [string is xdigit [lindex $character_decomposition_mapping 0]]==0
} {
continue
}
set iCode [expr "0x$code"]
set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
if {[info exists tl_lookup_table($iCode)]} continue
if { ($iAscii >= 97 && $iAscii <= 122)
|| ($iAscii >= 65 && $iAscii <= 90)
} {
lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
set dia($iDia) 1
}
}
foreach d [array names dia] {
lappend lRet [list $d ""]
}
set lRet [lsort -integer -index 0 $lRet]
close $fd
set lRet
}
#-------------------------------------------------------------------------
# Parameter $zName must be a path to the file UnicodeData.txt. This command
# reads the file and returns a list of codepoints (integers). The list
# contains all codepoints in the UnicodeData.txt assigned to any "General
# Category" that is not a "Letter" or "Number".
#
proc an_load_unicodedata_text {zName} {
set fd [open $zName]
set lField {
code
character_name
general_category
canonical_combining_classes
bidirectional_category
character_decomposition_mapping
decimal_digit_value
digit_value
numeric_value
mirrored
unicode_1_name
iso10646_comment_field
uppercase_mapping
lowercase_mapping
titlecase_mapping
}
set lRet [list]
while { ![eof $fd] } {
set line [gets $fd]
if {$line == ""} continue
set fields [split $line ";"]
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
foreach $lField $fields {}
set iCode [expr "0x$code"]
set bAlnum [expr {
[lsearch {L N} [string range $general_category 0 0]] >= 0
|| $general_category=="Co"
}]
if { !$bAlnum } { lappend lRet $iCode }
}
close $fd
set lRet
}
proc tl_load_casefolding_txt {zName} {
global tl_lookup_table
set fd [open $zName]
while { ![eof $fd] } {
set line [gets $fd]
if {[string range $line 0 0] == "#"} continue
if {$line == ""} continue
foreach x {a b c d} {unset -nocomplain $x}
foreach {a b c d} [split $line ";"] {}
set a2 [list]
set c2 [list]
foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
set b [string trim $b]
set d [string trim $d]
if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
}
}

252
ext/fts5/extract_api_docs.tcl Normal file
View File

@ -0,0 +1,252 @@
#
# 2014 August 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#--------------------------------------------------------------------------
#
# This script extracts the documentation for the API used by fts5 auxiliary
# functions from header file fts5.h. It outputs html text on stdout that
# is included in the documentation on the web.
#
set ::fts5_docs_output ""
if {[info commands hd_putsnl]==""} {
if {[llength $argv]>0} { set ::extract_api_docs_mode [lindex $argv 0] }
proc output {text} {
puts $text
}
} else {
proc output {text} {
append ::fts5_docs_output "$text\n"
}
}
if {[info exists ::extract_api_docs_mode]==0} {set ::extract_api_docs_mode api}
set input_file [file join [file dir [info script]] fts5.h]
set fd [open $input_file]
set data [read $fd]
close $fd
# Argument $data is the entire text of the fts5.h file. This function
# extracts the definition of the Fts5ExtensionApi structure from it and
# returns a key/value list of structure member names and definitions. i.e.
#
# iVersion {int iVersion} xUserData {void *(*xUserData)(Fts5Context*)} ...
#
proc get_struct_members {data} {
# Extract the structure definition from the fts5.h file.
regexp "struct Fts5ExtensionApi {(.*?)};" $data -> defn
# Remove all comments from the structure definition
regsub -all {/[*].*?[*]/} $defn {} defn2
set res [list]
foreach member [split $defn2 {;}] {
set member [string trim $member]
if {$member!=""} {
catch { set name [lindex $member end] }
regexp {.*?[(][*]([^)]*)[)]} $member -> name
lappend res $name $member
}
}
set res
}
proc get_struct_docs {data names} {
# Extract the structure definition from the fts5.h file.
regexp {EXTENSION API FUNCTIONS(.*?)[*]/} $data -> docs
set current_doc ""
set current_header ""
foreach line [split $docs "\n"] {
regsub {[*]*} $line {} line
if {[regexp {^ } $line]} {
append current_doc "$line\n"
} elseif {[string trim $line]==""} {
if {$current_header!=""} { append current_doc "\n" }
} else {
if {$current_doc != ""} {
lappend res $current_header $current_doc
set current_doc ""
}
set subject n/a
regexp {^ *([[:alpha:]]*)} $line -> subject
if {[lsearch $names $subject]>=0} {
set current_header $subject
} else {
set current_header [string trim $line]
}
}
}
if {$current_doc != ""} {
lappend res $current_header $current_doc
}
set res
}
proc get_tokenizer_docs {data} {
regexp {(xCreate:.*?)[*]/} $data -> docs
set res "<dl>\n"
foreach line [split [string trim $docs] "\n"] {
regexp {[*][*](.*)} $line -> line
if {[regexp {^ ?x.*:} $line]} {
append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
continue
}
if {[regexp {SYNONYM SUPPORT} $line]} {
set line "</dl><h3>Synonym Support</h3>"
}
if {[string trim $line] == ""} {
append res "<p>\n"
} else {
append res "$line\n"
}
}
set res
}
proc get_api_docs {data} {
# Initialize global array M as a map from Fts5StructureApi member name
# to member definition. i.e.
#
# iVersion -> {int iVersion}
# xUserData -> {void *(*xUserData)(Fts5Context*)}
# ...
#
array set M [get_struct_members $data]
# Initialize global list D as a map from section name to documentation
# text. Most (all?) section names are structure member names.
#
set D [get_struct_docs $data [array names M]]
output "<dl>"
foreach {sub docs} $D {
if {[info exists M($sub)]} {
set hdr $M($sub)
set link " id=$sub"
} else {
set link ""
}
#output "<hr color=#eeeee style=\"margin:1em 8.4ex 0 8.4ex;\"$link>"
#set style "padding-left:6ex;font-size:1.4em;display:block"
#output "<h style=\"$style\"><pre>$hdr</pre></h>"
regsub -line {^ *[)]} $hdr ")" hdr
output "<dt style=\"white-space:pre;font-family:monospace;font-size:120%\""
output "$link>"
output "<b>$hdr</b></dt><dd>"
set mode ""
set margin " style=margin-top:0.1em"
foreach line [split [string trim $docs] "\n"] {
if {[string trim $line]==""} {
if {$mode != ""} {output "</$mode>"}
set mode ""
} elseif {$mode == ""} {
if {[regexp {^ } $line]} {
set mode codeblock
} else {
set mode p
}
output "<$mode$margin>"
set margin ""
}
output $line
}
if {$mode != ""} {output "</$mode>"}
output "</dd>"
}
output "</dl>"
}
proc get_fts5_struct {data start end} {
set res ""
set bOut 0
foreach line [split $data "\n"] {
if {$bOut==0} {
if {[regexp $start $line]} {
set bOut 1
}
}
if {$bOut} {
append res "$line\n"
}
if {$bOut} {
if {[regexp $end $line]} {
set bOut 0
}
}
}
set map [list /* <i>/* */ */</i>]
string map $map $res
}
proc main {data} {
switch $::extract_api_docs_mode {
fts5_api {
output [get_fts5_struct $data "typedef struct fts5_api" "^\};"]
}
fts5_tokenizer {
output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"]
output [get_fts5_struct $data \
"Flags that may be passed as the third argument to xTokenize()" \
"#define FTS5_TOKEN_COLOCATED"
]
}
fts5_extension {
output [get_fts5_struct $data "typedef.*Fts5ExtensionApi" "^.;"]
}
Fts5ExtensionApi {
set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
set map [list]
foreach {k v} [get_struct_members $data] {
if {[string match x* $k]==0} continue
lappend map $k "<a href=#$k>$k</a>"
}
output [string map $map $struct]
}
api {
get_api_docs $data
}
tokenizer_api {
output [get_tokenizer_docs $data]
}
default {
}
}
}
main $data
set ::fts5_docs_output

578
ext/fts5/fts5.h Normal file
View File

@ -0,0 +1,578 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Interfaces to extend FTS5. Using the interfaces defined in this file,
** FTS5 may be extended with:
**
** * custom tokenizers, and
** * custom auxiliary functions.
*/
#ifndef _FTS5_H
#define _FTS5_H
#include "sqlite3.h"
#ifdef __cplusplus
extern "C" {
#endif
/*************************************************************************
** CUSTOM AUXILIARY FUNCTIONS
**
** Virtual table implementations may overload SQL functions by implementing
** the sqlite3_module.xFindFunction() method.
*/
typedef struct Fts5ExtensionApi Fts5ExtensionApi;
typedef struct Fts5Context Fts5Context;
typedef struct Fts5PhraseIter Fts5PhraseIter;
typedef void (*fts5_extension_function)(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
);
struct Fts5PhraseIter {
const unsigned char *a;
const unsigned char *b;
};
/*
** EXTENSION API FUNCTIONS
**
** xUserData(pFts):
** Return a copy of the context pointer the extension function was
** registered with.
**
** xColumnTotalSize(pFts, iCol, pnToken):
** If parameter iCol is less than zero, set output variable *pnToken
** to the total number of tokens in the FTS5 table. Or, if iCol is
** non-negative but less than the number of columns in the table, return
** the total number of tokens in column iCol, considering all rows in
** the FTS5 table.
**
** If parameter iCol is greater than or equal to the number of columns
** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
** an OOM condition or IO error), an appropriate SQLite error code is
** returned.
**
** xColumnCount(pFts):
** Return the number of columns in the table.
**
** xColumnSize(pFts, iCol, pnToken):
** If parameter iCol is less than zero, set output variable *pnToken
** to the total number of tokens in the current row. Or, if iCol is
** non-negative but less than the number of columns in the table, set
** *pnToken to the number of tokens in column iCol of the current row.
**
** If parameter iCol is greater than or equal to the number of columns
** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
** an OOM condition or IO error), an appropriate SQLite error code is
** returned.
**
** This function may be quite inefficient if used with an FTS5 table
** created with the "columnsize=0" option.
**
** xColumnText:
** This function attempts to retrieve the text of column iCol of the
** current document. If successful, (*pz) is set to point to a buffer
** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
** if an error occurs, an SQLite error code is returned and the final values
** of (*pz) and (*pn) are undefined.
**
** xPhraseCount:
** Returns the number of phrases in the current query expression.
**
** xPhraseSize:
** Returns the number of tokens in phrase iPhrase of the query. Phrases
** are numbered starting from zero.
**
** xInstCount:
** Set *pnInst to the total number of occurrences of all phrases within
** the query within the current row. Return SQLITE_OK if successful, or
** an error code (i.e. SQLITE_NOMEM) if an error occurs.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option. If the FTS5 table is created
** with either "detail=none" or "detail=column" and "content=" option
** (i.e. if it is a contentless table), then this API always returns 0.
**
** xInst:
** Query for the details of phrase match iIdx within the current row.
** Phrase matches are numbered starting from zero, so the iIdx argument
** should be greater than or equal to zero and smaller than the value
** output by xInstCount().
**
** Usually, output parameter *piPhrase is set to the phrase number, *piCol
** to the column in which it occurs and *piOff the token offset of the
** first token of the phrase. The exception is if the table was created
** with the offsets=0 option specified. In this case *piOff is always
** set to -1.
**
** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM)
** if an error occurs.
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option.
**
** xRowid:
** Returns the rowid of the current row.
**
** xTokenize:
** Tokenize text using the tokenizer belonging to the FTS5 table.
**
** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
** This API function is used to query the FTS table for phrase iPhrase
** of the current query. Specifically, a query equivalent to:
**
** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
**
** with $p set to a phrase equivalent to the phrase iPhrase of the
** current query is executed. For each row visited, the callback function
** passed as the fourth argument is invoked. The context and API objects
** passed to the callback function may be used to access the properties of
** each matched row. Invoking Api.xUserData() returns a copy of the pointer
** passed as the third argument to pUserData.
**
** If the callback function returns any value other than SQLITE_OK, the
** query is abandoned and the xQueryPhrase function returns immediately.
** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
** Otherwise, the error code is propagated upwards.
**
** If the query runs to completion without incident, SQLITE_OK is returned.
** Or, if some error occurs before the query completes or is aborted by
** the callback, an SQLite error code is returned.
**
**
** xSetAuxdata(pFts5, pAux, xDelete)
**
** Save the pointer passed as the second argument as the extension functions
** "auxiliary data". The pointer may then be retrieved by the current or any
** future invocation of the same fts5 extension function made as part of
** of the same MATCH query using the xGetAuxdata() API.
**
** Each extension function is allocated a single auxiliary data slot for
** each FTS query (MATCH expression). If the extension function is invoked
** more than once for a single FTS query, then all invocations share a
** single auxiliary data context.
**
** If there is already an auxiliary data pointer when this function is
** invoked, then it is replaced by the new pointer. If an xDelete callback
** was specified along with the original pointer, it is invoked at this
** point.
**
** The xDelete callback, if one is specified, is also invoked on the
** auxiliary data pointer after the FTS5 query has finished.
**
** If an error (e.g. an OOM condition) occurs within this function, an
** the auxiliary data is set to NULL and an error code returned. If the
** xDelete parameter was not NULL, it is invoked on the auxiliary data
** pointer before returning.
**
**
** xGetAuxdata(pFts5, bClear)
**
** Returns the current auxiliary data pointer for the fts5 extension
** function. See the xSetAuxdata() method for details.
**
** If the bClear argument is non-zero, then the auxiliary data is cleared
** (set to NULL) before this function returns. In this case the xDelete,
** if any, is not invoked.
**
**
** xRowCount(pFts5, pnRow)
**
** This function is used to retrieve the total number of rows in the table.
** In other words, the same value that would be returned by:
**
** SELECT count(*) FROM ftstable;
**
** xPhraseFirst()
** This function is used, along with type Fts5PhraseIter and the xPhraseNext
** method, to iterate through all instances of a single query phrase within
** the current row. This is the same information as is accessible via the
** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
** to use, this API may be faster under some circumstances. To iterate
** through instances of phrase iPhrase, use the following code:
**
** Fts5PhraseIter iter;
** int iCol, iOff;
** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
** iCol>=0;
** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
** ){
** // An instance of phrase iPhrase at offset iOff of column iCol
** }
**
** The Fts5PhraseIter structure is defined above. Applications should not
** modify this structure directly - it should only be used as shown above
** with the xPhraseFirst() and xPhraseNext() API methods (and by
** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" or "detail=column" option. If the FTS5 table is created
** with either "detail=none" or "detail=column" and "content=" option
** (i.e. if it is a contentless table), then this API always iterates
** through an empty set (all calls to xPhraseFirst() set iCol to -1).
**
** xPhraseNext()
** See xPhraseFirst above.
**
** xPhraseFirstColumn()
** This function and xPhraseNextColumn() are similar to the xPhraseFirst()
** and xPhraseNext() APIs described above. The difference is that instead
** of iterating through all instances of a phrase in the current row, these
** APIs are used to iterate through the set of columns in the current row
** that contain one or more instances of a specified phrase. For example:
**
** Fts5PhraseIter iter;
** int iCol;
** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
** iCol>=0;
** pApi->xPhraseNextColumn(pFts, &iter, &iCol)
** ){
** // Column iCol contains at least one instance of phrase iPhrase
** }
**
** This API can be quite slow if used with an FTS5 table created with the
** "detail=none" option. If the FTS5 table is created with either
** "detail=none" "content=" option (i.e. if it is a contentless table),
** then this API always iterates through an empty set (all calls to
** xPhraseFirstColumn() set iCol to -1).
**
** The information accessed using this API and its companion
** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
** (or xInst/xInstCount). The chief advantage of this API is that it is
** significantly more efficient than those alternatives when used with
** "detail=column" tables.
**
** xPhraseNextColumn()
** See xPhraseFirstColumn above.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 3 */
void *(*xUserData)(Fts5Context*);
int (*xColumnCount)(Fts5Context*);
int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
int (*xTokenize)(Fts5Context*,
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
int (*xPhraseCount)(Fts5Context*);
int (*xPhraseSize)(Fts5Context*, int iPhrase);
int (*xInstCount)(Fts5Context*, int *pnInst);
int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
sqlite3_int64 (*xRowid)(Fts5Context*);
int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
);
int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
void *(*xGetAuxdata)(Fts5Context*, int bClear);
int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
};
/*
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/
/*************************************************************************
** CUSTOM TOKENIZERS
**
** Applications may also register custom tokenizer types. A tokenizer
** is registered by providing fts5 with a populated instance of the
** following structure. All structure methods must be defined, setting
** any member of the fts5_tokenizer struct to NULL leads to undefined
** behaviour. The structure methods are expected to function as follows:
**
** xCreate:
** This function is used to allocate and inititalize a tokenizer instance.
** A tokenizer instance is required to actually tokenize text.
**
** The first argument passed to this function is a copy of the (void*)
** pointer provided by the application when the fts5_tokenizer object
** was registered with FTS5 (the third argument to xCreateTokenizer()).
** The second and third arguments are an array of nul-terminated strings
** containing the tokenizer arguments, if any, specified following the
** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
** to create the FTS5 table.
**
** The final argument is an output variable. If successful, (*ppOut)
** should be set to point to the new tokenizer handle and SQLITE_OK
** returned. If an error occurs, some value other than SQLITE_OK should
** be returned. In this case, fts5 assumes that the final value of *ppOut
** is undefined.
**
** xDelete:
** This function is invoked to delete a tokenizer handle previously
** allocated using xCreate(). Fts5 guarantees that this function will
** be invoked exactly once for each successful call to xCreate().
**
** xTokenize:
** This function is expected to tokenize the nText byte string indicated
** by argument pText. pText may or may not be nul-terminated. The first
** argument passed to this function is a pointer to an Fts5Tokenizer object
** returned by an earlier call to xCreate().
**
** The second argument indicates the reason that FTS5 is requesting
** tokenization of the supplied text. This is always one of the following
** four values:
**
** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
** or removed from the FTS table. The tokenizer is being invoked to
** determine the set of tokens to add to (or delete from) the
** FTS index.
**
** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
** against the FTS index. The tokenizer is being called to tokenize
** a bareword or quoted string specified as part of the query.
**
** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
** followed by a "*" character, indicating that the last token
** returned by the tokenizer will be treated as a token prefix.
**
** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
** satisfy an fts5_api.xTokenize() request made by an auxiliary
** function. Or an fts5_api.xColumnSize() request made by the same
** on a columnsize=0 database.
** </ul>
**
** For each token in the input string, the supplied callback xToken() must
** be invoked. The first argument to it should be a copy of the pointer
** passed as the second argument to xTokenize(). The third and fourth
** arguments are a pointer to a buffer containing the token text, and the
** size of the token in bytes. The 4th and 5th arguments are the byte offsets
** of the first byte of and first byte immediately following the text from
** which the token is derived within the input.
**
** The second argument passed to the xToken() callback ("tflags") should
** normally be set to 0. The exception is if the tokenizer supports
** synonyms. In this case see the discussion below for details.
**
** FTS5 assumes the xToken() callback is invoked for each token in the
** order that they occur within the input text.
**
** If an xToken() callback returns any value other than SQLITE_OK, then
** the tokenization should be abandoned and the xTokenize() method should
** immediately return a copy of the xToken() return value. Or, if the
** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
** if an error occurs with the xTokenize() implementation itself, it
** may abandon the tokenization and return any error code other than
** SQLITE_OK or SQLITE_DONE.
**
** SYNONYM SUPPORT
**
** Custom tokenizers may also support synonyms. Consider a case in which a
** user wishes to query for a phrase such as "first place". Using the
** built-in tokenizers, the FTS5 query 'first + place' will match instances
** of "first place" within the document set, but not alternative forms
** such as "1st place". In some applications, it would be better to match
** all instances of "first place" or "1st place" regardless of which form
** the user specified in the MATCH query text.
**
** There are several ways to approach this in FTS5:
**
** <ol><li> By mapping all synonyms to a single token. In this case, the
** In the above example, this means that the tokenizer returns the
** same token for inputs "first" and "1st". Say that token is in
** fact "first", so that when the user inserts the document "I won
** 1st place" entries are added to the index for tokens "i", "won",
** "first" and "place". If the user then queries for '1st + place',
** the tokenizer substitutes "first" for "1st" and the query works
** as expected.
**
** <li> By adding multiple synonyms for a single term to the FTS index.
** In this case, when tokenizing query text, the tokenizer may
** provide multiple synonyms for a single term within the document.
** FTS5 then queries the index for each synonym individually. For
** example, faced with the query:
**
** <codeblock>
** ... MATCH 'first place'</codeblock>
**
** the tokenizer offers both "1st" and "first" as synonyms for the
** first token in the MATCH query and FTS5 effectively runs a query
** similar to:
**
** <codeblock>
** ... MATCH '(first OR 1st) place'</codeblock>
**
** except that, for the purposes of auxiliary functions, the query
** still appears to contain just two phrases - "(first OR 1st)"
** being treated as a single phrase.
**
** <li> By adding multiple synonyms for a single term to the FTS index.
** Using this method, when tokenizing document text, the tokenizer
** provides multiple synonyms for each token. So that when a
** document such as "I won first place" is tokenized, entries are
** added to the FTS index for "i", "won", "first", "1st" and
** "place".
**
** This way, even if the tokenizer does not provide synonyms
** when tokenizing query text (it should not - to do would be
** inefficient), it doesn't matter if the user queries for
** 'first + place' or '1st + place', as there are entires in the
** FTS index corresponding to both forms of the first token.
** </ol>
**
** Whether it is parsing document or query text, any call to xToken that
** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
** is considered to supply a synonym for the previous token. For example,
** when parsing the document "I won first place", a tokenizer that supports
** synonyms would call xToken() 5 times, as follows:
**
** <codeblock>
** xToken(pCtx, 0, "i", 1, 0, 1);
** xToken(pCtx, 0, "won", 3, 2, 5);
** xToken(pCtx, 0, "first", 5, 6, 11);
** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
** xToken(pCtx, 0, "place", 5, 12, 17);
**</codeblock>
**
** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
** xToken() is called. Multiple synonyms may be specified for a single token
** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
** There is no limit to the number of synonyms that may be provided for a
** single token.
**
** In many cases, method (1) above is the best approach. It does not add
** extra data to the FTS index or require FTS5 to query for multiple terms,
** so it is efficient in terms of disk space and query speed. However, it
** does not support prefix queries very well. If, as suggested above, the
** token "first" is subsituted for "1st" by the tokenizer, then the query:
**
** <codeblock>
** ... MATCH '1s*'</codeblock>
**
** will not match documents that contain the token "1st" (as the tokenizer
** will probably not map "1s" to any prefix of "first").
**
** For full prefix support, method (3) may be preferred. In this case,
** because the index contains entries for both "first" and "1st", prefix
** queries such as 'fi*' or '1s*' will match correctly. However, because
** extra entries are added to the FTS index, this method uses more space
** within the database.
**
** Method (2) offers a midpoint between (1) and (3). Using this method,
** a query such as '1s*' will match documents that contain the literal
** token "1st", but not "first" (assuming the tokenizer is not able to
** provide synonyms for prefixes). However, a non-prefix query like '1st'
** will match against "1st" and "first". This method does not require
** extra disk space, as no extra entries are added to the FTS index.
** On the other hand, it may require more CPU cycles to run MATCH queries,
** as separate queries of the FTS index are required for each synonym.
**
** When using methods (2) or (3), it is important that the tokenizer only
** provide synonyms when tokenizing document text (method (2)) or query
** text (method (3)), not both. Doing so will not cause any errors, but is
** inefficient.
*/
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct fts5_tokenizer fts5_tokenizer;
struct fts5_tokenizer {
int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
void (*xDelete)(Fts5Tokenizer*);
int (*xTokenize)(Fts5Tokenizer*,
void *pCtx,
int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
int (*xToken)(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
)
);
};
/* Flags that may be passed as the third argument to xTokenize() */
#define FTS5_TOKENIZE_QUERY 0x0001
#define FTS5_TOKENIZE_PREFIX 0x0002
#define FTS5_TOKENIZE_DOCUMENT 0x0004
#define FTS5_TOKENIZE_AUX 0x0008
/* Flags that may be passed by the tokenizer implementation back to FTS5
** as the third argument to the supplied xToken callback. */
#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
/*
** END OF CUSTOM TOKENIZERS
*************************************************************************/
/*************************************************************************
** FTS5 EXTENSION REGISTRATION API
*/
typedef struct fts5_api fts5_api;
struct fts5_api {
int iVersion; /* Currently always set to 2 */
/* Create a new tokenizer */
int (*xCreateTokenizer)(
fts5_api *pApi,
const char *zName,
void *pContext,
fts5_tokenizer *pTokenizer,
void (*xDestroy)(void*)
);
/* Find an existing tokenizer */
int (*xFindTokenizer)(
fts5_api *pApi,
const char *zName,
void **ppContext,
fts5_tokenizer *pTokenizer
);
/* Create a new auxiliary function */
int (*xCreateFunction)(
fts5_api *pApi,
const char *zName,
void *pContext,
fts5_extension_function xFunction,
void (*xDestroy)(void*)
);
};
/*
** END OF REGISTRATION API
*************************************************************************/
#ifdef __cplusplus
} /* end of the 'extern "C"' block */
#endif
#endif /* _FTS5_H */

776
ext/fts5/fts5Int.h Normal file
View File

@ -0,0 +1,776 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#ifndef _FTS5INT_H
#define _FTS5INT_H
#include "fts5.h"
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include <string.h>
#include <assert.h>
#ifndef SQLITE_AMALGAMATION
typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned short u16;
typedef short i16;
typedef sqlite3_int64 i64;
typedef sqlite3_uint64 u64;
#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
#define testcase(x)
#define ALWAYS(x) 1
#define NEVER(x) 0
#define MIN(x,y) (((x) < (y)) ? (x) : (y))
#define MAX(x,y) (((x) > (y)) ? (x) : (y))
/*
** Constants for the largest and smallest possible 64-bit signed integers.
*/
# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
#endif
/*
** Maximum number of prefix indexes on single FTS5 table. This must be
** less than 32. If it is set to anything large than that, an #error
** directive in fts5_index.c will cause the build to fail.
*/
#define FTS5_MAX_PREFIX_INDEXES 31
#define FTS5_DEFAULT_NEARDIST 10
#define FTS5_DEFAULT_RANK "bm25"
/* Name of rank and rowid columns */
#define FTS5_RANK_NAME "rank"
#define FTS5_ROWID_NAME "rowid"
#ifdef SQLITE_DEBUG
# define FTS5_CORRUPT sqlite3Fts5Corrupt()
int sqlite3Fts5Corrupt(void);
#else
# define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
#endif
/*
** The assert_nc() macro is similar to the assert() macro, except that it
** is used for assert() conditions that are true only if it can be
** guranteed that the database is not corrupt.
*/
#ifdef SQLITE_DEBUG
extern int sqlite3_fts5_may_be_corrupt;
# define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
#else
# define assert_nc(x) assert(x)
#endif
/* Mark a function parameter as unused, to suppress nuisance compiler
** warnings. */
#ifndef UNUSED_PARAM
# define UNUSED_PARAM(X) (void)(X)
#endif
#ifndef UNUSED_PARAM2
# define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y)
#endif
typedef struct Fts5Global Fts5Global;
typedef struct Fts5Colset Fts5Colset;
/* If a NEAR() clump or phrase may only match a specific set of columns,
** then an object of the following type is used to record the set of columns.
** Each entry in the aiCol[] array is a column that may be matched.
**
** This object is used by fts5_expr.c and fts5_index.c.
*/
struct Fts5Colset {
int nCol;
int aiCol[1];
};
/**************************************************************************
** Interface to code in fts5_config.c. fts5_config.c contains contains code
** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
*/
typedef struct Fts5Config Fts5Config;
/*
** An instance of the following structure encodes all information that can
** be gleaned from the CREATE VIRTUAL TABLE statement.
**
** And all information loaded from the %_config table.
**
** nAutomerge:
** The minimum number of segments that an auto-merge operation should
** attempt to merge together. A value of 1 sets the object to use the
** compile time default. Zero disables auto-merge altogether.
**
** zContent:
**
** zContentRowid:
** The value of the content_rowid= option, if one was specified. Or
** the string "rowid" otherwise. This text is not quoted - if it is
** used as part of an SQL statement it needs to be quoted appropriately.
**
** zContentExprlist:
**
** pzErrmsg:
** This exists in order to allow the fts5_index.c module to return a
** decent error message if it encounters a file-format version it does
** not understand.
**
** bColumnsize:
** True if the %_docsize table is created.
**
** bPrefixIndex:
** This is only used for debugging. If set to false, any prefix indexes
** are ignored. This value is configured using:
**
** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
**
*/
struct Fts5Config {
sqlite3 *db; /* Database handle */
char *zDb; /* Database holding FTS index (e.g. "main") */
char *zName; /* Name of FTS index */
int nCol; /* Number of columns */
char **azCol; /* Column names */
u8 *abUnindexed; /* True for unindexed columns */
int nPrefix; /* Number of prefix indexes */
int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
int eContent; /* An FTS5_CONTENT value */
char *zContent; /* content table */
char *zContentRowid; /* "content_rowid=" option value */
int bColumnsize; /* "columnsize=" option value (dflt==1) */
int eDetail; /* FTS5_DETAIL_XXX value */
char *zContentExprlist;
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
/* Values loaded from the %_config table */
int iCookie; /* Incremented when %_config is modified */
int pgsz; /* Approximate page size used in %_data */
int nAutomerge; /* 'automerge' setting */
int nCrisisMerge; /* Maximum allowed segments per level */
int nHashSize; /* Bytes of memory for in-memory hash */
char *zRank; /* Name of rank function */
char *zRankArgs; /* Arguments to rank function */
/* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
char **pzErrmsg;
#ifdef SQLITE_DEBUG
int bPrefixIndex; /* True to use prefix-indexes */
#endif
};
/* Current expected value of %_config table 'version' field */
#define FTS5_CURRENT_VERSION 4
#define FTS5_CONTENT_NORMAL 0
#define FTS5_CONTENT_NONE 1
#define FTS5_CONTENT_EXTERNAL 2
#define FTS5_DETAIL_FULL 0
#define FTS5_DETAIL_NONE 1
#define FTS5_DETAIL_COLUMNS 2
int sqlite3Fts5ConfigParse(
Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
);
void sqlite3Fts5ConfigFree(Fts5Config*);
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
void sqlite3Fts5Dequote(char *z);
/* Load the contents of the %_config table */
int sqlite3Fts5ConfigLoad(Fts5Config*, int);
/* Set the value of a single config attribute */
int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
/*
** End of interface to code in fts5_config.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_buffer.c.
*/
/*
** Buffer object for the incremental building of string data.
*/
typedef struct Fts5Buffer Fts5Buffer;
struct Fts5Buffer {
u8 *p;
int n;
int nSpace;
};
int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
void sqlite3Fts5BufferFree(Fts5Buffer*);
void sqlite3Fts5BufferZero(Fts5Buffer*);
void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
#define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
#define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
#define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
#define fts5BufferGrow(pRc,pBuf,nn) ( \
(u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
)
/* Write and decode big-endian 32-bit integer values */
void sqlite3Fts5Put32(u8*, int);
int sqlite3Fts5Get32(const u8*);
#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)
typedef struct Fts5PoslistReader Fts5PoslistReader;
struct Fts5PoslistReader {
/* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
const u8 *a; /* Position list to iterate through */
int n; /* Size of buffer at a[] in bytes */
int i; /* Current offset in a[] */
u8 bFlag; /* For client use (any custom purpose) */
/* Output variables */
u8 bEof; /* Set to true at EOF */
i64 iPos; /* (iCol<<32) + iPos */
};
int sqlite3Fts5PoslistReaderInit(
const u8 *a, int n, /* Poslist buffer to iterate through */
Fts5PoslistReader *pIter /* Iterator object to initialize */
);
int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
typedef struct Fts5PoslistWriter Fts5PoslistWriter;
struct Fts5PoslistWriter {
i64 iPrev;
};
int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
int sqlite3Fts5PoslistNext64(
const u8 *a, int n, /* Buffer containing poslist */
int *pi, /* IN/OUT: Offset within a[] */
i64 *piOff /* IN/OUT: Current offset */
);
/* Malloc utility */
void *sqlite3Fts5MallocZero(int *pRc, int nByte);
char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
/* Character set tests (like isspace(), isalpha() etc.) */
int sqlite3Fts5IsBareword(char t);
/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
typedef struct Fts5Termset Fts5Termset;
int sqlite3Fts5TermsetNew(Fts5Termset**);
int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
void sqlite3Fts5TermsetFree(Fts5Termset*);
/*
** End of interface to code in fts5_buffer.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_index.c. fts5_index.c contains contains code
** to access the data stored in the %_data table.
*/
typedef struct Fts5Index Fts5Index;
typedef struct Fts5IndexIter Fts5IndexIter;
struct Fts5IndexIter {
i64 iRowid;
const u8 *pData;
int nData;
u8 bEof;
};
#define sqlite3Fts5IterEof(x) ((x)->bEof)
/*
** Values used as part of the flags argument passed to IndexQuery().
*/
#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */
#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */
#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */
/* The following are used internally by the fts5_index.c module. They are
** defined here only to make it easier to avoid clashes with the flags
** above. */
#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
#define FTS5INDEX_QUERY_NOOUTPUT 0x0020
/*
** Create/destroy an Fts5Index object.
*/
int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
int sqlite3Fts5IndexClose(Fts5Index *p);
/*
** Return a simple checksum value based on the arguments.
*/
u64 sqlite3Fts5IndexEntryCksum(
i64 iRowid,
int iCol,
int iPos,
int iIdx,
const char *pTerm,
int nTerm
);
/*
** Argument p points to a buffer containing utf-8 text that is n bytes in
** size. Return the number of bytes in the nChar character prefix of the
** buffer, or 0 if there are less than nChar characters in total.
*/
int sqlite3Fts5IndexCharlenToBytelen(
const char *p,
int nByte,
int nChar
);
/*
** Open a new iterator to iterate though all rowids that match the
** specified token or token prefix.
*/
int sqlite3Fts5IndexQuery(
Fts5Index *p, /* FTS index to query */
const char *pToken, int nToken, /* Token (or prefix) to query for */
int flags, /* Mask of FTS5INDEX_QUERY_X flags */
Fts5Colset *pColset, /* Match these columns only */
Fts5IndexIter **ppIter /* OUT: New iterator object */
);
/*
** The various operations on open token or token prefix iterators opened
** using sqlite3Fts5IndexQuery().
*/
int sqlite3Fts5IterNext(Fts5IndexIter*);
int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
/*
** Close an iterator opened by sqlite3Fts5IndexQuery().
*/
void sqlite3Fts5IterClose(Fts5IndexIter*);
/*
** This interface is used by the fts5vocab module.
*/
const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
int sqlite3Fts5IterNextScan(Fts5IndexIter*);
/*
** Insert or remove data to or from the index. Each time a document is
** added to or removed from the index, this function is called one or more
** times.
**
** For an insert, it must be called once for each token in the new document.
** If the operation is a delete, it must be called (at least) once for each
** unique token in the document with an iCol value less than zero. The iPos
** argument is ignored for a delete.
*/
int sqlite3Fts5IndexWrite(
Fts5Index *p, /* Index to write to */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
const char *pToken, int nToken /* Token to add or remove to or from index */
);
/*
** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
** document iDocid.
*/
int sqlite3Fts5IndexBeginWrite(
Fts5Index *p, /* Index to write to */
int bDelete, /* True if current operation is a delete */
i64 iDocid /* Docid to add or remove data from */
);
/*
** Flush any data stored in the in-memory hash tables to the database.
** If the bCommit flag is true, also close any open blob handles.
*/
int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit);
/*
** Discard any data stored in the in-memory hash tables. Do not write it
** to the database. Additionally, assume that the contents of the %_data
** table may have changed on disk. So any in-memory caches of %_data
** records must be invalidated.
*/
int sqlite3Fts5IndexRollback(Fts5Index *p);
/*
** Get or set the "averages" values.
*/
int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
/*
** Functions called by the storage module as part of integrity-check.
*/
int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
/*
** Called during virtual module initialization to register UDF
** fts5_decode() with SQLite
*/
int sqlite3Fts5IndexInit(sqlite3*);
int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
/*
** Return the total number of entries read from the %_data table by
** this connection since it was created.
*/
int sqlite3Fts5IndexReads(Fts5Index *p);
int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_varint.c.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
int sqlite3Fts5GetVarintLen(u32 iVal);
u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
#define fts5GetVarint sqlite3Fts5GetVarint
#define fts5FastGetVarint32(a, iOff, nVal) { \
nVal = (a)[iOff++]; \
if( nVal & 0x80 ){ \
iOff--; \
iOff += fts5GetVarint32(&(a)[iOff], nVal); \
} \
}
/*
** End of interface to code in fts5_varint.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5.c.
*/
int sqlite3Fts5GetTokenizer(
Fts5Global*,
const char **azArg,
int nArg,
Fts5Tokenizer**,
fts5_tokenizer**,
char **pzErr
);
Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **);
/*
** End of interface to code in fts5.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_hash.c.
*/
typedef struct Fts5Hash Fts5Hash;
/*
** Create a hash table, free a hash table.
*/
int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
void sqlite3Fts5HashFree(Fts5Hash*);
int sqlite3Fts5HashWrite(
Fts5Hash*,
i64 iRowid, /* Rowid for this entry */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
char bByte,
const char *pToken, int nToken /* Token to add or remove to or from index */
);
/*
** Empty (but do not delete) a hash table.
*/
void sqlite3Fts5HashClear(Fts5Hash*);
int sqlite3Fts5HashQuery(
Fts5Hash*, /* Hash table to query */
const char *pTerm, int nTerm, /* Query term */
const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
int *pnDoclist /* OUT: Size of doclist in bytes */
);
int sqlite3Fts5HashScanInit(
Fts5Hash*, /* Hash table to query */
const char *pTerm, int nTerm /* Query prefix */
);
void sqlite3Fts5HashScanNext(Fts5Hash*);
int sqlite3Fts5HashScanEof(Fts5Hash*);
void sqlite3Fts5HashScanEntry(Fts5Hash *,
const char **pzTerm, /* OUT: term (nul-terminated) */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
);
/*
** End of interface to code in fts5_hash.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_storage.c. fts5_storage.c contains contains
** code to access the data stored in the %_content and %_docsize tables.
*/
#define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
#define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
#define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */
typedef struct Fts5Storage Fts5Storage;
int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p);
int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
int sqlite3Fts5DropAll(Fts5Config*);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
int sqlite3Fts5StorageIntegrity(Fts5Storage *p);
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);
int sqlite3Fts5StorageConfigValue(
Fts5Storage *p, const char*, sqlite3_value*, int
);
int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
/*
** End of interface to code in fts5_storage.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_expr.c.
*/
typedef struct Fts5Expr Fts5Expr;
typedef struct Fts5ExprNode Fts5ExprNode;
typedef struct Fts5Parse Fts5Parse;
typedef struct Fts5Token Fts5Token;
typedef struct Fts5ExprPhrase Fts5ExprPhrase;
typedef struct Fts5ExprNearset Fts5ExprNearset;
struct Fts5Token {
const char *p; /* Token text (not NULL terminated) */
int n; /* Size of buffer p in bytes */
};
/* Parse a MATCH expression. */
int sqlite3Fts5ExprNew(
Fts5Config *pConfig,
const char *zExpr,
Fts5Expr **ppNew,
char **pzErr
);
/*
** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
** rc = sqlite3Fts5ExprNext(pExpr)
** ){
** // The document with rowid iRowid matches the expression!
** i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
** }
*/
int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
int sqlite3Fts5ExprEof(Fts5Expr*);
i64 sqlite3Fts5ExprRowid(Fts5Expr*);
void sqlite3Fts5ExprFree(Fts5Expr*);
/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
int sqlite3Fts5ExprPopulatePoslists(
Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
);
void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
void sqlite3Fts5ExprClearEof(Fts5Expr*);
int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y. */
void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
Fts5ExprNode *sqlite3Fts5ParseNode(
Fts5Parse *pParse,
int eType,
Fts5ExprNode *pLeft,
Fts5ExprNode *pRight,
Fts5ExprNearset *pNear
);
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse,
Fts5ExprPhrase *pPhrase,
Fts5Token *pToken,
int bPrefix
);
Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5Parse*,
Fts5ExprNearset*,
Fts5ExprPhrase*
);
Fts5Colset *sqlite3Fts5ParseColset(
Fts5Parse*,
Fts5Colset*,
Fts5Token *
);
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
/*
** End of interface to code in fts5_expr.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_aux.c.
*/
int sqlite3Fts5AuxInit(fts5_api*);
/*
** End of interface to code in fts5_aux.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_tokenizer.c.
*/
int sqlite3Fts5TokenizerInit(fts5_api*);
/*
** End of interface to code in fts5_tokenizer.c.
**************************************************************************/
/**************************************************************************
** Interface to code in fts5_vocab.c.
*/
int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
/*
** End of interface to code in fts5_vocab.c.
**************************************************************************/
/**************************************************************************
** Interface to automatically generated code in fts5_unicode2.c.
*/
int sqlite3Fts5UnicodeIsalnum(int c);
int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/
#endif

562
ext/fts5/fts5_aux.c Normal file
View File

@ -0,0 +1,562 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
#include "fts5Int.h"
#include <math.h> /* amalgamator: keep */
/*
** Object used to iterate through all "coalesced phrase instances" in
** a single column of the current row. If the phrase instances in the
** column being considered do not overlap, this object simply iterates
** through them. Or, if they do overlap (share one or more tokens in
** common), each set of overlapping instances is treated as a single
** match. See documentation for the highlight() auxiliary function for
** details.
**
** Usage is:
**
** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter);
** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter);
** rc = fts5CInstIterNext(&iter)
** ){
** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd);
** }
**
*/
typedef struct CInstIter CInstIter;
struct CInstIter {
const Fts5ExtensionApi *pApi; /* API offered by current FTS version */
Fts5Context *pFts; /* First arg to pass to pApi functions */
int iCol; /* Column to search */
int iInst; /* Next phrase instance index */
int nInst; /* Total number of phrase instances */
/* Output variables */
int iStart; /* First token in coalesced phrase instance */
int iEnd; /* Last token in coalesced phrase instance */
};
/*
** Advance the iterator to the next coalesced phrase instance. Return
** an SQLite error code if an error occurs, or SQLITE_OK otherwise.
*/
static int fts5CInstIterNext(CInstIter *pIter){
int rc = SQLITE_OK;
pIter->iStart = -1;
pIter->iEnd = -1;
while( rc==SQLITE_OK && pIter->iInst<pIter->nInst ){
int ip; int ic; int io;
rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io);
if( rc==SQLITE_OK ){
if( ic==pIter->iCol ){
int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip);
if( pIter->iStart<0 ){
pIter->iStart = io;
pIter->iEnd = iEnd;
}else if( io<=pIter->iEnd ){
if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd;
}else{
break;
}
}
pIter->iInst++;
}
}
return rc;
}
/*
** Initialize the iterator object indicated by the final parameter to
** iterate through coalesced phrase instances in column iCol.
*/
static int fts5CInstIterInit(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
int iCol,
CInstIter *pIter
){
int rc;
memset(pIter, 0, sizeof(CInstIter));
pIter->pApi = pApi;
pIter->pFts = pFts;
pIter->iCol = iCol;
rc = pApi->xInstCount(pFts, &pIter->nInst);
if( rc==SQLITE_OK ){
rc = fts5CInstIterNext(pIter);
}
return rc;
}
/*************************************************************************
** Start of highlight() implementation.
*/
typedef struct HighlightContext HighlightContext;
struct HighlightContext {
CInstIter iter; /* Coalesced Instance Iterator */
int iPos; /* Current token offset in zIn[] */
int iRangeStart; /* First token to include */
int iRangeEnd; /* If non-zero, last token to include */
const char *zOpen; /* Opening highlight */
const char *zClose; /* Closing highlight */
const char *zIn; /* Input text */
int nIn; /* Size of input text in bytes */
int iOff; /* Current offset within zIn[] */
char *zOut; /* Output value */
};
/*
** Append text to the HighlightContext output string - p->zOut. Argument
** z points to a buffer containing n bytes of text to append. If n is
** negative, everything up until the first '\0' is appended to the output.
**
** If *pRc is set to any value other than SQLITE_OK when this function is
** called, it is a no-op. If an error (i.e. an OOM condition) is encountered,
** *pRc is set to an error code before returning.
*/
static void fts5HighlightAppend(
int *pRc,
HighlightContext *p,
const char *z, int n
){
if( *pRc==SQLITE_OK ){
if( n<0 ) n = (int)strlen(z);
p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
}
}
/*
** Tokenizer callback used by implementation of highlight() function.
*/
static int fts5HighlightCb(
void *pContext, /* Pointer to HighlightContext object */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start offset of token */
int iEndOff /* End offset of token */
){
HighlightContext *p = (HighlightContext*)pContext;
int rc = SQLITE_OK;
int iPos;
UNUSED_PARAM2(pToken, nToken);
if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
iPos = p->iPos++;
if( p->iRangeEnd>0 ){
if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
}
if( iPos==p->iter.iStart ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zOpen, -1);
p->iOff = iStartOff;
}
if( iPos==p->iter.iEnd ){
if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
fts5HighlightAppend(&rc, p, p->zOpen, -1);
}
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
fts5HighlightAppend(&rc, p, p->zClose, -1);
p->iOff = iEndOff;
if( rc==SQLITE_OK ){
rc = fts5CInstIterNext(&p->iter);
}
}
if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
p->iOff = iEndOff;
if( iPos<p->iter.iEnd ){
fts5HighlightAppend(&rc, p, p->zClose, -1);
}
}
return rc;
}
/*
** Implementation of highlight() function.
*/
static void fts5HighlightFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
HighlightContext ctx;
int rc;
int iCol;
if( nVal!=3 ){
const char *zErr = "wrong number of arguments to function highlight()";
sqlite3_result_error(pCtx, zErr, -1);
return;
}
iCol = sqlite3_value_int(apVal[0]);
memset(&ctx, 0, sizeof(HighlightContext));
ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}
sqlite3_free(ctx.zOut);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}
}
/*
** End of highlight() implementation.
**************************************************************************/
/*
** Implementation of snippet() function.
*/
static void fts5SnippetFunction(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
HighlightContext ctx;
int rc = SQLITE_OK; /* Return code */
int iCol; /* 1st argument to snippet() */
const char *zEllips; /* 4th argument to snippet() */
int nToken; /* 5th argument to snippet() */
int nInst = 0; /* Number of instance matches this row */
int i; /* Used to iterate through instances */
int nPhrase; /* Number of phrases in query */
unsigned char *aSeen; /* Array of "seen instance" flags */
int iBestCol; /* Column containing best snippet */
int iBestStart = 0; /* First token of best snippet */
int iBestLast; /* Last token of best snippet */
int nBestScore = 0; /* Score of best snippet */
int nColSize = 0; /* Total size of iBestCol in tokens */
if( nVal!=5 ){
const char *zErr = "wrong number of arguments to function snippet()";
sqlite3_result_error(pCtx, zErr, -1);
return;
}
memset(&ctx, 0, sizeof(HighlightContext));
iCol = sqlite3_value_int(apVal[0]);
ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
zEllips = (const char*)sqlite3_value_text(apVal[3]);
nToken = sqlite3_value_int(apVal[4]);
iBestLast = nToken-1;
iBestCol = (iCol>=0 ? iCol : 0);
nPhrase = pApi->xPhraseCount(pFts);
aSeen = sqlite3_malloc(nPhrase);
if( aSeen==0 ){
rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
rc = pApi->xInstCount(pFts, &nInst);
}
for(i=0; rc==SQLITE_OK && i<nInst; i++){
int ip, iSnippetCol, iStart;
memset(aSeen, 0, nPhrase);
rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
int nScore = 1000;
int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
int j;
aSeen[ip] = 1;
for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
int ic; int io; int iFinal;
rc = pApi->xInst(pFts, j, &ip, &ic, &io);
iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
nScore += aSeen[ip] ? 1000 : 1;
aSeen[ip] = 1;
if( iFinal>iLast ) iLast = iFinal;
}
}
if( rc==SQLITE_OK && nScore>nBestScore ){
iBestCol = iSnippetCol;
iBestStart = iStart;
iBestLast = iLast;
nBestScore = nScore;
}
}
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
}
if( rc==SQLITE_OK ){
rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
}
if( ctx.zIn ){
if( rc==SQLITE_OK ){
rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
}
if( (iBestStart+nToken-1)>iBestLast ){
iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
}
if( iBestStart+nToken>nColSize ){
iBestStart = nColSize - nToken;
}
if( iBestStart<0 ) iBestStart = 0;
ctx.iRangeStart = iBestStart;
ctx.iRangeEnd = iBestStart + nToken - 1;
if( iBestStart>0 ){
fts5HighlightAppend(&rc, &ctx, zEllips, -1);
}
if( rc==SQLITE_OK ){
rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
}
if( ctx.iRangeEnd>=(nColSize-1) ){
fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
}else{
fts5HighlightAppend(&rc, &ctx, zEllips, -1);
}
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
}else{
sqlite3_result_error_code(pCtx, rc);
}
sqlite3_free(ctx.zOut);
}
sqlite3_free(aSeen);
}
/************************************************************************/
/*
** The first time the bm25() function is called for a query, an instance
** of the following structure is allocated and populated.
*/
typedef struct Fts5Bm25Data Fts5Bm25Data;
struct Fts5Bm25Data {
int nPhrase; /* Number of phrases in query */
double avgdl; /* Average number of tokens in each row */
double *aIDF; /* IDF for each phrase */
double *aFreq; /* Array used to calculate phrase freq. */
};
/*
** Callback used by fts5Bm25GetData() to count the number of rows in the
** table matched by each individual phrase within the query.
*/
static int fts5CountCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
void *pUserData /* Pointer to sqlite3_int64 variable */
){
sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
UNUSED_PARAM2(pApi, pFts);
(*pn)++;
return SQLITE_OK;
}
/*
** Set *ppData to point to the Fts5Bm25Data object for the current query.
** If the object has not already been allocated, allocate and populate it
** now.
*/
static int fts5Bm25GetData(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */
){
int rc = SQLITE_OK; /* Return code */
Fts5Bm25Data *p; /* Object to return */
p = pApi->xGetAuxdata(pFts, 0);
if( p==0 ){
int nPhrase; /* Number of phrases in query */
sqlite3_int64 nRow = 0; /* Number of rows in table */
sqlite3_int64 nToken = 0; /* Number of tokens in table */
int nByte; /* Bytes of space to allocate */
int i;
/* Allocate the Fts5Bm25Data object */
nPhrase = pApi->xPhraseCount(pFts);
nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double);
p = (Fts5Bm25Data*)sqlite3_malloc(nByte);
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
memset(p, 0, nByte);
p->nPhrase = nPhrase;
p->aIDF = (double*)&p[1];
p->aFreq = &p->aIDF[nPhrase];
}
/* Calculate the average document length for this FTS5 table */
if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow);
if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken);
if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow;
/* Calculate an IDF for each phrase in the query */
for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
sqlite3_int64 nHit = 0;
rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb);
if( rc==SQLITE_OK ){
/* Calculate the IDF (Inverse Document Frequency) for phrase i.
** This is done using the standard BM25 formula as found on wikipedia:
**
** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) )
**
** where "N" is the total number of documents in the set and nHit
** is the number that contain at least one instance of the phrase
** under consideration.
**
** The problem with this is that if (N < 2*nHit), the IDF is
** negative. Which is undesirable. So the mimimum allowable IDF is
** (1e-6) - roughly the same as a term that appears in just over
** half of set of 5,000,000 documents. */
double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) );
if( idf<=0.0 ) idf = 1e-6;
p->aIDF[i] = idf;
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(p);
}else{
rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
}
if( rc!=SQLITE_OK ) p = 0;
}
*ppData = p;
return rc;
}
/*
** Implementation of bm25() function.
*/
static void fts5Bm25Function(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
const double k1 = 1.2; /* Constant "k1" from BM25 formula */
const double b = 0.75; /* Constant "b" from BM25 formula */
int rc = SQLITE_OK; /* Error code */
double score = 0.0; /* SQL function return value */
Fts5Bm25Data *pData; /* Values allocated/calculated once only */
int i; /* Iterator variable */
int nInst = 0; /* Value returned by xInstCount() */
double D = 0.0; /* Total number of tokens in row */
double *aFreq = 0; /* Array of phrase freq. for current row */
/* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation)
** for each phrase in the query for the current row. */
rc = fts5Bm25GetData(pApi, pFts, &pData);
if( rc==SQLITE_OK ){
aFreq = pData->aFreq;
memset(aFreq, 0, sizeof(double) * pData->nPhrase);
rc = pApi->xInstCount(pFts, &nInst);
}
for(i=0; rc==SQLITE_OK && i<nInst; i++){
int ip; int ic; int io;
rc = pApi->xInst(pFts, i, &ip, &ic, &io);
if( rc==SQLITE_OK ){
double w = (nVal > ic) ? sqlite3_value_double(apVal[ic]) : 1.0;
aFreq[ip] += w;
}
}
/* Figure out the total size of the current row in tokens. */
if( rc==SQLITE_OK ){
int nTok;
rc = pApi->xColumnSize(pFts, -1, &nTok);
D = (double)nTok;
}
/* Determine the BM25 score for the current row. */
for(i=0; rc==SQLITE_OK && i<pData->nPhrase; i++){
score += pData->aIDF[i] * (
( aFreq[i] * (k1 + 1.0) ) /
( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) )
);
}
/* If no error has occurred, return the calculated score. Otherwise,
** throw an SQL exception. */
if( rc==SQLITE_OK ){
sqlite3_result_double(pCtx, -1.0 * score);
}else{
sqlite3_result_error_code(pCtx, rc);
}
}
int sqlite3Fts5AuxInit(fts5_api *pApi){
struct Builtin {
const char *zFunc; /* Function name (nul-terminated) */
void *pUserData; /* User-data pointer */
fts5_extension_function xFunc;/* Callback function */
void (*xDestroy)(void*); /* Destructor function */
} aBuiltin [] = {
{ "snippet", 0, fts5SnippetFunction, 0 },
{ "highlight", 0, fts5HighlightFunction, 0 },
{ "bm25", 0, fts5Bm25Function, 0 },
};
int rc = SQLITE_OK; /* Return code */
int i; /* To iterate through builtin functions */
for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
rc = pApi->xCreateFunction(pApi,
aBuiltin[i].zFunc,
aBuiltin[i].pUserData,
aBuiltin[i].xFunc,
aBuiltin[i].xDestroy
);
}
return rc;
}

392
ext/fts5/fts5_buffer.c Normal file
View File

@ -0,0 +1,392 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
#include "fts5Int.h"
int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
if( (u32)pBuf->nSpace<nByte ){
u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
u8 *pNew;
while( nNew<nByte ){
nNew = nNew * 2;
}
pNew = sqlite3_realloc(pBuf->p, nNew);
if( pNew==0 ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
pBuf->nSpace = nNew;
pBuf->p = pNew;
}
}
return 0;
}
/*
** Encode value iVal as an SQLite varint and append it to the buffer object
** pBuf. If an OOM error occurs, set the error code in p.
*/
void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
if( fts5BufferGrow(pRc, pBuf, 9) ) return;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
}
void sqlite3Fts5Put32(u8 *aBuf, int iVal){
aBuf[0] = (iVal>>24) & 0x00FF;
aBuf[1] = (iVal>>16) & 0x00FF;
aBuf[2] = (iVal>> 8) & 0x00FF;
aBuf[3] = (iVal>> 0) & 0x00FF;
}
int sqlite3Fts5Get32(const u8 *aBuf){
return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3];
}
/*
** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
void sqlite3Fts5BufferAppendBlob(
int *pRc,
Fts5Buffer *pBuf,
u32 nData,
const u8 *pData
){
assert_nc( *pRc || nData>=0 );
if( fts5BufferGrow(pRc, pBuf, nData) ) return;
memcpy(&pBuf->p[pBuf->n], pData, nData);
pBuf->n += nData;
}
/*
** Append the nul-terminated string zStr to the buffer pBuf. This function
** ensures that the byte following the buffer data is set to 0x00, even
** though this byte is not included in the pBuf->n count.
*/
void sqlite3Fts5BufferAppendString(
int *pRc,
Fts5Buffer *pBuf,
const char *zStr
){
int nStr = (int)strlen(zStr);
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr);
pBuf->n--;
}
/*
** Argument zFmt is a printf() style format string. This function performs
** the printf() style processing, then appends the results to buffer pBuf.
**
** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte
** following the buffer data is set to 0x00, even though this byte is not
** included in the pBuf->n count.
*/
void sqlite3Fts5BufferAppendPrintf(
int *pRc,
Fts5Buffer *pBuf,
char *zFmt, ...
){
if( *pRc==SQLITE_OK ){
char *zTmp;
va_list ap;
va_start(ap, zFmt);
zTmp = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zTmp==0 ){
*pRc = SQLITE_NOMEM;
}else{
sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
sqlite3_free(zTmp);
}
}
}
char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
va_list ap;
va_start(ap, zFmt);
zRet = sqlite3_vmprintf(zFmt, ap);
va_end(ap);
if( zRet==0 ){
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
/*
** Free any buffer allocated by pBuf. Zero the structure before returning.
*/
void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
sqlite3_free(pBuf->p);
memset(pBuf, 0, sizeof(Fts5Buffer));
}
/*
** Zero the contents of the buffer object. But do not free the associated
** memory allocation.
*/
void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
pBuf->n = 0;
}
/*
** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
** the error code in p. If an error has already occurred when this function
** is called, it is a no-op.
*/
void sqlite3Fts5BufferSet(
int *pRc,
Fts5Buffer *pBuf,
int nData,
const u8 *pData
){
pBuf->n = 0;
sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
}
int sqlite3Fts5PoslistNext64(
const u8 *a, int n, /* Buffer containing poslist */
int *pi, /* IN/OUT: Offset within a[] */
i64 *piOff /* IN/OUT: Current offset */
){
int i = *pi;
if( i>=n ){
/* EOF */
*piOff = -1;
return 1;
}else{
i64 iOff = *piOff;
int iVal;
fts5FastGetVarint32(a, i, iVal);
if( iVal==1 ){
fts5FastGetVarint32(a, i, iVal);
iOff = ((i64)iVal) << 32;
fts5FastGetVarint32(a, i, iVal);
}
*piOff = iOff + (iVal-2);
*pi = i;
return 0;
}
}
/*
** Advance the iterator object passed as the only argument. Return true
** if the iterator reaches EOF, or false otherwise.
*/
int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){
pIter->bEof = 1;
}
return pIter->bEof;
}
int sqlite3Fts5PoslistReaderInit(
const u8 *a, int n, /* Poslist buffer to iterate through */
Fts5PoslistReader *pIter /* Iterator object to initialize */
){
memset(pIter, 0, sizeof(*pIter));
pIter->a = a;
pIter->n = n;
sqlite3Fts5PoslistReaderNext(pIter);
return pIter->bEof;
}
/*
** Append position iPos to the position list being accumulated in buffer
** pBuf, which must be already be large enough to hold the new data.
** The previous position written to this list is *piPrev. *piPrev is set
** to iPos before returning.
*/
void sqlite3Fts5PoslistSafeAppend(
Fts5Buffer *pBuf,
i64 *piPrev,
i64 iPos
){
static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
if( (iPos & colmask) != (*piPrev & colmask) ){
pBuf->p[pBuf->n++] = 1;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
*piPrev = (iPos & colmask);
}
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
*piPrev = iPos;
}
int sqlite3Fts5PoslistWriterAppend(
Fts5Buffer *pBuf,
Fts5PoslistWriter *pWriter,
i64 iPos
){
int rc = 0; /* Initialized only to suppress erroneous warning from Clang */
if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
return SQLITE_OK;
}
void *sqlite3Fts5MallocZero(int *pRc, int nByte){
void *pRet = 0;
if( *pRc==SQLITE_OK ){
pRet = sqlite3_malloc(nByte);
if( pRet==0 && nByte>0 ){
*pRc = SQLITE_NOMEM;
}else{
memset(pRet, 0, nByte);
}
}
return pRet;
}
/*
** Return a nul-terminated copy of the string indicated by pIn. If nIn
** is non-negative, then it is the length of the string in bytes. Otherwise,
** the length of the string is determined using strlen().
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
*/
char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
char *zRet = 0;
if( *pRc==SQLITE_OK ){
if( nIn<0 ){
nIn = (int)strlen(pIn);
}
zRet = (char*)sqlite3_malloc(nIn+1);
if( zRet ){
memcpy(zRet, pIn, nIn);
zRet[nIn] = '\0';
}else{
*pRc = SQLITE_NOMEM;
}
}
return zRet;
}
/*
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
** * All non-ASCII characters,
** * The 52 upper and lower case ASCII characters, and
** * The 10 integer ASCII characters.
** * The underscore character "_" (0x5F).
** * The unicode "subsitute" character (0x1A).
*/
int sqlite3Fts5IsBareword(char t){
u8 aBareword[128] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */
};
return (t & 0x80) || aBareword[(int)t];
}
/*************************************************************************
*/
typedef struct Fts5TermsetEntry Fts5TermsetEntry;
struct Fts5TermsetEntry {
char *pTerm;
int nTerm;
int iIdx; /* Index (main or aPrefix[] entry) */
Fts5TermsetEntry *pNext;
};
struct Fts5Termset {
Fts5TermsetEntry *apHash[512];
};
int sqlite3Fts5TermsetNew(Fts5Termset **pp){
int rc = SQLITE_OK;
*pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
return rc;
}
int sqlite3Fts5TermsetAdd(
Fts5Termset *p,
int iIdx,
const char *pTerm, int nTerm,
int *pbPresent
){
int rc = SQLITE_OK;
*pbPresent = 0;
if( p ){
int i;
u32 hash = 13;
Fts5TermsetEntry *pEntry;
/* Calculate a hash value for this term. This is the same hash checksum
** used by the fts5_hash.c module. This is not important for correct
** operation of the module, but is necessary to ensure that some tests
** designed to produce hash table collisions really do work. */
for(i=nTerm-1; i>=0; i--){
hash = (hash << 3) ^ hash ^ pTerm[i];
}
hash = (hash << 3) ^ hash ^ iIdx;
hash = hash % ArraySize(p->apHash);
for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
if( pEntry->iIdx==iIdx
&& pEntry->nTerm==nTerm
&& memcmp(pEntry->pTerm, pTerm, nTerm)==0
){
*pbPresent = 1;
break;
}
}
if( pEntry==0 ){
pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
if( pEntry ){
pEntry->pTerm = (char*)&pEntry[1];
pEntry->nTerm = nTerm;
pEntry->iIdx = iIdx;
memcpy(pEntry->pTerm, pTerm, nTerm);
pEntry->pNext = p->apHash[hash];
p->apHash[hash] = pEntry;
}
}
}
return rc;
}
void sqlite3Fts5TermsetFree(Fts5Termset *p){
if( p ){
u32 i;
for(i=0; i<ArraySize(p->apHash); i++){
Fts5TermsetEntry *pEntry = p->apHash[i];
while( pEntry ){
Fts5TermsetEntry *pDel = pEntry;
pEntry = pEntry->pNext;
sqlite3_free(pDel);
}
}
sqlite3_free(p);
}
}

945
ext/fts5/fts5_config.c Normal file
View File

@ -0,0 +1,945 @@
/*
** 2014 Jun 09
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/
#include "fts5Int.h"
#define FTS5_DEFAULT_PAGE_SIZE 4050
#define FTS5_DEFAULT_AUTOMERGE 4
#define FTS5_DEFAULT_CRISISMERGE 16
#define FTS5_DEFAULT_HASHSIZE (1024*1024)
/* Maximum allowed page size */
#define FTS5_MAX_PAGE_SIZE (128*1024)
static int fts5_iswhitespace(char x){
return (x==' ');
}
static int fts5_isopenquote(char x){
return (x=='"' || x=='\'' || x=='[' || x=='`');
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a white-space character.
*/
static const char *fts5ConfigSkipWhitespace(const char *pIn){
const char *p = pIn;
if( p ){
while( fts5_iswhitespace(*p) ){ p++; }
}
return p;
}
/*
** Argument pIn points to a character that is part of a nul-terminated
** string. Return a pointer to the first character following *pIn in
** the string that is not a "bareword" character.
*/
static const char *fts5ConfigSkipBareword(const char *pIn){
const char *p = pIn;
while ( sqlite3Fts5IsBareword(*p) ) p++;
if( p==pIn ) p = 0;
return p;
}
static int fts5_isdigit(char a){
return (a>='0' && a<='9');
}
static const char *fts5ConfigSkipLiteral(const char *pIn){
const char *p = pIn;
switch( *p ){
case 'n': case 'N':
if( sqlite3_strnicmp("null", p, 4)==0 ){
p = &p[4];
}else{
p = 0;
}
break;
case 'x': case 'X':
p++;
if( *p=='\'' ){
p++;
while( (*p>='a' && *p<='f')
|| (*p>='A' && *p<='F')
|| (*p>='0' && *p<='9')
){
p++;
}
if( *p=='\'' && 0==((p-pIn)%2) ){
p++;
}else{
p = 0;
}
}else{
p = 0;
}
break;
case '\'':
p++;
while( p ){
if( *p=='\'' ){
p++;
if( *p!='\'' ) break;
}
p++;
if( *p==0 ) p = 0;
}
break;
default:
/* maybe a number */
if( *p=='+' || *p=='-' ) p++;
while( fts5_isdigit(*p) ) p++;
/* At this point, if the literal was an integer, the parse is
** finished. Or, if it is a floating point value, it may continue
** with either a decimal point or an 'E' character. */
if( *p=='.' && fts5_isdigit(p[1]) ){
p += 2;
while( fts5_isdigit(*p) ) p++;
}
if( p==pIn ) p = 0;
break;
}
return p;
}
/*
** The first character of the string pointed to by argument z is guaranteed
** to be an open-quote character (see function fts5_isopenquote()).
**
** This function searches for the corresponding close-quote character within
** the string and, if found, dequotes the string in place and adds a new
** nul-terminator byte.
**
** If the close-quote is found, the value returned is the byte offset of
** the character immediately following it. Or, if the close-quote is not
** found, -1 is returned. If -1 is returned, the buffer is left in an
** undefined state.
*/
static int fts5Dequote(char *z){
char q;
int iIn = 1;
int iOut = 0;
q = z[0];
/* Set stack variable q to the close-quote character */
assert( q=='[' || q=='\'' || q=='"' || q=='`' );
if( q=='[' ) q = ']';
while( ALWAYS(z[iIn]) ){
if( z[iIn]==q ){
if( z[iIn+1]!=q ){
/* Character iIn was the close quote. */
iIn++;
break;
}else{
/* Character iIn and iIn+1 form an escaped quote character. Skip
** the input cursor past both and copy a single quote character
** to the output buffer. */
iIn += 2;
z[iOut++] = q;
}
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
return iIn;
}
/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters. The conversion is done in-place. If the
** input does not begin with a quote character, then this routine
** is a no-op.
**
** Examples:
**
** "abc" becomes abc
** 'xyz' becomes xyz
** [pqr] becomes pqr
** `mno` becomes mno
*/
void sqlite3Fts5Dequote(char *z){
char quote; /* Quote character (if any ) */
assert( 0==fts5_iswhitespace(z[0]) );
quote = z[0];
if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
fts5Dequote(z);
}
}
struct Fts5Enum {
const char *zName;
int eVal;
};
typedef struct Fts5Enum Fts5Enum;
static int fts5ConfigSetEnum(
const Fts5Enum *aEnum,
const char *zEnum,
int *peVal
){
int nEnum = (int)strlen(zEnum);
int i;
int iVal = -1;
for(i=0; aEnum[i].zName; i++){
if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
if( iVal>=0 ) return SQLITE_ERROR;
iVal = aEnum[i].eVal;
}
}
*peVal = iVal;
return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
}
/*
** Parse a "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
Fts5Global *pGlobal,
Fts5Config *pConfig, /* Configuration object to update */
const char *zCmd, /* Special command to parse */
const char *zArg, /* Argument to parse */
char **pzErr /* OUT: Error message */
){
int rc = SQLITE_OK;
int nCmd = (int)strlen(zCmd);
if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
const char *p;
int bFirst = 1;
if( pConfig->aPrefix==0 ){
pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
if( rc ) return rc;
}
p = zArg;
while( 1 ){
int nPre = 0;
while( p[0]==' ' ) p++;
if( bFirst==0 && p[0]==',' ){
p++;
while( p[0]==' ' ) p++;
}else if( p[0]=='\0' ){
break;
}
if( p[0]<'0' || p[0]>'9' ){
*pzErr = sqlite3_mprintf("malformed prefix=... directive");
rc = SQLITE_ERROR;
break;
}
if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
*pzErr = sqlite3_mprintf(
"too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
);
rc = SQLITE_ERROR;
break;
}
while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
nPre = nPre*10 + (p[0] - '0');
p++;
}
if( nPre<=0 || nPre>=1000 ){
*pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
rc = SQLITE_ERROR;
break;
}
pConfig->aPrefix[pConfig->nPrefix] = nPre;
pConfig->nPrefix++;
bFirst = 0;
}
assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
return rc;
}
if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
const char *p = (const char*)zArg;
int nArg = (int)strlen(zArg) + 1;
char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
char *pSpace = pDel;
if( azArg && pSpace ){
if( pConfig->pTok ){
*pzErr = sqlite3_mprintf("multiple tokenize=... directives");
rc = SQLITE_ERROR;
}else{
for(nArg=0; p && *p; nArg++){
const char *p2 = fts5ConfigSkipWhitespace(p);
if( *p2=='\'' ){
p = fts5ConfigSkipLiteral(p2);
}else{
p = fts5ConfigSkipBareword(p2);
}
if( p ){
memcpy(pSpace, p2, p-p2);
azArg[nArg] = pSpace;
sqlite3Fts5Dequote(pSpace);
pSpace += (p - p2) + 1;
p = fts5ConfigSkipWhitespace(p);
}
}
if( p==0 ){
*pzErr = sqlite3_mprintf("parse error in tokenize directive");
rc = SQLITE_ERROR;
}else{
rc = sqlite3Fts5GetTokenizer(pGlobal,
(const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
pzErr
);
}
}
}
sqlite3_free(azArg);
sqlite3_free(pDel);
return rc;
}
if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
*pzErr = sqlite3_mprintf("multiple content=... directives");
rc = SQLITE_ERROR;
}else{
if( zArg[0] ){
pConfig->eContent = FTS5_CONTENT_EXTERNAL;
pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg);
}else{
pConfig->eContent = FTS5_CONTENT_NONE;
}
}
return rc;
}
if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
if( pConfig->zContentRowid ){
*pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
rc = SQLITE_ERROR;
}else{
pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
}
return rc;
}
if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
*pzErr = sqlite3_mprintf("malformed columnsize=... directive");
rc = SQLITE_ERROR;
}else{
pConfig->bColumnsize = (zArg[0]=='1');
}
return rc;
}
if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
const Fts5Enum aDetail[] = {
{ "none", FTS5_DETAIL_NONE },
{ "full", FTS5_DETAIL_FULL },
{ "columns", FTS5_DETAIL_COLUMNS },
{ 0, 0 }
};
if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
*pzErr = sqlite3_mprintf("malformed detail=... directive");
}
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
return SQLITE_ERROR;
}
/*
** Allocate an instance of the default tokenizer ("simple") at
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
return sqlite3Fts5GetTokenizer(
pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
);
}
/*
** Gobble up the first bareword or quoted word from the input buffer zIn.
** Return a pointer to the character immediately following the last in
** the gobbled word if successful, or a NULL pointer otherwise (failed
** to find close-quote character).
**
** Before returning, set pzOut to point to a new buffer containing a
** nul-terminated, dequoted copy of the gobbled word. If the word was
** quoted, *pbQuoted is also set to 1 before returning.
**
** If *pRc is other than SQLITE_OK when this function is called, it is
** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
** set if a parse error (failed to find close quote) occurs.
*/
static const char *fts5ConfigGobbleWord(
int *pRc, /* IN/OUT: Error code */
const char *zIn, /* Buffer to gobble string/bareword from */
char **pzOut, /* OUT: malloc'd buffer containing str/bw */
int *pbQuoted /* OUT: Set to true if dequoting required */
){
const char *zRet = 0;
int nIn = (int)strlen(zIn);
char *zOut = sqlite3_malloc(nIn+1);
assert( *pRc==SQLITE_OK );
*pbQuoted = 0;
*pzOut = 0;
if( zOut==0 ){
*pRc = SQLITE_NOMEM;
}else{
memcpy(zOut, zIn, nIn+1);
if( fts5_isopenquote(zOut[0]) ){
int ii = fts5Dequote(zOut);
zRet = &zIn[ii];
*pbQuoted = 1;
}else{
zRet = fts5ConfigSkipBareword(zIn);
zOut[zRet-zIn] = '\0';
}
}
if( zRet==0 ){
sqlite3_free(zOut);
}else{
*pzOut = zOut;
}
return zRet;
}
static int fts5ConfigParseColumn(
Fts5Config *p,
char *zCol,
char *zArg,
char **pzErr
){
int rc = SQLITE_OK;
if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME)
|| 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME)
){
*pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
rc = SQLITE_ERROR;
}else if( zArg ){
if( 0==sqlite3_stricmp(zArg, "unindexed") ){
p->abUnindexed[p->nCol] = 1;
}else{
*pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg);
rc = SQLITE_ERROR;
}
}
p->azCol[p->nCol++] = zCol;
return rc;
}
/*
** Populate the Fts5Config.zContentExprlist string.
*/
static int fts5ConfigMakeExprlist(Fts5Config *p){
int i;
int rc = SQLITE_OK;
Fts5Buffer buf = {0, 0, 0};
sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
if( p->eContent!=FTS5_CONTENT_NONE ){
for(i=0; i<p->nCol; i++){
if( p->eContent==FTS5_CONTENT_EXTERNAL ){
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
}else{
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
}
}
}
assert( p->zContentExprlist==0 );
p->zContentExprlist = (char*)buf.p;
return rc;
}
/*
** Arguments nArg/azArg contain the string arguments passed to the xCreate
** or xConnect method of the virtual table. This function attempts to
** allocate an instance of Fts5Config containing the results of parsing
** those arguments.
**
** If successful, SQLITE_OK is returned and *ppOut is set to point to the
** new Fts5Config object. If an error occurs, an SQLite error code is
** returned, *ppOut is set to NULL and an error message may be left in
** *pzErr. It is the responsibility of the caller to eventually free any
** such error message using sqlite3_free().
*/
int sqlite3Fts5ConfigParse(
Fts5Global *pGlobal,
sqlite3 *db,
int nArg, /* Number of arguments */
const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
Fts5Config **ppOut, /* OUT: Results of parse */
char **pzErr /* OUT: Error message */
){
int rc = SQLITE_OK; /* Return code */
Fts5Config *pRet; /* New object to return */
int i;
int nByte;
*ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
if( pRet==0 ) return SQLITE_NOMEM;
memset(pRet, 0, sizeof(Fts5Config));
pRet->db = db;
pRet->iCookie = -1;
nByte = nArg * (sizeof(char*) + sizeof(u8));
pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
pRet->bColumnsize = 1;
pRet->eDetail = FTS5_DETAIL_FULL;
#ifdef SQLITE_DEBUG
pRet->bPrefixIndex = 1;
#endif
if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
*pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
rc = SQLITE_ERROR;
}
for(i=3; rc==SQLITE_OK && i<nArg; i++){
const char *zOrig = azArg[i];
const char *z;
char *zOne = 0;
char *zTwo = 0;
int bOption = 0;
int bMustBeCol = 0;
z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
z = fts5ConfigSkipWhitespace(z);
if( z && *z=='=' ){
bOption = 1;
z++;
if( bMustBeCol ) z = 0;
}
z = fts5ConfigSkipWhitespace(z);
if( z && z[0] ){
int bDummy;
z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
if( z && z[0] ) z = 0;
}
if( rc==SQLITE_OK ){
if( z==0 ){
*pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
rc = SQLITE_ERROR;
}else{
if( bOption ){
rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
}else{
rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
zOne = 0;
}
}
}
sqlite3_free(zOne);
sqlite3_free(zTwo);
}
/* If a tokenizer= option was successfully parsed, the tokenizer has
** already been allocated. Otherwise, allocate an instance of the default
** tokenizer (unicode61) now. */
if( rc==SQLITE_OK && pRet->pTok==0 ){
rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
}
/* If no zContent option was specified, fill in the default values. */
if( rc==SQLITE_OK && pRet->zContent==0 ){
const char *zTail = 0;
assert( pRet->eContent==FTS5_CONTENT_NORMAL
|| pRet->eContent==FTS5_CONTENT_NONE
);
if( pRet->eContent==FTS5_CONTENT_NORMAL ){
zTail = "content";
}else if( pRet->bColumnsize ){
zTail = "docsize";
}
if( zTail ){
pRet->zContent = sqlite3Fts5Mprintf(
&rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
);
}
}
if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
}
/* Formulate the zContentExprlist text */
if( rc==SQLITE_OK ){
rc = fts5ConfigMakeExprlist(pRet);
}
if( rc!=SQLITE_OK ){
sqlite3Fts5ConfigFree(pRet);
*ppOut = 0;
}
return rc;
}
/*
** Free the configuration object passed as the only argument.
*/
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
if( pConfig ){
int i;
if( pConfig->pTok ){
pConfig->pTokApi->xDelete(pConfig->pTok);
}
sqlite3_free(pConfig->zDb);
sqlite3_free(pConfig->zName);
for(i=0; i<pConfig->nCol; i++){
sqlite3_free(pConfig->azCol[i]);
}
sqlite3_free(pConfig->azCol);
sqlite3_free(pConfig->aPrefix);
sqlite3_free(pConfig->zRank);
sqlite3_free(pConfig->zRankArgs);
sqlite3_free(pConfig->zContent);
sqlite3_free(pConfig->zContentRowid);
sqlite3_free(pConfig->zContentExprlist);
sqlite3_free(pConfig);
}
}
/*
** Call sqlite3_declare_vtab() based on the contents of the configuration
** object passed as the only argument. Return SQLITE_OK if successful, or
** an SQLite error code if an error occurs.
*/
int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
int i;
int rc = SQLITE_OK;
char *zSql;
zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x(");
for(i=0; zSql && i<pConfig->nCol; i++){
const char *zSep = (i==0?"":", ");
zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]);
}
zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)",
zSql, pConfig->zName, FTS5_RANK_NAME
);
assert( zSql || rc==SQLITE_NOMEM );
if( zSql ){
rc = sqlite3_declare_vtab(pConfig->db, zSql);
sqlite3_free(zSql);
}
return rc;
}
/*
** Tokenize the text passed via the second and third arguments.
**
** The callback is invoked once for each token in the input text. The
** arguments passed to it are, in order:
**
** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
** const char *pToken // Pointer to buffer containing token
** int nToken // Size of token in bytes
** int iStart // Byte offset of start of token within input text
** int iEnd // Byte offset of end of token within input text
** int iPos // Position of token in input (first token is 0)
**
** If the callback returns a non-zero value the tokenization is abandoned
** and no further callbacks are issued.
**
** This function returns SQLITE_OK if successful or an SQLite error code
** if an error occurs. If the tokenization was abandoned early because
** the callback returned SQLITE_DONE, this is not an error and this function
** still returns SQLITE_OK. Or, if the tokenization was abandoned early
** because the callback returned another non-zero value, it is assumed
** to be an SQLite error code and returned to the caller.
*/
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
){
if( pText==0 ) return SQLITE_OK;
return pConfig->pTokApi->xTokenize(
pConfig->pTok, pCtx, flags, pText, nText, xToken
);
}
/*
** Argument pIn points to the first character in what is expected to be
** a comma-separated list of SQL literals followed by a ')' character.
** If it actually is this, return a pointer to the ')'. Otherwise, return
** NULL to indicate a parse error.
*/
static const char *fts5ConfigSkipArgs(const char *pIn){
const char *p = pIn;
while( 1 ){
p = fts5ConfigSkipWhitespace(p);
p = fts5ConfigSkipLiteral(p);
p = fts5ConfigSkipWhitespace(p);
if( p==0 || *p==')' ) break;
if( *p!=',' ){
p = 0;
break;
}
p++;
}
return p;
}
/*
** Parameter zIn contains a rank() function specification. The format of
** this is:
**
** + Bareword (function name)
** + Open parenthesis - "("
** + Zero or more SQL literals in a comma separated list
** + Close parenthesis - ")"
*/
int sqlite3Fts5ConfigParseRank(
const char *zIn, /* Input string */
char **pzRank, /* OUT: Rank function name */
char **pzRankArgs /* OUT: Rank function arguments */
){
const char *p = zIn;
const char *pRank;
char *zRank = 0;
char *zRankArgs = 0;
int rc = SQLITE_OK;
*pzRank = 0;
*pzRankArgs = 0;
if( p==0 ){
rc = SQLITE_ERROR;
}else{
p = fts5ConfigSkipWhitespace(p);
pRank = p;
p = fts5ConfigSkipBareword(p);
if( p ){
zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
if( zRank ) memcpy(zRank, pRank, p-pRank);
}else{
rc = SQLITE_ERROR;
}
if( rc==SQLITE_OK ){
p = fts5ConfigSkipWhitespace(p);
if( *p!='(' ) rc = SQLITE_ERROR;
p++;
}
if( rc==SQLITE_OK ){
const char *pArgs;
p = fts5ConfigSkipWhitespace(p);
pArgs = p;
if( *p!=')' ){
p = fts5ConfigSkipArgs(p);
if( p==0 ){
rc = SQLITE_ERROR;
}else{
zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
}
}
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(zRank);
assert( zRankArgs==0 );
}else{
*pzRank = zRank;
*pzRankArgs = zRankArgs;
}
return rc;
}
int sqlite3Fts5ConfigSetValue(
Fts5Config *pConfig,
const char *zKey,
sqlite3_value *pVal,
int *pbBadkey
){
int rc = SQLITE_OK;
if( 0==sqlite3_stricmp(zKey, "pgsz") ){
int pgsz = 0;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
pgsz = sqlite3_value_int(pVal);
}
if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){
*pbBadkey = 1;
}else{
pConfig->pgsz = pgsz;
}
}
else if( 0==sqlite3_stricmp(zKey, "hashsize") ){
int nHashSize = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nHashSize = sqlite3_value_int(pVal);
}
if( nHashSize<=0 ){
*pbBadkey = 1;
}else{
pConfig->nHashSize = nHashSize;
}
}
else if( 0==sqlite3_stricmp(zKey, "automerge") ){
int nAutomerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nAutomerge = sqlite3_value_int(pVal);
}
if( nAutomerge<0 || nAutomerge>64 ){
*pbBadkey = 1;
}else{
if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
pConfig->nAutomerge = nAutomerge;
}
}
else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
int nCrisisMerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
nCrisisMerge = sqlite3_value_int(pVal);
}
if( nCrisisMerge<0 ){
*pbBadkey = 1;
}else{
if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nCrisisMerge = nCrisisMerge;
}
}
else if( 0==sqlite3_stricmp(zKey, "rank") ){
const char *zIn = (const char*)sqlite3_value_text(pVal);
char *zRank;
char *zRankArgs;
rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
if( rc==SQLITE_OK ){
sqlite3_free(pConfig->zRank);
sqlite3_free(pConfig->zRankArgs);
pConfig->zRank = zRank;
pConfig->zRankArgs = zRankArgs;
}else if( rc==SQLITE_ERROR ){
rc = SQLITE_OK;
*pbBadkey = 1;
}
}else{
*pbBadkey = 1;
}
return rc;
}
/*
** Load the contents of the %_config table into memory.
*/
int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
char *zSql;
sqlite3_stmt *p = 0;
int rc = SQLITE_OK;
int iVersion = 0;
/* Set default values */
pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
if( zSql ){
rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
sqlite3_free(zSql);
}
assert( rc==SQLITE_OK || p==0 );
if( rc==SQLITE_OK ){
while( SQLITE_ROW==sqlite3_step(p) ){
const char *zK = (const char*)sqlite3_column_text(p, 0);
sqlite3_value *pVal = sqlite3_column_value(p, 1);
if( 0==sqlite3_stricmp(zK, "version") ){
iVersion = sqlite3_value_int(pVal);
}else{
int bDummy = 0;
sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
}
}
rc = sqlite3_finalize(p);
}
if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
rc = SQLITE_ERROR;
if( pConfig->pzErrmsg ){
assert( 0==*pConfig->pzErrmsg );
*pConfig->pzErrmsg = sqlite3_mprintf(
"invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
iVersion, FTS5_CURRENT_VERSION
);
}
}
if( rc==SQLITE_OK ){
pConfig->iCookie = iCookie;
}
return rc;
}

2560
ext/fts5/fts5_expr.c Normal file
View File

File diff suppressed because it is too large Load Diff

527
ext/fts5/fts5_hash.c Normal file
View File

@ -0,0 +1,527 @@
/*
** 2014 August 11
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
#include "fts5Int.h"
typedef struct Fts5HashEntry Fts5HashEntry;
/*
** This file contains the implementation of an in-memory hash table used
** to accumuluate "term -> doclist" content before it is flused to a level-0
** segment.
*/
struct Fts5Hash {
int eDetail; /* Copy of Fts5Config.eDetail */
int *pnByte; /* Pointer to bytes counter */
int nEntry; /* Number of entries currently in hash */
int nSlot; /* Size of aSlot[] array */
Fts5HashEntry *pScan; /* Current ordered scan item */
Fts5HashEntry **aSlot; /* Array of hash slots */
};
/*
** Each entry in the hash table is represented by an object of the
** following type. Each object, its key (zKey[]) and its current data
** are stored in a single memory allocation. The position list data
** immediately follows the key data in memory.
**
** The data that follows the key is in a similar, but not identical format
** to the doclist data stored in the database. It is:
**
** * Rowid, as a varint
** * Position list, without 0x00 terminator.
** * Size of previous position list and rowid, as a 4 byte
** big-endian integer.
**
** iRowidOff:
** Offset of last rowid written to data area. Relative to first byte of
** structure.
**
** nData:
** Bytes of data written since iRowidOff.
*/
struct Fts5HashEntry {
Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */
Fts5HashEntry *pScanNext; /* Next entry in sorted order */
int nAlloc; /* Total size of allocation */
int iSzPoslist; /* Offset of space for 4-byte poslist size */
int nData; /* Total bytes of data (incl. structure) */
int nKey; /* Length of zKey[] in bytes */
u8 bDel; /* Set delete-flag @ iSzPoslist */
u8 bContent; /* Set content-flag (detail=none mode) */
i16 iCol; /* Column of last value written */
int iPos; /* Position of last value written */
i64 iRowid; /* Rowid of last value written */
char zKey[8]; /* Nul-terminated entry key */
};
/*
** Size of Fts5HashEntry without the zKey[] array.
*/
#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8)
/*
** Allocate a new hash table.
*/
int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
int rc = SQLITE_OK;
Fts5Hash *pNew;
*ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
if( pNew==0 ){
rc = SQLITE_NOMEM;
}else{
int nByte;
memset(pNew, 0, sizeof(Fts5Hash));
pNew->pnByte = pnByte;
pNew->eDetail = pConfig->eDetail;
pNew->nSlot = 1024;
nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
if( pNew->aSlot==0 ){
sqlite3_free(pNew);
*ppNew = 0;
rc = SQLITE_NOMEM;
}else{
memset(pNew->aSlot, 0, nByte);
}
}
return rc;
}
/*
** Free a hash table object.
*/
void sqlite3Fts5HashFree(Fts5Hash *pHash){
if( pHash ){
sqlite3Fts5HashClear(pHash);
sqlite3_free(pHash->aSlot);
sqlite3_free(pHash);
}
}
/*
** Empty (but do not delete) a hash table.
*/
void sqlite3Fts5HashClear(Fts5Hash *pHash){
int i;
for(i=0; i<pHash->nSlot; i++){
Fts5HashEntry *pNext;
Fts5HashEntry *pSlot;
for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
pNext = pSlot->pHashNext;
sqlite3_free(pSlot);
}
}
memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
pHash->nEntry = 0;
}
static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){
int i;
unsigned int h = 13;
for(i=n-1; i>=0; i--){
h = (h << 3) ^ h ^ p[i];
}
return (h % nSlot);
}
static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){
int i;
unsigned int h = 13;
for(i=n-1; i>=0; i--){
h = (h << 3) ^ h ^ p[i];
}
h = (h << 3) ^ h ^ b;
return (h % nSlot);
}
/*
** Resize the hash table by doubling the number of slots.
*/
static int fts5HashResize(Fts5Hash *pHash){
int nNew = pHash->nSlot*2;
int i;
Fts5HashEntry **apNew;
Fts5HashEntry **apOld = pHash->aSlot;
apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*));
if( !apNew ) return SQLITE_NOMEM;
memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
for(i=0; i<pHash->nSlot; i++){
while( apOld[i] ){
int iHash;
Fts5HashEntry *p = apOld[i];
apOld[i] = p->pHashNext;
iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey));
p->pHashNext = apNew[iHash];
apNew[iHash] = p;
}
}
sqlite3_free(apOld);
pHash->nSlot = nNew;
pHash->aSlot = apNew;
return SQLITE_OK;
}
static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
if( p->iSzPoslist ){
u8 *pPtr = (u8*)p;
if( pHash->eDetail==FTS5_DETAIL_NONE ){
assert( p->nData==p->iSzPoslist );
if( p->bDel ){
pPtr[p->nData++] = 0x00;
if( p->bContent ){
pPtr[p->nData++] = 0x00;
}
}
}else{
int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
int nPos = nSz*2 + p->bDel; /* Value of nPos field */
assert( p->bDel==0 || p->bDel==1 );
if( nPos<=127 ){
pPtr[p->iSzPoslist] = (u8)nPos;
}else{
int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
p->nData += (nByte-1);
}
}
p->iSzPoslist = 0;
p->bDel = 0;
p->bContent = 0;
}
}
/*
** Add an entry to the in-memory hash table. The key is the concatenation
** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
**
** (bByte || pToken) -> (iRowid,iCol,iPos)
**
** Or, if iCol is negative, then the value is a delete marker.
*/
int sqlite3Fts5HashWrite(
Fts5Hash *pHash,
i64 iRowid, /* Rowid for this entry */
int iCol, /* Column token appears in (-ve -> delete) */
int iPos, /* Position of token within column */
char bByte, /* First byte of token */
const char *pToken, int nToken /* Token to add or remove to or from index */
){
unsigned int iHash;
Fts5HashEntry *p;
u8 *pPtr;
int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
int bNew; /* If non-delete entry should be written */
bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
/* Attempt to locate an existing hash entry */
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
if( p->zKey[0]==bByte
&& p->nKey==nToken
&& memcmp(&p->zKey[1], pToken, nToken)==0
){
break;
}
}
/* If an existing hash entry cannot be found, create a new one. */
if( p==0 ){
/* Figure out how much space to allocate */
int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
if( nByte<128 ) nByte = 128;
/* Grow the Fts5Hash.aSlot[] array if necessary. */
if( (pHash->nEntry*2)>=pHash->nSlot ){
int rc = fts5HashResize(pHash);
if( rc!=SQLITE_OK ) return rc;
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
}
/* Allocate new Fts5HashEntry and add it to the hash table. */
p = (Fts5HashEntry*)sqlite3_malloc(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, FTS5_HASHENTRYSIZE);
p->nAlloc = nByte;
p->zKey[0] = bByte;
memcpy(&p->zKey[1], pToken, nToken);
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
p->nKey = nToken;
p->zKey[nToken+1] = '\0';
p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
p->pHashNext = pHash->aSlot[iHash];
pHash->aSlot[iHash] = p;
pHash->nEntry++;
/* Add the first rowid field to the hash-entry */
p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
p->iRowid = iRowid;
p->iSzPoslist = p->nData;
if( pHash->eDetail!=FTS5_DETAIL_NONE ){
p->nData += 1;
p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
}
nIncr += p->nData;
}else{
/* Appending to an existing hash-entry. Check that there is enough
** space to append the largest possible new entry. Worst case scenario
** is:
**
** + 9 bytes for a new rowid,
** + 4 byte reserved for the "poslist size" varint.
** + 1 byte for a "new column" byte,
** + 3 bytes for a new column number (16-bit max) as a varint,
** + 5 bytes for the new position offset (32-bit max).
*/
if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
int nNew = p->nAlloc * 2;
Fts5HashEntry *pNew;
Fts5HashEntry **pp;
pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
if( pNew==0 ) return SQLITE_NOMEM;
pNew->nAlloc = nNew;
for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
*pp = pNew;
p = pNew;
}
nIncr -= p->nData;
}
assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
pPtr = (u8*)p;
/* If this is a new rowid, append the 4-byte size field for the previous
** entry, and the new rowid for this entry. */
if( iRowid!=p->iRowid ){
fts5HashAddPoslistSize(pHash, p);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
p->iRowid = iRowid;
bNew = 1;
p->iSzPoslist = p->nData;
if( pHash->eDetail!=FTS5_DETAIL_NONE ){
p->nData += 1;
p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
p->iPos = 0;
}
}
if( iCol>=0 ){
if( pHash->eDetail==FTS5_DETAIL_NONE ){
p->bContent = 1;
}else{
/* Append a new column value, if necessary */
assert( iCol>=p->iCol );
if( iCol!=p->iCol ){
if( pHash->eDetail==FTS5_DETAIL_FULL ){
pPtr[p->nData++] = 0x01;
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
p->iCol = iCol;
p->iPos = 0;
}else{
bNew = 1;
p->iCol = iPos = iCol;
}
}
/* Append the new position offset, if necessary */
if( bNew ){
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
p->iPos = iPos;
}
}
}else{
/* This is a delete. Set the delete flag. */
p->bDel = 1;
}
nIncr += p->nData;
*pHash->pnByte += nIncr;
return SQLITE_OK;
}
/*
** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
** each sorted in key order. This function merges the two lists into a
** single list and returns a pointer to its first element.
*/
static Fts5HashEntry *fts5HashEntryMerge(
Fts5HashEntry *pLeft,
Fts5HashEntry *pRight
){
Fts5HashEntry *p1 = pLeft;
Fts5HashEntry *p2 = pRight;
Fts5HashEntry *pRet = 0;
Fts5HashEntry **ppOut = &pRet;
while( p1 || p2 ){
if( p1==0 ){
*ppOut = p2;
p2 = 0;
}else if( p2==0 ){
*ppOut = p1;
p1 = 0;
}else{
int i = 0;
while( p1->zKey[i]==p2->zKey[i] ) i++;
if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
/* p2 is smaller */
*ppOut = p2;
ppOut = &p2->pScanNext;
p2 = p2->pScanNext;
}else{
/* p1 is smaller */
*ppOut = p1;
ppOut = &p1->pScanNext;
p1 = p1->pScanNext;
}
*ppOut = 0;
}
}
return pRet;
}
/*
** Extract all tokens from hash table iHash and link them into a list
** in sorted order. The hash table is cleared before returning. It is
** the responsibility of the caller to free the elements of the returned
** list.
*/
static int fts5HashEntrySort(
Fts5Hash *pHash,
const char *pTerm, int nTerm, /* Query prefix, if any */
Fts5HashEntry **ppSorted
){
const int nMergeSlot = 32;
Fts5HashEntry **ap;
Fts5HashEntry *pList;
int iSlot;
int i;
*ppSorted = 0;
ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot);
if( !ap ) return SQLITE_NOMEM;
memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
Fts5HashEntry *pIter;
for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
Fts5HashEntry *pEntry = pIter;
pEntry->pScanNext = 0;
for(i=0; ap[i]; i++){
pEntry = fts5HashEntryMerge(pEntry, ap[i]);
ap[i] = 0;
}
ap[i] = pEntry;
}
}
}
pList = 0;
for(i=0; i<nMergeSlot; i++){
pList = fts5HashEntryMerge(pList, ap[i]);
}
pHash->nEntry = 0;
sqlite3_free(ap);
*ppSorted = pList;
return SQLITE_OK;
}
/*
** Query the hash table for a doclist associated with term pTerm/nTerm.
*/
int sqlite3Fts5HashQuery(
Fts5Hash *pHash, /* Hash table to query */
const char *pTerm, int nTerm, /* Query term */
const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */
int *pnDoclist /* OUT: Size of doclist in bytes */
){
unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
Fts5HashEntry *p;
for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
}
if( p ){
fts5HashAddPoslistSize(pHash, p);
*ppDoclist = (const u8*)&p->zKey[nTerm+1];
*pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
}else{
*ppDoclist = 0;
*pnDoclist = 0;
}
return SQLITE_OK;
}
int sqlite3Fts5HashScanInit(
Fts5Hash *p, /* Hash table to query */
const char *pTerm, int nTerm /* Query prefix */
){
return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
}
void sqlite3Fts5HashScanNext(Fts5Hash *p){
assert( !sqlite3Fts5HashScanEof(p) );
p->pScan = p->pScan->pScanNext;
}
int sqlite3Fts5HashScanEof(Fts5Hash *p){
return (p->pScan==0);
}
void sqlite3Fts5HashScanEntry(
Fts5Hash *pHash,
const char **pzTerm, /* OUT: term (nul-terminated) */
const u8 **ppDoclist, /* OUT: pointer to doclist */
int *pnDoclist /* OUT: size of doclist in bytes */
){
Fts5HashEntry *p;
if( (p = pHash->pScan) ){
int nTerm = (int)strlen(p->zKey);
fts5HashAddPoslistSize(pHash, p);
*pzTerm = p->zKey;
*ppDoclist = (const u8*)&p->zKey[nTerm+1];
*pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
}else{
*pzTerm = 0;
*ppDoclist = 0;
*pnDoclist = 0;
}
}

6281
ext/fts5/fts5_index.c Normal file
View File

File diff suppressed because it is too large Load Diff

2710
ext/fts5/fts5_main.c Normal file
View File

File diff suppressed because it is too large Load Diff

1125
ext/fts5/fts5_storage.c Normal file
View File

File diff suppressed because it is too large Load Diff

1150
ext/fts5/fts5_tcl.c Normal file
View File

File diff suppressed because it is too large Load Diff

424
ext/fts5/fts5_test_mi.c Normal file
View File

@ -0,0 +1,424 @@
/*
** 2015 Aug 04
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains test code only, it is not included in release
** versions of FTS5. It contains the implementation of an FTS5 auxiliary
** function very similar to the FTS4 function matchinfo():
**
** https://www.sqlite.org/fts3.html#matchinfo
**
** Known differences are that:
**
** 1) this function uses the FTS5 definition of "matchable phrase", which
** excludes any phrases that are part of an expression sub-tree that
** does not match the current row. This comes up for MATCH queries
** such as:
**
** "a OR (b AND c)"
**
** In FTS4, if a single row contains instances of tokens "a" and "c",
** but not "b", all instances of "c" are considered matches. In FTS5,
** they are not (as the "b AND c" sub-tree does not match the current
** row.
**
** 2) For the values returned by 'x' that apply to all rows of the table,
** NEAR constraints are not considered. But for the number of hits in
** the current row, they are.
**
** This file exports a single function that may be called to register the
** matchinfo() implementation with a database handle:
**
** int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
*/
#ifdef SQLITE_TEST
#ifdef SQLITE_ENABLE_FTS5
#include "fts5.h"
#include <tcl.h>
#include <assert.h>
#include <string.h>
typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
typedef unsigned int u32;
struct Fts5MatchinfoCtx {
int nCol; /* Number of cols in FTS5 table */
int nPhrase; /* Number of phrases in FTS5 query */
char *zArg; /* nul-term'd copy of 2nd arg */
int nRet; /* Number of elements in aRet[] */
u32 *aRet; /* Array of 32-bit unsigned ints to return */
};
/*
** Return a pointer to the fts5_api pointer for database connection db.
** If an error occurs, return NULL and leave an error in the database
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
static fts5_api *fts5_api_from_db(sqlite3 *db){
fts5_api *pRet = 0;
sqlite3_stmt *pStmt = 0;
if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
&& SQLITE_ROW==sqlite3_step(pStmt)
&& sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
){
memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
}
sqlite3_finalize(pStmt);
return pRet;
}
/*
** Argument f should be a flag accepted by matchinfo() (a valid character
** in the string passed as the second argument). If it is not, -1 is
** returned. Otherwise, if f is a valid matchinfo flag, the value returned
** is the number of 32-bit integers added to the output array if the
** table has nCol columns and the query nPhrase phrases.
*/
static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
int ret = -1;
switch( f ){
case 'p': ret = 1; break;
case 'c': ret = 1; break;
case 'x': ret = 3 * nCol * nPhrase; break;
case 'y': ret = nCol * nPhrase; break;
case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
case 'n': ret = 1; break;
case 'a': ret = nCol; break;
case 'l': ret = nCol; break;
case 's': ret = nCol; break;
}
return ret;
}
static int fts5MatchinfoIter(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
Fts5MatchinfoCtx *p,
int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
){
int i;
int n = 0;
int rc = SQLITE_OK;
char f;
for(i=0; (f = p->zArg[i]); i++){
rc = x(pApi, pFts, p, f, &p->aRet[n]);
if( rc!=SQLITE_OK ) break;
n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
}
return rc;
}
static int fts5MatchinfoXCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
void *pUserData
){
Fts5PhraseIter iter;
int iCol, iOff;
u32 *aOut = (u32*)pUserData;
int iPrev = -1;
for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
iCol>=0;
pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
){
aOut[iCol*3+1]++;
if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
iPrev = iCol;
}
return SQLITE_OK;
}
static int fts5MatchinfoGlobalCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
Fts5MatchinfoCtx *p,
char f,
u32 *aOut
){
int rc = SQLITE_OK;
switch( f ){
case 'p':
aOut[0] = p->nPhrase;
break;
case 'c':
aOut[0] = p->nCol;
break;
case 'x': {
int i;
for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
void *pPtr = (void*)&aOut[i * p->nCol * 3];
rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
}
break;
}
case 'n': {
sqlite3_int64 nRow;
rc = pApi->xRowCount(pFts, &nRow);
aOut[0] = (u32)nRow;
break;
}
case 'a': {
sqlite3_int64 nRow = 0;
rc = pApi->xRowCount(pFts, &nRow);
if( nRow==0 ){
memset(aOut, 0, sizeof(u32) * p->nCol);
}else{
int i;
for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
sqlite3_int64 nToken;
rc = pApi->xColumnTotalSize(pFts, i, &nToken);
if( rc==SQLITE_OK){
aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
}
}
}
break;
}
}
return rc;
}
static int fts5MatchinfoLocalCb(
const Fts5ExtensionApi *pApi,
Fts5Context *pFts,
Fts5MatchinfoCtx *p,
char f,
u32 *aOut
){
int i;
int rc = SQLITE_OK;
switch( f ){
case 'b': {
int iPhrase;
int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
for(i=0; i<nInt; i++) aOut[i] = 0;
for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
Fts5PhraseIter iter;
int iCol;
for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
iCol>=0;
pApi->xPhraseNextColumn(pFts, &iter, &iCol)
){
aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
}
}
break;
}
case 'x':
case 'y': {
int nMul = (f=='x' ? 3 : 1);
int iPhrase;
for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
Fts5PhraseIter iter;
int iOff, iCol;
for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
iOff>=0;
pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
){
if( f=='b' ){
aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
}else{
aOut[nMul * (iCol + iPhrase * p->nCol)]++;
}
}
}
break;
}
case 'l': {
for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
int nToken;
rc = pApi->xColumnSize(pFts, i, &nToken);
aOut[i] = (u32)nToken;
}
break;
}
case 's': {
int nInst;
memset(aOut, 0, sizeof(u32) * p->nCol);
rc = pApi->xInstCount(pFts, &nInst);
for(i=0; rc==SQLITE_OK && i<nInst; i++){
int iPhrase, iOff, iCol = 0;
int iNextPhrase;
int iNextOff;
u32 nSeq = 1;
int j;
rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
iNextPhrase = iPhrase+1;
iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
int ip, ic, io;
rc = pApi->xInst(pFts, j, &ip, &ic, &io);
if( ic!=iCol || io>iNextOff ) break;
if( ip==iNextPhrase && io==iNextOff ){
nSeq++;
iNextPhrase = ip+1;
iNextOff = io + pApi->xPhraseSize(pFts, ip);
}
}
if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
}
break;
}
}
return rc;
}
static Fts5MatchinfoCtx *fts5MatchinfoNew(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning error message */
const char *zArg /* Matchinfo flag string */
){
Fts5MatchinfoCtx *p;
int nCol;
int nPhrase;
int i;
int nInt;
int nByte;
int rc;
nCol = pApi->xColumnCount(pFts);
nPhrase = pApi->xPhraseCount(pFts);
nInt = 0;
for(i=0; zArg[i]; i++){
int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
if( n<0 ){
char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
sqlite3_result_error(pCtx, zErr, -1);
sqlite3_free(zErr);
return 0;
}
nInt += n;
}
nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */
+ sizeof(u32) * nInt /* The p->aRet[] array */
+ (i+1); /* The p->zArg string */
p = (Fts5MatchinfoCtx*)sqlite3_malloc(nByte);
if( p==0 ){
sqlite3_result_error_nomem(pCtx);
return 0;
}
memset(p, 0, nByte);
p->nCol = nCol;
p->nPhrase = nPhrase;
p->aRet = (u32*)&p[1];
p->nRet = nInt;
p->zArg = (char*)&p->aRet[nInt];
memcpy(p->zArg, zArg, i);
rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
sqlite3_free(p);
p = 0;
}
return p;
}
static void fts5MatchinfoFunc(
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
Fts5Context *pFts, /* First arg to pass to pApi functions */
sqlite3_context *pCtx, /* Context for returning result/error */
int nVal, /* Number of values in apVal[] array */
sqlite3_value **apVal /* Array of trailing arguments */
){
const char *zArg;
Fts5MatchinfoCtx *p;
int rc = SQLITE_OK;
if( nVal>0 ){
zArg = (const char*)sqlite3_value_text(apVal[0]);
}else{
zArg = "pcx";
}
p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
if( p==0 ){
rc = SQLITE_NOMEM;
}else{
rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
}
}
if( rc==SQLITE_OK ){
rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
}
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}else{
/* No errors has occured, so return a copy of the array of integers. */
int nByte = p->nRet * sizeof(u32);
sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
}
}
int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
int rc; /* Return code */
fts5_api *pApi; /* FTS5 API functions */
/* Extract the FTS5 API pointer from the database handle. The
** fts5_api_from_db() function above is copied verbatim from the
** FTS5 documentation. Refer there for details. */
pApi = fts5_api_from_db(db);
/* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
** with this database handle, or an error (OOM perhaps?) has occurred.
**
** Also check that the fts5_api object is version 2 or newer.
*/
if( pApi==0 || pApi->iVersion<1 ){
return SQLITE_ERROR;
}
/* Register the implementation of matchinfo() */
rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
return rc;
}
#endif /* SQLITE_ENABLE_FTS5 */
#endif /* SQLITE_TEST */

482
ext/fts5/fts5_test_tok.c Normal file
View File

@ -0,0 +1,482 @@
/*
** 2013 Apr 22
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains code for the "fts5tokenize" virtual table module.
** An fts5tokenize virtual table is created as follows:
**
** CREATE VIRTUAL TABLE <tbl> USING fts5tokenize(
** <tokenizer-name>, <arg-1>, ...
** );
**
** The table created has the following schema:
**
** CREATE TABLE <tbl>(input HIDDEN, token, start, end, position)
**
** When queried, the query must include a WHERE clause of type:
**
** input = <string>
**
** The virtual table module tokenizes this <string>, using the FTS3
** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE
** statement and returns one row for each token in the result. With
** fields set as follows:
**
** input: Always set to a copy of <string>
** token: A token from the input.
** start: Byte offset of the token within the input <string>.
** end: Byte offset of the byte immediately following the end of the
** token within the input string.
** pos: Token offset of token within input.
**
*/
#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5)
#include <fts5.h>
#include <string.h>
#include <assert.h>
typedef struct Fts5tokTable Fts5tokTable;
typedef struct Fts5tokCursor Fts5tokCursor;
typedef struct Fts5tokRow Fts5tokRow;
/*
** Virtual table structure.
*/
struct Fts5tokTable {
sqlite3_vtab base; /* Base class used by SQLite core */
fts5_tokenizer tok; /* Tokenizer functions */
Fts5Tokenizer *pTok; /* Tokenizer instance */
};
/*
** A container for a rows values.
*/
struct Fts5tokRow {
char *zToken;
int iStart;
int iEnd;
int iPos;
};
/*
** Virtual table cursor structure.
*/
struct Fts5tokCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
int iRowid; /* Current 'rowid' value */
char *zInput; /* Input string */
int nRow; /* Number of entries in aRow[] */
Fts5tokRow *aRow; /* Array of rows to return */
};
static void fts5tokDequote(char *z){
char q = z[0];
if( q=='[' || q=='\'' || q=='"' || q=='`' ){
int iIn = 1;
int iOut = 0;
if( q=='[' ) q = ']';
while( z[iIn] ){
if( z[iIn]==q ){
if( z[iIn+1]!=q ){
/* Character iIn was the close quote. */
iIn++;
break;
}else{
/* Character iIn and iIn+1 form an escaped quote character. Skip
** the input cursor past both and copy a single quote character
** to the output buffer. */
iIn += 2;
z[iOut++] = q;
}
}else{
z[iOut++] = z[iIn++];
}
}
z[iOut] = '\0';
}
}
/*
** The second argument, argv[], is an array of pointers to nul-terminated
** strings. This function makes a copy of the array and strings into a
** single block of memory. It then dequotes any of the strings that appear
** to be quoted.
**
** If successful, output parameter *pazDequote is set to point at the
** array of dequoted strings and SQLITE_OK is returned. The caller is
** responsible for eventually calling sqlite3_free() to free the array
** in this case. Or, if an error occurs, an SQLite error code is returned.
** The final value of *pazDequote is undefined in this case.
*/
static int fts5tokDequoteArray(
int argc, /* Number of elements in argv[] */
const char * const *argv, /* Input array */
char ***pazDequote /* Output array */
){
int rc = SQLITE_OK; /* Return code */
if( argc==0 ){
*pazDequote = 0;
}else{
int i;
int nByte = 0;
char **azDequote;
for(i=0; i<argc; i++){
nByte += (int)(strlen(argv[i]) + 1);
}
*pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
if( azDequote==0 ){
rc = SQLITE_NOMEM;
}else{
char *pSpace = (char *)&azDequote[argc];
for(i=0; i<argc; i++){
int n = (int)strlen(argv[i]);
azDequote[i] = pSpace;
memcpy(pSpace, argv[i], n+1);
fts5tokDequote(pSpace);
pSpace += (n+1);
}
}
}
return rc;
}
/*
** Schema of the tokenizer table.
*/
#define FTS3_TOK_SCHEMA "CREATE TABLE x(input HIDDEN, token, start, end, position)"
/*
** This function does all the work for both the xConnect and xCreate methods.
** These tables have no persistent representation of their own, so xConnect
** and xCreate are identical operations.
**
** argv[0]: module name
** argv[1]: database name
** argv[2]: table name
** argv[3]: first argument (tokenizer name)
*/
static int fts5tokConnectMethod(
sqlite3 *db, /* Database connection */
void *pCtx, /* Pointer to fts5_api object */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
fts5_api *pApi = (fts5_api*)pCtx;
Fts5tokTable *pTab = 0;
int rc;
char **azDequote = 0;
int nDequote;
rc = sqlite3_declare_vtab(db,
"CREATE TABLE x(input HIDDEN, token, start, end, position)"
);
if( rc==SQLITE_OK ){
nDequote = argc-3;
rc = fts5tokDequoteArray(nDequote, &argv[3], &azDequote);
}
if( rc==SQLITE_OK ){
pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable));
if( pTab==0 ){
rc = SQLITE_NOMEM;
}else{
memset(pTab, 0, sizeof(Fts5tokTable));
}
}
if( rc==SQLITE_OK ){
void *pTokCtx = 0;
const char *zModule = 0;
if( nDequote>0 ){
zModule = azDequote[0];
}
rc = pApi->xFindTokenizer(pApi, zModule, &pTokCtx, &pTab->tok);
if( rc==SQLITE_OK ){
const char **azArg = (const char **)&azDequote[1];
int nArg = nDequote>0 ? nDequote-1 : 0;
rc = pTab->tok.xCreate(pTokCtx, azArg, nArg, &pTab->pTok);
}
}
if( rc!=SQLITE_OK ){
sqlite3_free(pTab);
pTab = 0;
}
*ppVtab = (sqlite3_vtab*)pTab;
sqlite3_free(azDequote);
return rc;
}
/*
** This function does the work for both the xDisconnect and xDestroy methods.
** These tables have no persistent representation of their own, so xDisconnect
** and xDestroy are identical operations.
*/
static int fts5tokDisconnectMethod(sqlite3_vtab *pVtab){
Fts5tokTable *pTab = (Fts5tokTable *)pVtab;
if( pTab->pTok ){
pTab->tok.xDelete(pTab->pTok);
}
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** xBestIndex - Analyze a WHERE and ORDER BY clause.
*/
static int fts5tokBestIndexMethod(
sqlite3_vtab *pVTab,
sqlite3_index_info *pInfo
){
int i;
for(i=0; i<pInfo->nConstraint; i++){
if( pInfo->aConstraint[i].usable
&& pInfo->aConstraint[i].iColumn==0
&& pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ
){
pInfo->idxNum = 1;
pInfo->aConstraintUsage[i].argvIndex = 1;
pInfo->aConstraintUsage[i].omit = 1;
pInfo->estimatedCost = 1;
return SQLITE_OK;
}
}
pInfo->idxNum = 0;
assert( pInfo->estimatedCost>1000000.0 );
return SQLITE_OK;
}
/*
** xOpen - Open a cursor.
*/
static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts5tokCursor *pCsr;
pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor));
if( pCsr==0 ){
return SQLITE_NOMEM;
}
memset(pCsr, 0, sizeof(Fts5tokCursor));
*ppCsr = (sqlite3_vtab_cursor *)pCsr;
return SQLITE_OK;
}
/*
** Reset the tokenizer cursor passed as the only argument. As if it had
** just been returned by fts5tokOpenMethod().
*/
static void fts5tokResetCursor(Fts5tokCursor *pCsr){
int i;
for(i=0; i<pCsr->nRow; i++){
sqlite3_free(pCsr->aRow[i].zToken);
}
sqlite3_free(pCsr->zInput);
sqlite3_free(pCsr->aRow);
pCsr->zInput = 0;
pCsr->aRow = 0;
pCsr->nRow = 0;
pCsr->iRowid = 0;
}
/*
** xClose - Close a cursor.
*/
static int fts5tokCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
fts5tokResetCursor(pCsr);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** xNext - Advance the cursor to the next row, if any.
*/
static int fts5tokNextMethod(sqlite3_vtab_cursor *pCursor){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
pCsr->iRowid++;
return SQLITE_OK;
}
static int fts5tokCb(
void *pCtx, /* Pointer to Fts5tokCursor */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
){
Fts5tokCursor *pCsr = (Fts5tokCursor*)pCtx;
Fts5tokRow *pRow;
if( (pCsr->nRow & (pCsr->nRow-1))==0 ){
int nNew = pCsr->nRow ? pCsr->nRow*2 : 32;
Fts5tokRow *aNew;
aNew = (Fts5tokRow*)sqlite3_realloc(pCsr->aRow, nNew*sizeof(Fts5tokRow));
if( aNew==0 ) return SQLITE_NOMEM;
memset(&aNew[pCsr->nRow], 0, sizeof(Fts5tokRow)*(nNew-pCsr->nRow));
pCsr->aRow = aNew;
}
pRow = &pCsr->aRow[pCsr->nRow];
pRow->iStart = iStart;
pRow->iEnd = iEnd;
if( pCsr->nRow ){
pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1);
}
pRow->zToken = sqlite3_malloc(nToken+1);
if( pRow->zToken==0 ) return SQLITE_NOMEM;
memcpy(pRow->zToken, pToken, nToken);
pRow->zToken[nToken] = 0;
pCsr->nRow++;
return SQLITE_OK;
}
/*
** xFilter - Initialize a cursor to point at the start of its data.
*/
static int fts5tokFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *idxStr, /* Unused */
int nVal, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
int rc = SQLITE_ERROR;
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
Fts5tokTable *pTab = (Fts5tokTable *)(pCursor->pVtab);
fts5tokResetCursor(pCsr);
if( idxNum==1 ){
const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
int nByte = sqlite3_value_bytes(apVal[0]);
pCsr->zInput = sqlite3_malloc(nByte+1);
if( pCsr->zInput==0 ){
rc = SQLITE_NOMEM;
}else{
memcpy(pCsr->zInput, zByte, nByte);
pCsr->zInput[nByte] = 0;
rc = pTab->tok.xTokenize(
pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb
);
}
}
if( rc!=SQLITE_OK ) return rc;
return fts5tokNextMethod(pCursor);
}
/*
** xEof - Return true if the cursor is at EOF, or false otherwise.
*/
static int fts5tokEofMethod(sqlite3_vtab_cursor *pCursor){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
return (pCsr->iRowid>pCsr->nRow);
}
/*
** xColumn - Return a column value.
*/
static int fts5tokColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
Fts5tokRow *pRow = &pCsr->aRow[pCsr->iRowid-1];
/* CREATE TABLE x(input, token, start, end, position) */
switch( iCol ){
case 0:
sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT);
break;
case 1:
sqlite3_result_text(pCtx, pRow->zToken, -1, SQLITE_TRANSIENT);
break;
case 2:
sqlite3_result_int(pCtx, pRow->iStart);
break;
case 3:
sqlite3_result_int(pCtx, pRow->iEnd);
break;
default:
assert( iCol==4 );
sqlite3_result_int(pCtx, pRow->iPos);
break;
}
return SQLITE_OK;
}
/*
** xRowid - Return the current rowid for the cursor.
*/
static int fts5tokRowidMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite_int64 *pRowid /* OUT: Rowid value */
){
Fts5tokCursor *pCsr = (Fts5tokCursor *)pCursor;
*pRowid = (sqlite3_int64)pCsr->iRowid;
return SQLITE_OK;
}
/*
** Register the fts5tok module with database connection db. Return SQLITE_OK
** if successful or an error code if sqlite3_create_module() fails.
*/
int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
static const sqlite3_module fts5tok_module = {
0, /* iVersion */
fts5tokConnectMethod, /* xCreate */
fts5tokConnectMethod, /* xConnect */
fts5tokBestIndexMethod, /* xBestIndex */
fts5tokDisconnectMethod, /* xDisconnect */
fts5tokDisconnectMethod, /* xDestroy */
fts5tokOpenMethod, /* xOpen */
fts5tokCloseMethod, /* xClose */
fts5tokFilterMethod, /* xFilter */
fts5tokNextMethod, /* xNext */
fts5tokEofMethod, /* xEof */
fts5tokColumnMethod, /* xColumn */
fts5tokRowidMethod, /* xRowid */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindFunction */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
0 /* xRollbackTo */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
return rc;
}
#endif /* defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) */

1242
ext/fts5/fts5_tokenize.c Normal file
View File

File diff suppressed because it is too large Load Diff

360
ext/fts5/fts5_unicode2.c Normal file
View File

@ -0,0 +1,360 @@
/*
** 2012 May 25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
*/
/*
** DO NOT EDIT THIS MACHINE GENERATED FILE.
*/
#include <assert.h>
/*
** Return true if the argument corresponds to a unicode codepoint
** classified as either a letter or a number. Otherwise false.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3Fts5UnicodeIsalnum(int c){
/* Each unsigned integer in the following array corresponds to a contiguous
** range of unicode codepoints that are not either letters or numbers (i.e.
** codepoints for which this function should return 0).
**
** The most significant 22 bits in each 32-bit value contain the first
** codepoint in the range. The least significant 10 bits are used to store
** the size of the range (always at least 1). In other words, the value
** ((C<<22) + N) represents a range of N codepoints starting with codepoint
** C. It is not possible to represent a range larger than 1023 codepoints
** using this format.
*/
static const unsigned int aEntry[] = {
0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
0x380400F0,
};
static const unsigned int aAscii[4] = {
0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
};
if( (unsigned int)c<128 ){
return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
}else if( (unsigned int)c<(1<<22) ){
unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
int iRes = 0;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( key >= aEntry[iTest] ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( aEntry[0]<key );
assert( key>=aEntry[iRes] );
return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
}
return 1;
}
/*
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
** of the ASCII letter only. For example, if passed 235 - "LATIN
** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
static int fts5_remove_diacritic(int c){
unsigned short aDia[] = {
0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928,
3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234,
4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504,
6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529,
61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
62924, 63050, 63082, 63274, 63390,
};
char aChar[] = {
'\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c',
'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r',
's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o',
'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r',
'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0',
'\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h',
'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't',
'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a',
'e', 'i', 'o', 'u', 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
int iRes = 0;
int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
int iLo = 0;
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
if( key >= aDia[iTest] ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( key>=aDia[iRes] );
return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
}
/*
** Return true if the argument interpreted as a unicode codepoint
** is a diacritical modifier character.
*/
int sqlite3Fts5UnicodeIsdiacritic(int c){
unsigned int mask0 = 0x08029FDF;
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
(mask0 & (1 << (c-768))) :
(mask1 & (1 << (c-768-32)));
}
/*
** Interpret the argument as a unicode codepoint. If the codepoint
** is an upper case character that has a lower case equivalent,
** return the codepoint corresponding to the lower case version.
** Otherwise, return a copy of the argument.
**
** The results are undefined if the value passed to this function
** is less than zero.
*/
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** codepoints starting at codepoint iCode.
**
** If the least significant bit in flags is clear, then the rule applies
** to all nRange codepoints (i.e. all nRange codepoints are upper case and
** need to be folded). Or, if it is set, then the rule only applies to
** every second codepoint in the range, starting with codepoint C.
**
** The 7 most significant bits in flags are an index into the aiOff[]
** array. If a specific codepoint C does require folding, then its lower
** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
**
** The contents of this array are generated by parsing the CaseFolding.txt
** file distributed as part of the "Unicode Character Database". See
** http://www.unicode.org for details.
*/
static const struct TableEntry {
unsigned short iCode;
unsigned char flags;
unsigned char nRange;
} aEntry[] = {
{65, 14, 26}, {181, 64, 1}, {192, 14, 23},
{216, 14, 7}, {256, 1, 48}, {306, 1, 6},
{313, 1, 16}, {330, 1, 46}, {376, 116, 1},
{377, 1, 6}, {383, 104, 1}, {385, 50, 1},
{386, 1, 4}, {390, 44, 1}, {391, 0, 1},
{393, 42, 2}, {395, 0, 1}, {398, 32, 1},
{399, 38, 1}, {400, 40, 1}, {401, 0, 1},
{403, 42, 1}, {404, 46, 1}, {406, 52, 1},
{407, 48, 1}, {408, 0, 1}, {412, 52, 1},
{413, 54, 1}, {415, 56, 1}, {416, 1, 6},
{422, 60, 1}, {423, 0, 1}, {425, 60, 1},
{428, 0, 1}, {430, 60, 1}, {431, 0, 1},
{433, 58, 2}, {435, 1, 4}, {439, 62, 1},
{440, 0, 1}, {444, 0, 1}, {452, 2, 1},
{453, 0, 1}, {455, 2, 1}, {456, 0, 1},
{458, 2, 1}, {459, 1, 18}, {478, 1, 18},
{497, 2, 1}, {498, 1, 4}, {502, 122, 1},
{503, 134, 1}, {504, 1, 40}, {544, 110, 1},
{546, 1, 18}, {570, 70, 1}, {571, 0, 1},
{573, 108, 1}, {574, 68, 1}, {577, 0, 1},
{579, 106, 1}, {580, 28, 1}, {581, 30, 1},
{582, 1, 10}, {837, 36, 1}, {880, 1, 4},
{886, 0, 1}, {902, 18, 1}, {904, 16, 3},
{908, 26, 1}, {910, 24, 2}, {913, 14, 17},
{931, 14, 9}, {962, 0, 1}, {975, 4, 1},
{976, 140, 1}, {977, 142, 1}, {981, 146, 1},
{982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
{1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
{1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
{1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
{1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
{1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
{4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
{7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
{7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
{7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
{8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
{8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
{8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
{8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
{8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
{8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
{8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
{8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
{8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
{11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
{11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
{11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
{11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
{11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
{42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
{42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
{42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
{42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
{65313, 14, 26},
};
static const unsigned short aiOff[] = {
1, 2, 8, 15, 16, 26, 28, 32,
37, 38, 40, 48, 63, 64, 69, 71,
79, 80, 116, 202, 203, 205, 206, 207,
209, 210, 211, 213, 214, 217, 218, 219,
775, 7264, 10792, 10795, 23228, 23256, 30204, 54721,
54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
65514, 65521, 65527, 65528, 65529,
};
int ret = c;
assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
if( c<128 ){
if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
}else if( c<65536 ){
const struct TableEntry *p;
int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
int iLo = 0;
int iRes = -1;
assert( c>aEntry[0].iCode );
while( iHi>=iLo ){
int iTest = (iHi + iLo) / 2;
int cmp = (c - aEntry[iTest].iCode);
if( cmp>=0 ){
iRes = iTest;
iLo = iTest+1;
}else{
iHi = iTest-1;
}
}
assert( iRes>=0 && c>=aEntry[iRes].iCode );
p = &aEntry[iRes];
if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
assert( ret>0 );
}
if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret);
}
else if( c>=66560 && c<66600 ){
ret = c + 40;
}
return ret;
}

345
ext/fts5/fts5_varint.c Normal file
View File

@ -0,0 +1,345 @@
/*
** 2015 May 30
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** Routines for varint serialization and deserialization.
*/
#include "fts5Int.h"
/*
** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
** Except, this version does handle the single byte case that the core
** version depends on being handled before its function is called.
*/
int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
u32 a,b;
/* The 1-byte case. Overwhelmingly the most common. */
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
/* Values between 0 and 127 */
*v = a;
return 1;
}
/* The 2-byte case */
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
/* Values between 128 and 16383 */
a &= 0x7f;
a = a<<7;
*v = a | b;
return 2;
}
/* The 3-byte case */
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
/* Values between 16384 and 2097151 */
a &= (0x7f<<14)|(0x7f);
b &= 0x7f;
b = b<<7;
*v = a | b;
return 3;
}
/* A 32-bit varint is used to store size information in btrees.
** Objects are rarely larger than 2MiB limit of a 3-byte varint.
** A 3-byte varint is sufficient, for example, to record the size
** of a 1048569-byte BLOB or string.
**
** We only unroll the first 1-, 2-, and 3- byte cases. The very
** rare larger cases can be handled by the slower 64-bit varint
** routine.
*/
{
u64 v64;
u8 n;
p -= 2;
n = sqlite3Fts5GetVarint(p, &v64);
*v = (u32)v64;
assert( n>3 && n<=9 );
return n;
}
}
/*
** Bitmasks used by sqlite3GetVarint(). These precomputed constants
** are defined here rather than simply putting the constant expressions
** inline in order to work around bugs in the RVT compiler.
**
** SLOT_2_0 A mask for (0x7f<<14) | 0x7f
**
** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0
*/
#define SLOT_2_0 0x001fc07f
#define SLOT_4_2_0 0xf01fc07f
/*
** Read a 64-bit variable-length integer from memory starting at p[0].
** Return the number of bytes read. The value is stored in *v.
*/
u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){
u32 a,b,s;
a = *p;
/* a: p0 (unmasked) */
if (!(a&0x80))
{
*v = a;
return 1;
}
p++;
b = *p;
/* b: p1 (unmasked) */
if (!(b&0x80))
{
a &= 0x7f;
a = a<<7;
a |= b;
*v = a;
return 2;
}
/* Verify that constants are precomputed correctly */
assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) );
assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) );
p++;
a = a<<14;
a |= *p;
/* a: p0<<14 | p2 (unmasked) */
if (!(a&0x80))
{
a &= SLOT_2_0;
b &= 0x7f;
b = b<<7;
a |= b;
*v = a;
return 3;
}
/* CSE1 from below */
a &= SLOT_2_0;
p++;
b = b<<14;
b |= *p;
/* b: p1<<14 | p3 (unmasked) */
if (!(b&0x80))
{
b &= SLOT_2_0;
/* moved CSE1 up */
/* a &= (0x7f<<14)|(0x7f); */
a = a<<7;
a |= b;
*v = a;
return 4;
}
/* a: p0<<14 | p2 (masked) */
/* b: p1<<14 | p3 (unmasked) */
/* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
/* moved CSE1 up */
/* a &= (0x7f<<14)|(0x7f); */
b &= SLOT_2_0;
s = a;
/* s: p0<<14 | p2 (masked) */
p++;
a = a<<14;
a |= *p;
/* a: p0<<28 | p2<<14 | p4 (unmasked) */
if (!(a&0x80))
{
/* we can skip these cause they were (effectively) done above in calc'ing s */
/* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
/* b &= (0x7f<<14)|(0x7f); */
b = b<<7;
a |= b;
s = s>>18;
*v = ((u64)s)<<32 | a;
return 5;
}
/* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
s = s<<7;
s |= b;
/* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
p++;
b = b<<14;
b |= *p;
/* b: p1<<28 | p3<<14 | p5 (unmasked) */
if (!(b&0x80))
{
/* we can skip this cause it was (effectively) done above in calc'ing s */
/* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
a &= SLOT_2_0;
a = a<<7;
a |= b;
s = s>>18;
*v = ((u64)s)<<32 | a;
return 6;
}
p++;
a = a<<14;
a |= *p;
/* a: p2<<28 | p4<<14 | p6 (unmasked) */
if (!(a&0x80))
{
a &= SLOT_4_2_0;
b &= SLOT_2_0;
b = b<<7;
a |= b;
s = s>>11;
*v = ((u64)s)<<32 | a;
return 7;
}
/* CSE2 from below */
a &= SLOT_2_0;
p++;
b = b<<14;
b |= *p;
/* b: p3<<28 | p5<<14 | p7 (unmasked) */
if (!(b&0x80))
{
b &= SLOT_4_2_0;
/* moved CSE2 up */
/* a &= (0x7f<<14)|(0x7f); */
a = a<<7;
a |= b;
s = s>>4;
*v = ((u64)s)<<32 | a;
return 8;
}
p++;
a = a<<15;
a |= *p;
/* a: p4<<29 | p6<<15 | p8 (unmasked) */
/* moved CSE2 up */
/* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */
b &= SLOT_2_0;
b = b<<8;
a |= b;
s = s<<4;
b = p[-4];
b &= 0x7f;
b = b>>3;
s |= b;
*v = ((u64)s)<<32 | a;
return 9;
}
/*
** The variable-length integer encoding is as follows:
**
** KEY:
** A = 0xxxxxxx 7 bits of data and one flag bit
** B = 1xxxxxxx 7 bits of data and one flag bit
** C = xxxxxxxx 8 bits of data
**
** 7 bits - A
** 14 bits - BA
** 21 bits - BBA
** 28 bits - BBBA
** 35 bits - BBBBA
** 42 bits - BBBBBA
** 49 bits - BBBBBBA
** 56 bits - BBBBBBBA
** 64 bits - BBBBBBBBC
*/
#ifdef SQLITE_NOINLINE
# define FTS5_NOINLINE SQLITE_NOINLINE
#else
# define FTS5_NOINLINE
#endif
/*
** Write a 64-bit variable-length integer to memory starting at p[0].
** The length of data write will be between 1 and 9 bytes. The number
** of bytes written is returned.
**
** A variable-length integer consists of the lower 7 bits of each byte
** for all bytes that have the 8th bit set and one byte with the 8th
** bit clear. Except, if we get to the 9th byte, it stores the full
** 8 bits and is the last byte.
*/
static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){
int i, j, n;
u8 buf[10];
if( v & (((u64)0xff000000)<<32) ){
p[8] = (u8)v;
v >>= 8;
for(i=7; i>=0; i--){
p[i] = (u8)((v & 0x7f) | 0x80);
v >>= 7;
}
return 9;
}
n = 0;
do{
buf[n++] = (u8)((v & 0x7f) | 0x80);
v >>= 7;
}while( v!=0 );
buf[0] &= 0x7f;
assert( n<=9 );
for(i=0, j=n-1; j>=0; j--, i++){
p[i] = buf[j];
}
return n;
}
int sqlite3Fts5PutVarint(unsigned char *p, u64 v){
if( v<=0x7f ){
p[0] = v&0x7f;
return 1;
}
if( v<=0x3fff ){
p[0] = ((v>>7)&0x7f)|0x80;
p[1] = v&0x7f;
return 2;
}
return fts5PutVarint64(p,v);
}
int sqlite3Fts5GetVarintLen(u32 iVal){
#if 0
if( iVal<(1 << 7 ) ) return 1;
#endif
assert( iVal>=(1 << 7) );
if( iVal<(1 << 14) ) return 2;
if( iVal<(1 << 21) ) return 3;
if( iVal<(1 << 28) ) return 4;
return 5;
}

641
ext/fts5/fts5_vocab.c Normal file
View File

@ -0,0 +1,641 @@
/*
** 2015 May 08
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This is an SQLite virtual table module implementing direct access to an
** existing FTS5 index. The module may create several different types of
** tables:
**
** col:
** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
**
** One row for each term/column combination. The value of $doc is set to
** the number of fts5 rows that contain at least one instance of term
** $term within column $col. Field $cnt is set to the total number of
** instances of term $term in column $col (in any row of the fts5 table).
**
** row:
** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
**
** One row for each term in the database. The value of $doc is set to
** the number of fts5 rows that contain at least one instance of term
** $term. Field $cnt is set to the total number of instances of term
** $term in the database.
*/
#include "fts5Int.h"
typedef struct Fts5VocabTable Fts5VocabTable;
typedef struct Fts5VocabCursor Fts5VocabCursor;
struct Fts5VocabTable {
sqlite3_vtab base;
char *zFts5Tbl; /* Name of fts5 table */
char *zFts5Db; /* Db containing fts5 table */
sqlite3 *db; /* Database handle */
Fts5Global *pGlobal; /* FTS5 global object for this database */
int eType; /* FTS5_VOCAB_COL or ROW */
};
struct Fts5VocabCursor {
sqlite3_vtab_cursor base;
sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */
Fts5Index *pIndex; /* Associated FTS5 index */
int bEof; /* True if this cursor is at EOF */
Fts5IndexIter *pIter; /* Term/rowid iterator object */
int nLeTerm; /* Size of zLeTerm in bytes */
char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
/* These are used by 'col' tables only */
Fts5Config *pConfig; /* Fts5 table configuration */
int iCol;
i64 *aCnt;
i64 *aDoc;
/* Output values used by 'row' and 'col' tables */
i64 rowid; /* This table's current rowid value */
Fts5Buffer term; /* Current value of 'term' column */
};
#define FTS5_VOCAB_COL 0
#define FTS5_VOCAB_ROW 1
#define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt"
#define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt"
/*
** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
*/
#define FTS5_VOCAB_TERM_EQ 0x01
#define FTS5_VOCAB_TERM_GE 0x02
#define FTS5_VOCAB_TERM_LE 0x04
/*
** Translate a string containing an fts5vocab table type to an
** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
** and return SQLITE_ERROR.
*/
static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
int rc = SQLITE_OK;
char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
if( rc==SQLITE_OK ){
sqlite3Fts5Dequote(zCopy);
if( sqlite3_stricmp(zCopy, "col")==0 ){
*peType = FTS5_VOCAB_COL;
}else
if( sqlite3_stricmp(zCopy, "row")==0 ){
*peType = FTS5_VOCAB_ROW;
}else
{
*pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy);
rc = SQLITE_ERROR;
}
sqlite3_free(zCopy);
}
return rc;
}
/*
** The xDisconnect() virtual table method.
*/
static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** The xDestroy() virtual table method.
*/
static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
sqlite3_free(pTab);
return SQLITE_OK;
}
/*
** This function is the implementation of both the xConnect and xCreate
** methods of the FTS3 virtual table.
**
** The argv[] array contains the following:
**
** argv[0] -> module name ("fts5vocab")
** argv[1] -> database name
** argv[2] -> table name
**
** then:
**
** argv[3] -> name of fts5 table
** argv[4] -> type of fts5vocab table
**
** or, for tables in the TEMP schema only.
**
** argv[3] -> name of fts5 tables database
** argv[4] -> name of fts5 table
** argv[5] -> type of fts5vocab table
*/
static int fts5VocabInitVtab(
sqlite3 *db, /* The SQLite database connection */
void *pAux, /* Pointer to Fts5Global object */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
char **pzErr /* Write any error message here */
){
const char *azSchema[] = {
"CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")",
"CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")"
};
Fts5VocabTable *pRet = 0;
int rc = SQLITE_OK; /* Return code */
int bDb;
bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
if( argc!=5 && bDb==0 ){
*pzErr = sqlite3_mprintf("wrong number of vtable arguments");
rc = SQLITE_ERROR;
}else{
int nByte; /* Bytes of space to allocate */
const char *zDb = bDb ? argv[3] : argv[1];
const char *zTab = bDb ? argv[4] : argv[3];
const char *zType = bDb ? argv[5] : argv[4];
int nDb = (int)strlen(zDb)+1;
int nTab = (int)strlen(zTab)+1;
int eType = 0;
rc = fts5VocabTableType(zType, pzErr, &eType);
if( rc==SQLITE_OK ){
assert( eType>=0 && eType<ArraySize(azSchema) );
rc = sqlite3_declare_vtab(db, azSchema[eType]);
}
nByte = sizeof(Fts5VocabTable) + nDb + nTab;
pRet = sqlite3Fts5MallocZero(&rc, nByte);
if( pRet ){
pRet->pGlobal = (Fts5Global*)pAux;
pRet->eType = eType;
pRet->db = db;
pRet->zFts5Tbl = (char*)&pRet[1];
pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
memcpy(pRet->zFts5Tbl, zTab, nTab);
memcpy(pRet->zFts5Db, zDb, nDb);
sqlite3Fts5Dequote(pRet->zFts5Tbl);
sqlite3Fts5Dequote(pRet->zFts5Db);
}
}
*ppVTab = (sqlite3_vtab*)pRet;
return rc;
}
/*
** The xConnect() and xCreate() methods for the virtual table. All the
** work is done in function fts5VocabInitVtab().
*/
static int fts5VocabConnectMethod(
sqlite3 *db, /* Database connection */
void *pAux, /* Pointer to tokenizer hash table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
}
static int fts5VocabCreateMethod(
sqlite3 *db, /* Database connection */
void *pAux, /* Pointer to tokenizer hash table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
char **pzErr /* OUT: sqlite3_malloc'd error message */
){
return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
}
/*
** Implementation of the xBestIndex method.
*/
static int fts5VocabBestIndexMethod(
sqlite3_vtab *pUnused,
sqlite3_index_info *pInfo
){
int i;
int iTermEq = -1;
int iTermGe = -1;
int iTermLe = -1;
int idxNum = 0;
int nArg = 0;
UNUSED_PARAM(pUnused);
for(i=0; i<pInfo->nConstraint; i++){
struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
if( p->usable==0 ) continue;
if( p->iColumn==0 ){ /* term column */
if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ) iTermEq = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_LE ) iTermLe = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_LT ) iTermLe = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_GE ) iTermGe = i;
if( p->op==SQLITE_INDEX_CONSTRAINT_GT ) iTermGe = i;
}
}
if( iTermEq>=0 ){
idxNum |= FTS5_VOCAB_TERM_EQ;
pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
pInfo->estimatedCost = 100;
}else{
pInfo->estimatedCost = 1000000;
if( iTermGe>=0 ){
idxNum |= FTS5_VOCAB_TERM_GE;
pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
pInfo->estimatedCost = pInfo->estimatedCost / 2;
}
if( iTermLe>=0 ){
idxNum |= FTS5_VOCAB_TERM_LE;
pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
pInfo->estimatedCost = pInfo->estimatedCost / 2;
}
}
pInfo->idxNum = idxNum;
return SQLITE_OK;
}
/*
** Implementation of xOpen method.
*/
static int fts5VocabOpenMethod(
sqlite3_vtab *pVTab,
sqlite3_vtab_cursor **ppCsr
){
Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
Fts5Index *pIndex = 0;
Fts5Config *pConfig = 0;
Fts5VocabCursor *pCsr = 0;
int rc = SQLITE_OK;
sqlite3_stmt *pStmt = 0;
char *zSql = 0;
zSql = sqlite3Fts5Mprintf(&rc,
"SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
);
if( zSql ){
rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
}
sqlite3_free(zSql);
assert( rc==SQLITE_OK || pStmt==0 );
if( rc==SQLITE_ERROR ) rc = SQLITE_OK;
if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
i64 iId = sqlite3_column_int64(pStmt, 0);
pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig);
}
if( rc==SQLITE_OK && pIndex==0 ){
rc = sqlite3_finalize(pStmt);
pStmt = 0;
if( rc==SQLITE_OK ){
pVTab->zErrMsg = sqlite3_mprintf(
"no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
);
rc = SQLITE_ERROR;
}
}
if( rc==SQLITE_OK ){
int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor);
pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
}
if( pCsr ){
pCsr->pIndex = pIndex;
pCsr->pStmt = pStmt;
pCsr->pConfig = pConfig;
pCsr->aCnt = (i64*)&pCsr[1];
pCsr->aDoc = &pCsr->aCnt[pConfig->nCol];
}else{
sqlite3_finalize(pStmt);
}
*ppCsr = (sqlite3_vtab_cursor*)pCsr;
return rc;
}
static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
pCsr->rowid = 0;
sqlite3Fts5IterClose(pCsr->pIter);
pCsr->pIter = 0;
sqlite3_free(pCsr->zLeTerm);
pCsr->nLeTerm = -1;
pCsr->zLeTerm = 0;
}
/*
** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
fts5VocabResetCursor(pCsr);
sqlite3Fts5BufferFree(&pCsr->term);
sqlite3_finalize(pCsr->pStmt);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Advance the cursor to the next row in the table.
*/
static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
int rc = SQLITE_OK;
int nCol = pCsr->pConfig->nCol;
pCsr->rowid++;
if( pTab->eType==FTS5_VOCAB_COL ){
for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
if( pCsr->aDoc[pCsr->iCol] ) break;
}
}
if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){
if( sqlite3Fts5IterEof(pCsr->pIter) ){
pCsr->bEof = 1;
}else{
const char *zTerm;
int nTerm;
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( pCsr->nLeTerm>=0 ){
int nCmp = MIN(nTerm, pCsr->nLeTerm);
int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
pCsr->bEof = 1;
return SQLITE_OK;
}
}
sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
memset(pCsr->aCnt, 0, nCol * sizeof(i64));
memset(pCsr->aDoc, 0, nCol * sizeof(i64));
pCsr->iCol = 0;
assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
while( rc==SQLITE_OK ){
const u8 *pPos; int nPos; /* Position list */
i64 iPos = 0; /* 64-bit position read from poslist */
int iOff = 0; /* Current offset within position list */
pPos = pCsr->pIter->pData;
nPos = pCsr->pIter->nData;
switch( pCsr->pConfig->eDetail ){
case FTS5_DETAIL_FULL:
pPos = pCsr->pIter->pData;
nPos = pCsr->pIter->nData;
if( pTab->eType==FTS5_VOCAB_ROW ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aCnt[0]++;
}
pCsr->aDoc[0]++;
}else{
int iCol = -1;
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
int ii = FTS5_POS2COLUMN(iPos);
pCsr->aCnt[ii]++;
if( iCol!=ii ){
if( ii>=nCol ){
rc = FTS5_CORRUPT;
break;
}
pCsr->aDoc[ii]++;
iCol = ii;
}
}
}
break;
case FTS5_DETAIL_COLUMNS:
if( pTab->eType==FTS5_VOCAB_ROW ){
pCsr->aDoc[0]++;
}else{
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
assert_nc( iPos>=0 && iPos<nCol );
if( iPos>=nCol ){
rc = FTS5_CORRUPT;
break;
}
pCsr->aDoc[iPos]++;
}
}
break;
default:
assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
pCsr->aDoc[0]++;
break;
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
}
if( rc==SQLITE_OK ){
zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){
break;
}
if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
}
}
}
}
if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
assert( pCsr->iCol<pCsr->pConfig->nCol );
}
return rc;
}
/*
** This is the xFilter implementation for the virtual table.
*/
static int fts5VocabFilterMethod(
sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
int idxNum, /* Strategy index */
const char *zUnused, /* Unused */
int nUnused, /* Number of elements in apVal */
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
int rc = SQLITE_OK;
int iVal = 0;
int f = FTS5INDEX_QUERY_SCAN;
const char *zTerm = 0;
int nTerm = 0;
sqlite3_value *pEq = 0;
sqlite3_value *pGe = 0;
sqlite3_value *pLe = 0;
UNUSED_PARAM2(zUnused, nUnused);
fts5VocabResetCursor(pCsr);
if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
if( pEq ){
zTerm = (const char *)sqlite3_value_text(pEq);
nTerm = sqlite3_value_bytes(pEq);
f = 0;
}else{
if( pGe ){
zTerm = (const char *)sqlite3_value_text(pGe);
nTerm = sqlite3_value_bytes(pGe);
}
if( pLe ){
const char *zCopy = (const char *)sqlite3_value_text(pLe);
pCsr->nLeTerm = sqlite3_value_bytes(pLe);
pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1);
if( pCsr->zLeTerm==0 ){
rc = SQLITE_NOMEM;
}else{
memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
}
}
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
}
if( rc==SQLITE_OK ){
rc = fts5VocabNextMethod(pCursor);
}
return rc;
}
/*
** This is the xEof method of the virtual table. SQLite calls this
** routine to find out if it has reached the end of a result set.
*/
static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
return pCsr->bEof;
}
static int fts5VocabColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
int eDetail = pCsr->pConfig->eDetail;
int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
i64 iVal = 0;
if( iCol==0 ){
sqlite3_result_text(
pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
);
}else if( eType==FTS5_VOCAB_COL ){
assert( iCol==1 || iCol==2 || iCol==3 );
if( iCol==1 ){
if( eDetail!=FTS5_DETAIL_NONE ){
const char *z = pCsr->pConfig->azCol[pCsr->iCol];
sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
}
}else if( iCol==2 ){
iVal = pCsr->aDoc[pCsr->iCol];
}else{
iVal = pCsr->aCnt[pCsr->iCol];
}
}else{
assert( iCol==1 || iCol==2 );
if( iCol==1 ){
iVal = pCsr->aDoc[0];
}else{
iVal = pCsr->aCnt[0];
}
}
if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
return SQLITE_OK;
}
/*
** This is the xRowid method. The SQLite core calls this routine to
** retrieve the rowid for the current row of the result set. The
** rowid should be written to *pRowid.
*/
static int fts5VocabRowidMethod(
sqlite3_vtab_cursor *pCursor,
sqlite_int64 *pRowid
){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
*pRowid = pCsr->rowid;
return SQLITE_OK;
}
int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
static const sqlite3_module fts5Vocab = {
/* iVersion */ 2,
/* xCreate */ fts5VocabCreateMethod,
/* xConnect */ fts5VocabConnectMethod,
/* xBestIndex */ fts5VocabBestIndexMethod,
/* xDisconnect */ fts5VocabDisconnectMethod,
/* xDestroy */ fts5VocabDestroyMethod,
/* xOpen */ fts5VocabOpenMethod,
/* xClose */ fts5VocabCloseMethod,
/* xFilter */ fts5VocabFilterMethod,
/* xNext */ fts5VocabNextMethod,
/* xEof */ fts5VocabEofMethod,
/* xColumn */ fts5VocabColumnMethod,
/* xRowid */ fts5VocabRowidMethod,
/* xUpdate */ 0,
/* xBegin */ 0,
/* xSync */ 0,
/* xCommit */ 0,
/* xRollback */ 0,
/* xFindFunction */ 0,
/* xRename */ 0,
/* xSavepoint */ 0,
/* xRelease */ 0,
/* xRollbackTo */ 0,
};
void *p = (void*)pGlobal;
return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
}

184
ext/fts5/fts5parse.y Normal file
View File

@ -0,0 +1,184 @@
/*
** 2014 May 31
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
*/
// All token codes are small integers with #defines that begin with "TK_"
%token_prefix FTS5_
// The type of the data attached to each token is Token. This is also the
// default type for non-terminals.
//
%token_type {Fts5Token}
%default_type {Fts5Token}
// The generated parser function takes a 4th argument as follows:
%extra_argument {Fts5Parse *pParse}
// This code runs whenever there is a syntax error
//
%syntax_error {
UNUSED_PARAM(yymajor); /* Silence a compiler warning */
sqlite3Fts5ParseError(
pParse, "fts5: syntax error near \"%.*s\"",TOKEN.n,TOKEN.p
);
}
%stack_overflow {
UNUSED_PARAM(yypMinor); /* Silence a compiler warning */
sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow");
}
// The name of the generated procedure that implements the parser
// is as follows:
%name sqlite3Fts5Parser
// The following text is included near the beginning of the C source
// code file that implements the parser.
//
%include {
#include "fts5Int.h"
#include "fts5parse.h"
/*
** Disable all error recovery processing in the parser push-down
** automaton.
*/
#define YYNOERRORRECOVERY 1
/*
** Make yytestcase() the same as testcase()
*/
#define yytestcase(X) testcase(X)
/*
** Indicate that sqlite3ParserFree() will never be called with a null
** pointer.
*/
#define YYPARSEFREENOTNULL 1
/*
** Alternative datatype for the argument to the malloc() routine passed
** into sqlite3ParserAlloc(). The default is size_t.
*/
#define YYMALLOCARGTYPE u64
} // end %include
%left OR.
%left AND.
%left NOT.
%left TERM.
%left COLON.
input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
%destructor input { (void)pParse; }
%type cnearset {Fts5ExprNode*}
%type expr {Fts5ExprNode*}
%type exprlist {Fts5ExprNode*}
%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
%destructor expr { sqlite3Fts5ParseNodeFree($$); }
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
expr(A) ::= expr(X) AND expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
expr(A) ::= expr(X) OR expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
}
expr(A) ::= expr(X) NOT expr(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
}
expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X). {A = X;}
exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
cnearset(A) ::= nearset(X). {
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
}
cnearset(A) ::= colset(X) COLON nearset(Y). {
sqlite3Fts5ParseSetColset(pParse, Y, X);
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
}
%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }
colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
colsetlist(A) ::= colsetlist(Y) STRING(X). {
A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). {
A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
sqlite3Fts5ParseNear(pParse, &X);
sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
A = Y;
}
nearphrases(A) ::= phrase(X). {
A = sqlite3Fts5ParseNearset(pParse, 0, X);
}
nearphrases(A) ::= nearphrases(X) phrase(Y). {
A = sqlite3Fts5ParseNearset(pParse, X, Y);
}
/*
** The optional ", <integer>" at the end of the NEAR() arguments.
*/
neardist_opt(A) ::= . { A.p = 0; A.n = 0; }
neardist_opt(A) ::= COMMA STRING(X). { A = X; }
/*
** A phrase. A set of primitives connected by "+" operators. Examples:
**
** "the" + "quick brown" + fo *
** "the quick brown fo" *
** the+quick+brown+fo*
*/
%type phrase {Fts5ExprPhrase*}
%destructor phrase { sqlite3Fts5ParsePhraseFree($$); }
phrase(A) ::= phrase(X) PLUS STRING(Y) star_opt(Z). {
A = sqlite3Fts5ParseTerm(pParse, X, &Y, Z);
}
phrase(A) ::= STRING(Y) star_opt(Z). {
A = sqlite3Fts5ParseTerm(pParse, 0, &Y, Z);
}
/*
** Optional "*" character.
*/
%type star_opt {int}
star_opt(A) ::= STAR. { A = 1; }
star_opt(A) ::= . { A = 0; }

222
ext/fts5/mkportersteps.tcl Normal file
View File

@ -0,0 +1,222 @@
#
# 2014 Jun 09
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#-------------------------------------------------------------------------
#
# This script generates the implementations of the following C functions,
# which are part of the porter tokenizer implementation:
#
# static int fts5PorterStep1B(char *aBuf, int *pnBuf);
# static int fts5PorterStep1B2(char *aBuf, int *pnBuf);
# static int fts5PorterStep2(char *aBuf, int *pnBuf);
# static int fts5PorterStep3(char *aBuf, int *pnBuf);
# static int fts5PorterStep4(char *aBuf, int *pnBuf);
#
set O(Step1B2) {
{ at {} ate 1 }
{ bl {} ble 1 }
{ iz {} ize 1 }
}
set O(Step1B) {
{ "eed" fts5Porter_MGt0 "ee" 0 }
{ "ed" fts5Porter_Vowel "" 1 }
{ "ing" fts5Porter_Vowel "" 1 }
}
set O(Step2) {
{ "ational" fts5Porter_MGt0 "ate" }
{ "tional" fts5Porter_MGt0 "tion" }
{ "enci" fts5Porter_MGt0 "ence" }
{ "anci" fts5Porter_MGt0 "ance" }
{ "izer" fts5Porter_MGt0 "ize" }
{ "logi" fts5Porter_MGt0 "log" }
{ "bli" fts5Porter_MGt0 "ble" }
{ "alli" fts5Porter_MGt0 "al" }
{ "entli" fts5Porter_MGt0 "ent" }
{ "eli" fts5Porter_MGt0 "e" }
{ "ousli" fts5Porter_MGt0 "ous" }
{ "ization" fts5Porter_MGt0 "ize" }
{ "ation" fts5Porter_MGt0 "ate" }
{ "ator" fts5Porter_MGt0 "ate" }
{ "alism" fts5Porter_MGt0 "al" }
{ "iveness" fts5Porter_MGt0 "ive" }
{ "fulness" fts5Porter_MGt0 "ful" }
{ "ousness" fts5Porter_MGt0 "ous" }
{ "aliti" fts5Porter_MGt0 "al" }
{ "iviti" fts5Porter_MGt0 "ive" }
{ "biliti" fts5Porter_MGt0 "ble" }
}
set O(Step3) {
{ "icate" fts5Porter_MGt0 "ic" }
{ "ative" fts5Porter_MGt0 "" }
{ "alize" fts5Porter_MGt0 "al" }
{ "iciti" fts5Porter_MGt0 "ic" }
{ "ical" fts5Porter_MGt0 "ic" }
{ "ful" fts5Porter_MGt0 "" }
{ "ness" fts5Porter_MGt0 "" }
}
set O(Step4) {
{ "al" fts5Porter_MGt1 "" }
{ "ance" fts5Porter_MGt1 "" }
{ "ence" fts5Porter_MGt1 "" }
{ "er" fts5Porter_MGt1 "" }
{ "ic" fts5Porter_MGt1 "" }
{ "able" fts5Porter_MGt1 "" }
{ "ible" fts5Porter_MGt1 "" }
{ "ant" fts5Porter_MGt1 "" }
{ "ement" fts5Porter_MGt1 "" }
{ "ment" fts5Porter_MGt1 "" }
{ "ent" fts5Porter_MGt1 "" }
{ "ion" fts5Porter_MGt1_and_S_or_T "" }
{ "ou" fts5Porter_MGt1 "" }
{ "ism" fts5Porter_MGt1 "" }
{ "ate" fts5Porter_MGt1 "" }
{ "iti" fts5Porter_MGt1 "" }
{ "ous" fts5Porter_MGt1 "" }
{ "ive" fts5Porter_MGt1 "" }
{ "ize" fts5Porter_MGt1 "" }
}
proc sort_cb {lhs rhs} {
set L [string range [lindex $lhs 0] end-1 end-1]
set R [string range [lindex $rhs 0] end-1 end-1]
string compare $L $R
}
proc create_step_function {name data} {
set T(function) {
static int fts5Porter${name}(char *aBuf, int *pnBuf){
int ret = 0;
int nBuf = *pnBuf;
switch( aBuf[nBuf-2] ){
${switchbody}
}
return ret;
}
}
set T(case) {
case '${k}':
${ifstmts}
break;
}
set T(if_0_0_0) {
if( ${match} ){
*pnBuf = nBuf - $n;
}
}
set T(if_1_0_0) {
if( ${match} ){
if( ${cond} ){
*pnBuf = nBuf - $n;
}
}
}
set T(if_0_1_0) {
if( ${match} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
}
}
set T(if_1_1_0) {
if( ${match} ){
if( ${cond} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
}
}
}
set T(if_1_0_1) {
if( ${match} ){
if( ${cond} ){
*pnBuf = nBuf - $n;
ret = 1;
}
}
}
set T(if_0_1_1) {
if( ${match} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
ret = 1;
}
}
set T(if_1_1_1) {
if( ${match} ){
if( ${cond} ){
${memcpy}
*pnBuf = nBuf - $n + $nRep;
ret = 1;
}
}
}
set switchbody ""
foreach I $data {
set k [string range [lindex $I 0] end-1 end-1]
lappend aCase($k) $I
}
foreach k [lsort [array names aCase]] {
set ifstmts ""
foreach I $aCase($k) {
set zSuffix [lindex $I 0] ;# Suffix text for this rule
set zRep [lindex $I 2] ;# Replacement text for rule
set xCond [lindex $I 1] ;# Condition callback (or "")
set n [string length $zSuffix]
set nRep [string length $zRep]
set match "nBuf>$n && 0==memcmp(\"$zSuffix\", &aBuf\[nBuf-$n\], $n)"
set memcpy "memcpy(&aBuf\[nBuf-$n\], \"$zRep\", $nRep);"
set cond "${xCond}(aBuf, nBuf-$n)"
set bMemcpy [expr {$nRep>0}]
set bCond [expr {$xCond!=""}]
set bRet [expr {[llength $I]>3 && [lindex $I 3]}]
set t $T(if_${bCond}_${bMemcpy}_${bRet})
lappend ifstmts [string trim [subst -nocommands $t]]
}
set ifstmts [join $ifstmts "else "]
append switchbody [subst -nocommands $T(case)]
}
puts [subst -nocommands $T(function)]
}
puts [string trim {
/**************************************************************************
***************************************************************************
** GENERATED CODE STARTS HERE (mkportersteps.tcl)
*/
}]
foreach step [array names O] {
create_step_function $step $O($step)
}
puts [string trim {
/*
** GENERATED CODE ENDS HERE (mkportersteps.tcl)
***************************************************************************
**************************************************************************/
}]

642
ext/fts5/test/fts5_common.tcl Normal file
View File

@ -0,0 +1,642 @@
# 2014 Dec 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
if {![info exists testdir]} {
set testdir [file join [file dirname [info script]] .. .. .. test]
}
source $testdir/tester.tcl
ifcapable !fts5 {
finish_test
return
}
catch {
sqlite3_fts5_may_be_corrupt 0
reset_db
}
# If SQLITE_ENABLE_FTS5 is not defined, skip this test.
ifcapable !fts5 {
finish_test
return
}
proc fts5_test_poslist {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
lappend res [string map {{ } .} [$cmd xInst $i]]
}
set res
}
proc fts5_test_poslist2 {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
$cmd xPhraseForeach $i c o {
lappend res $i.$c.$o
}
}
#set res
sort_poslist $res
}
proc fts5_test_collist {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
$cmd xPhraseColumnForeach $i c { lappend res $i.$c }
}
set res
}
proc fts5_test_columnsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnSize $i]
}
set res
}
proc fts5_test_columntext {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnText $i]
}
set res
}
proc fts5_test_columntotalsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
lappend res [$cmd xColumnTotalSize $i]
}
set res
}
proc test_append_token {varname token iStart iEnd} {
upvar $varname var
lappend var $token
return "SQLITE_OK"
}
proc fts5_test_tokenize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
set tokens [list]
$cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
lappend res $tokens
}
set res
}
proc fts5_test_rowcount {cmd} {
$cmd xRowCount
}
proc test_queryphrase_cb {cnt cmd} {
upvar $cnt L
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
foreach {ip ic io} [$cmd xInst $i] break
set A($ic) 1
}
foreach ic [array names A] {
lset L $ic [expr {[lindex $L $ic] + 1}]
}
}
proc fts5_test_queryphrase {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
set cnt [list]
for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
$cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
lappend res $cnt
}
set res
}
proc fts5_test_phrasecount {cmd} {
$cmd xPhraseCount
}
proc fts5_test_all {cmd} {
set res [list]
lappend res columnsize [fts5_test_columnsize $cmd]
lappend res columntext [fts5_test_columntext $cmd]
lappend res columntotalsize [fts5_test_columntotalsize $cmd]
lappend res poslist [fts5_test_poslist $cmd]
lappend res tokenize [fts5_test_tokenize $cmd]
lappend res rowcount [fts5_test_rowcount $cmd]
set res
}
proc fts5_aux_test_functions {db} {
foreach f {
fts5_test_columnsize
fts5_test_columntext
fts5_test_columntotalsize
fts5_test_poslist
fts5_test_poslist2
fts5_test_collist
fts5_test_tokenize
fts5_test_rowcount
fts5_test_all
fts5_test_queryphrase
fts5_test_phrasecount
} {
sqlite3_fts5_create_function $db $f $f
}
}
proc fts5_level_segs {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
foreach L [lrange [db one $sql] 1 end] {
lappend ret [expr [llength $L] - 3]
}
set ret
}
proc fts5_level_segids {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
foreach L [lrange [db one $sql] 1 end] {
set lvl [list]
foreach S [lrange $L 3 end] {
regexp {id=([1234567890]*)} $S -> segid
lappend lvl $segid
}
lappend ret $lvl
}
set ret
}
proc fts5_rnddoc {n} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
}
set doc
}
#-------------------------------------------------------------------------
# Usage:
#
# nearset aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
#
# This command is used to test if a document (set of column values) matches
# the logical equivalent of a single FTS5 NEAR() clump and, if so, return
# the equivalent of an FTS5 position list.
#
# Parameter $aCol is passed a list of the column values for the document
# to test. Parameters $phrase1 and so on are the phrases.
#
# The result is a list of phrase hits. Each phrase hit is formatted as
# three integers separated by "." characters, in the following format:
#
# <phrase number> . <column number> . <token offset>
#
# Options:
#
# -near N (NEAR distance. Default 10)
# -col C (List of column indexes to match against)
# -pc VARNAME (variable in caller frame to use for phrase numbering)
# -dict VARNAME (array in caller frame to use for synonyms)
#
proc nearset {aCol args} {
# Process the command line options.
#
set O(-near) 10
set O(-col) {}
set O(-pc) ""
set O(-dict) ""
set nOpt [lsearch -exact $args --]
if {$nOpt<0} { error "no -- option" }
# Set $lPhrase to be a list of phrases. $nPhrase its length.
set lPhrase [lrange $args [expr $nOpt+1] end]
set nPhrase [llength $lPhrase]
foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
if {[info exists O($k)]==0} { error "unrecognized option $k" }
set O($k) $v
}
if {$O(-pc) == ""} {
set counter 0
} else {
upvar $O(-pc) counter
}
if {$O(-dict)!=""} { upvar $O(-dict) aDict }
for {set j 0} {$j < [llength $aCol]} {incr j} {
for {set i 0} {$i < $nPhrase} {incr i} {
set A($j,$i) [list]
}
}
# Loop through each column of the current row.
for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
# If there is a column filter, test whether this column is excluded. If
# so, skip to the next iteration of this loop. Otherwise, set zCol to the
# column value and nToken to the number of tokens that comprise it.
if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
set zCol [lindex $aCol $iCol]
set nToken [llength $zCol]
# Each iteration of the following loop searches a substring of the
# column value for phrase matches. The last token of the substring
# is token $iLast of the column value. The first token is:
#
# iFirst = ($iLast - $O(-near) - 1)
#
# where $sz is the length of the phrase being searched for. A phrase
# counts as matching the substring if its first token lies on or before
# $iLast and its last token on or after $iFirst.
#
# For example, if the query is "NEAR(a+b c, 2)" and the column value:
#
# "x x x x A B x x C x"
# 0 1 2 3 4 5 6 7 8 9"
#
# when (iLast==8 && iFirst=5) the range will contain both phrases and
# so both instances can be added to the output poslists.
#
set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
for { } {$iLast < $nToken} {incr iLast} {
catch { array unset B }
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
set p [lindex $lPhrase $iPhrase]
set nPm1 [expr {[llength $p] - 1}]
set iFirst [expr $iLast - $O(-near) - [llength $p]]
for {set i $iFirst} {$i <= $iLast} {incr i} {
set lCand [lrange $zCol $i [expr $i+$nPm1]]
set bMatch 1
foreach tok $p term $lCand {
if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break }
}
if {$bMatch} { lappend B($iPhrase) $i }
}
if {![info exists B($iPhrase)]} break
}
if {$iPhrase==$nPhrase} {
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
}
}
}
}
set res [list]
#puts [array names A]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
foreach a $A($iCol,$iPhrase) {
lappend res "$counter.$iCol.$a"
}
}
incr counter
}
#puts "$aCol -> $res"
sort_poslist $res
}
proc nearset_match {aDictVar tok term} {
if {[string match $tok $term]} { return 1 }
upvar $aDictVar aDict
if {[info exists aDict($tok)]} {
foreach s $aDict($tok) {
if {[string match $s $term]} { return 1 }
}
}
return 0;
}
#-------------------------------------------------------------------------
# Usage:
#
# sort_poslist LIST
#
# Sort a position list of the type returned by command [nearset]
#
proc sort_poslist {L} {
lsort -command instcompare $L
}
proc instcompare {lhs rhs} {
foreach {p1 c1 o1} [split $lhs .] {}
foreach {p2 c2 o2} [split $rhs .] {}
set res [expr $c1 - $c2]
if {$res==0} { set res [expr $o1 - $o2] }
if {$res==0} { set res [expr $p1 - $p2] }
return $res
}
#-------------------------------------------------------------------------
# Logical operators used by the commands returned by fts5_tcl_expr().
#
proc AND {args} {
foreach a $args {
if {[llength $a]==0} { return [list] }
}
sort_poslist [concat {*}$args]
}
proc OR {args} {
sort_poslist [concat {*}$args]
}
proc NOT {a b} {
if {[llength $b]>0} { return [list] }
return $a
}
#-------------------------------------------------------------------------
# This command is similar to [split], except that it also provides the
# start and end offsets of each token. For example:
#
# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8}
#
proc gobble_whitespace {textvar} {
upvar $textvar t
regexp {([ ]*)(.*)} $t -> space t
return [string length $space]
}
proc gobble_text {textvar wordvar} {
upvar $textvar t
upvar $wordvar w
regexp {([^ ]*)(.*)} $t -> w t
return [string length $w]
}
proc fts5_tokenize_split {text} {
set token ""
set ret [list]
set iOff [gobble_whitespace text]
while {[set nToken [gobble_text text word]]} {
lappend ret $word $iOff [expr $iOff+$nToken]
incr iOff $nToken
incr iOff [gobble_whitespace text]
}
set ret
}
#-------------------------------------------------------------------------
#
proc foreach_detail_mode {prefix script} {
set saved $::testprefix
foreach d [list full col none] {
set s [string map [list %DETAIL% $d] $script]
set ::detail $d
set ::testprefix "$prefix-$d"
reset_db
uplevel $s
unset ::detail
}
set ::testprefix $saved
}
proc detail_check {} {
if {$::detail != "none" && $::detail!="full" && $::detail!="col"} {
error "not in foreach_detail_mode {...} block"
}
}
proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
proc detail_is_col {} { detail_check ; expr {$::detail == "col" } }
proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
#-------------------------------------------------------------------------
# Convert a poslist of the type returned by fts5_test_poslist() to a
# collist as returned by fts5_test_collist().
#
proc fts5_poslist2collist {poslist} {
set res [list]
foreach h $poslist {
regexp {(.*)\.[1234567890]+} $h -> cand
lappend res $cand
}
set res [lsort -command fts5_collist_elem_compare -unique $res]
return $res
}
# Comparison function used by fts5_poslist2collist to sort collist entries.
proc fts5_collist_elem_compare {a b} {
foreach {a1 a2} [split $a .] {}
foreach {b1 b2} [split $b .] {}
if {$a1==$b1} { return [expr $a2 - $b2] }
return [expr $a1 - $b1]
}
#--------------------------------------------------------------------------
# Construct and return a tcl list equivalent to that returned by the SQL
# query executed against database handle [db]:
#
# SELECT
# rowid,
# fts5_test_poslist($tbl),
# fts5_test_collist($tbl)
# FROM $tbl('$expr')
# ORDER BY rowid $order;
#
proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} {
# Figure out the set of columns in the FTS5 table. This routine does
# not handle tables with UNINDEXED columns, but if it did, it would
# have to be here.
db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
set d ""
if {$aDictVar != ""} {
upvar $aDictVar aDict
set d aDict
}
set cols ""
foreach e $lCols { append cols ", '$e'" }
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] )
}]]
set res [list]
db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
set cols [list]
foreach col $lCols { lappend cols $x($col) }
set ::pc 0
set rowdata [eval $tclexpr]
if {$rowdata != ""} {
lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
}
}
set res
}
#-------------------------------------------------------------------------
# Similar to [fts5_query_data], but omit the collist field.
#
proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
set res [list]
if {$aDictVar!=""} {
upvar $aDictVar aDict
set dict aDict
} else {
set dict ""
}
foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
lappend res $rowid $poslist
}
set res
}
proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
set res [list]
if {$aDictVar!=""} {
upvar $aDictVar aDict
set dict aDict
} else {
set dict ""
}
foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
lappend res $rowid $collist
}
set res
}
#-------------------------------------------------------------------------
#
# This command will only work inside a [foreach_detail_mode] block. It tests
# whether or not expression $expr run on FTS5 table $tbl is supported by
# the current mode. If so, 1 is returned. If not, 0.
#
# detail=full (all queries supported)
# detail=col (all but phrase queries and NEAR queries)
# detail=none (all but phrase queries, NEAR queries, and column filters)
#
proc fts5_expr_ok {expr tbl} {
if {![detail_is_full]} {
set nearset "nearset_rc"
if {[detail_is_col]} { set nearset "nearset_rf" }
set ::expr_not_ok 0
db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
set cols ""
foreach e $lCols { append cols ", '$e'" }
set ::pc 0
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
}]]
eval $tclexpr
if {$::expr_not_ok} { return 0 }
}
return 1
}
# Helper for [fts5_expr_ok]
proc nearset_rf {aCol args} {
set idx [lsearch -exact $args --]
if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
set ::expr_not_ok 1
}
list
}
# Helper for [fts5_expr_ok]
proc nearset_rc {aCol args} {
nearset_rf $aCol {*}$args
if {[lsearch $args -col]>=0} {
set ::expr_not_ok 1
}
list
}
#-------------------------------------------------------------------------
# Code for a simple Tcl tokenizer that supports synonyms at query time.
#
proc tclnum_tokenize {mode tflags text} {
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
foreach s $::tclnum_syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
}
}
}
proc tclnum_create {args} {
set mode query
if {[llength $args]} {
set mode [lindex $args 0]
}
if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
return [list tclnum_tokenize $mode]
}
proc fts5_tclnum_register {db} {
foreach SYNDICT {
{zero 0}
{one 1 i}
{two 2 ii}
{three 3 iii}
{four 4 iv}
{five 5 v}
{six 6 vi}
{seven 7 vii}
{eight 8 viii}
{nine 9 ix}
{a1 a2 a3 a4 a5 a6 a7 a8 a9}
{b1 b2 b3 b4 b5 b6 b7 b8 b9}
{c1 c2 c3 c4 c5 c6 c7 c8 c9}
} {
foreach s $SYNDICT {
set o [list]
foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
set ::tclnum_syn($s) $o
}
}
sqlite3_fts5_create_tokenizer db tclnum tclnum_create
}
#
# End of tokenizer code.
#-------------------------------------------------------------------------

562
ext/fts5/test/fts5aa.test Normal file
View File

@ -0,0 +1,562 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aa
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $::testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
SELECT name, sql FROM sqlite_master;
} {
t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID}
t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID}
}
do_execsql_test 1.1 {
DROP TABLE t1;
SELECT name, sql FROM sqlite_master;
} {
}
#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
}
do_test 2.2 {
execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/}
foreach w {a b c d e f} {
do_execsql_test 2.3.$w.asc {
SELECT rowid FROM t1 WHERE t1 MATCH $w;
} {1}
do_execsql_test 2.3.$w.desc {
SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC;
} {1}
}
do_execsql_test 2.4 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
} {
do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
} {
do_execsql_test 4.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 4.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
1 {dd abc abc abc abcde} {aaa dd ddd ddd aab}
2 {dd aab d aaa b} {abcde c aaa aaa aaa}
3 {abcde dd b b dd} {abc abc d abc ddddd}
4 {aaa abcde dddd dddd abcde} {abc b b abcde abc}
5 {aab dddd d dddd c} {ddd abcde dddd abcde c}
6 {ddd dd b aab abcde} {d ddddd dddd c abc}
7 {d ddddd ddd c abcde} {c aab d abcde ddd}
8 {abcde aaa aab c c} {ddd c dddd b aaa}
9 {abcde aab ddddd c aab} {dddd dddd b c dd}
10 {ddd abcde dddd dd c} {dddd c c d abcde}
} {
do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_execsql_test 6.1 {
INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
}
do_execsql_test 6.2 {
INSERT INTO t1(t1) VALUES('integrity-check')
}
do_execsql_test 6.3 {
REPLACE INTO t1(rowid, x, y) VALUES('22', 'l l l', 'l l l');
}
do_execsql_test 6.4 {
REPLACE INTO t1(x, y) VALUES('x y z', 'x y z');
}
do_execsql_test 6.5 {
INSERT INTO t1(t1) VALUES('integrity-check')
}
do_execsql_test 6.6 {
SELECT rowid, * FROM t1;
} {
22 {l l l} {l l l}
23 {x y z} {x y z}
}
#-------------------------------------------------------------------------
#
reset_db
expr srand(0)
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
proc doc {} {
set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
set ret [list]
for {set j 0} {$j < 20} {incr j} {
lappend ret [lindex $v [expr int(rand()*[llength $v])]]
}
return $ret
}
proc dump_structure {} {
db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
foreach lvl [lrange $t 1 end] {
set seg [string repeat . [expr [llength $lvl]-2]]
puts "[lrange $lvl 0 1] $seg"
}
}
}
for {set i 1} {$i <= 10} {incr i} {
do_test 7.$i {
for {set j 0} {$j < 10} {incr j} {
set x [doc]
set y [doc]
set z [doc]
set rowid [expr int(rand() * 100)]
execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
} {}
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_execsql_test 8.1 {
INSERT INTO t1 VALUES('the quick brown fox');
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
expr srand(0)
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
proc doc {} {
set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
set ret [list]
for {set j 0} {$j < 20} {incr j} {
lappend ret [lindex $v [expr int(rand()*[llength $v])]]
}
return $ret
}
proc dump_structure {} {
db eval {SELECT fts5_decode(id, block) AS t FROM t1_data WHERE id=10} {
foreach lvl [lrange $t 1 end] {
set seg [string repeat . [expr [llength $lvl]-2]]
puts "[lrange $lvl 0 1] $seg"
}
}
}
for {set i 1} {$i <= 10} {incr i} {
do_test 9.$i {
for {set j 0} {$j < 100} {incr j} {
set x [doc]
set y [doc]
set z [doc]
set rowid [expr int(rand() * 100)]
execsql { REPLACE INTO t1(rowid,x,y,z) VALUES($rowid, $x, $y, $z) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check'); }
} {}
if {[set_test_counter errors]} break
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
set d10 {
1 {g f d b f} {h h e i a}
2 {f i g j e} {i j c f f}
3 {e e i f a} {e h f d f}
4 {h j f j i} {h a c f j}
5 {d b j c g} {f e i b e}
6 {a j a e e} {j d f d e}
7 {g i j c h} {j d h c a}
8 {j j i d d} {e e d f b}
9 {c j j d c} {h j i f g}
10 {b f h i a} {c f b b j}
}
foreach {rowid x y} $d10 {
do_execsql_test 10.1.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 10.1.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach rowid {5 9 8 1 2 4 10 7 3 5 6} {
do_execsql_test 10.2.$rowid.1 { DELETE FROM t1 WHERE rowid = $rowid }
do_execsql_test 10.2.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach {rowid x y} $d10 {
do_execsql_test 10.3.$rowid.1 { INSERT INTO t1 VALUES($x, $y) }
do_execsql_test 10.3.$rowid.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
do_execsql_test 10.4.1 { DELETE FROM t1 }
do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
#-------------------------------------------------------------------------
#
do_catchsql_test 11.1 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
} {1 {reserved fts5 table name: rank}}
do_catchsql_test 11.3 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
} {1 {reserved fts5 column name: rowid}}
#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
} {}
do_catchsql_test 12.2 {
SELECT t2 FROM t2 WHERE t2 MATCH '*stuff'
} {1 {unknown special query: stuff}}
do_test 12.3 {
set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
string is integer $res
} {1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
} {}
do_execsql_test 13.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'o';
} {1 2}
do_execsql_test 13.4 {
DELETE FROM t1 WHERE rowid=2;
} {}
do_execsql_test 13.5 {
SELECT rowid FROM t1 WHERE t1 MATCH 'o';
} {1}
do_execsql_test 13.6 {
SELECT rowid FROM t1 WHERE t1 MATCH '""';
} {}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 14.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH d(x,y) AS (
SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
UNION ALL
SELECT NULL, 'xyz xyz xyz xyz xyz xyz' FROM d
)
INSERT INTO t1 SELECT * FROM d LIMIT 200;
}
do_execsql_test 15.x {
INSERT INTO t1(t1) VALUES('integrity-check');
}
do_test 14.2 {
set nRow 0
db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
db eval {
BEGIN;
CREATE TABLE t2(a, b);
ROLLBACK;
}
incr nRow
}
set nRow
} {200}
do_test 14.3 {
set nRow 0
db eval { BEGIN; }
db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
db eval {
SAVEPOINT aaa;
CREATE TABLE t2(a, b);
ROLLBACK TO aaa;
RELEASE aaa;
}
incr nRow
}
set nRow
} {200}
do_execsql_test 15.0 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
do_execsql_test 15.1 {
UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
}
do_catchsql_test 15.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
#
do_execsql_test 16.1 {
CREATE VIRTUAL TABLE n1 USING fts5(a);
INSERT INTO n1 VALUES('a b c d');
}
proc funk {} {
db eval { UPDATE n1_config SET v=50 WHERE k='version' }
set fd [db incrblob main n1_data block 10]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd "\x44\x45"
close $fd
}
db func funk funk
do_catchsql_test 16.2 {
SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
} {1 {SQL logic error or missing database}}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
INSERT INTO b2 VALUES('a');
INSERT INTO b2 VALUES('b');
INSERT INTO b2 VALUES('c');
}
do_test 17.2 {
set res [list]
db eval { SELECT * FROM b2 ORDER BY rowid ASC } {
lappend res [execsql { SELECT * FROM b2 ORDER BY rowid ASC }]
}
set res
} {{a b c} {a b c} {a b c}}
if {[string match n* %DETAIL%]==0} {
reset_db
do_execsql_test 17.3 {
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO c2 VALUES('x x x', 'x x x');
SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
} {1}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
INSERT INTO uio VALUES(NULL);
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
SELECT count(*) FROM uio;
} {256}
do_execsql_test 17.2 {
SELECT count(*) FROM uio WHERE rowid BETWEEN 8 AND 17
} {10}
do_execsql_test 17.3 {
SELECT rowid FROM uio WHERE rowid BETWEEN 8 AND 17
} {8 9 10 11 12 13 14 15 16 17}
do_execsql_test 17.4 {
SELECT rowid FROM uio WHERE rowid BETWEEN 8 AND 17 ORDER BY rowid DESC
} {17 16 15 14 13 12 11 10 9 8}
do_execsql_test 17.5 {
SELECT count(*) FROM uio
} {256}
do_execsql_test 17.6 {
INSERT INTO uio(rowid) VALUES(9223372036854775807);
INSERT INTO uio(rowid) VALUES(-9223372036854775808);
SELECT count(*) FROM uio;
} {258}
do_execsql_test 17.7 {
SELECT min(rowid), max(rowid) FROM uio;
} {-9223372036854775808 9223372036854775807}
do_execsql_test 17.8 {
INSERT INTO uio DEFAULT VALUES;
SELECT min(rowid), max(rowid), count(*) FROM uio;
} {-9223372036854775808 9223372036854775807 259}
do_execsql_test 17.9 {
SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10;
} {-9223372036854775808 9 10}
#--------------------------------------------------------------------
#
do_execsql_test 18.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
INSERT INTO t1 VALUES('abc*', NULL);
INSERT INTO t2 VALUES(1, 'abcdefg');
}
do_execsql_test 18.2 {
SELECT t1.rowid, t2.rowid FROM t1, t2 WHERE t2 MATCH t1.a AND t1.rowid = t2.c
} {1 1}
do_execsql_test 18.3 {
SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c
} {1 1}
#--------------------------------------------------------------------
# fts5 table in the temp schema.
#
reset_db
do_execsql_test 19.0 {
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1 VALUES('x y z');
INSERT INTO t1 VALUES('w x 1');
SELECT rowid FROM t1 WHERE t1 MATCH 'x';
} {1 2}
#--------------------------------------------------------------------
# Test that 6 and 7 byte varints can be read.
#
reset_db
do_execsql_test 20.0 {
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
}
set ::ids [list \
0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
]
do_test 20.1 {
foreach id $::ids {
execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') }
}
execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
} $::ids
}
finish_test

297
ext/fts5/test/fts5ab.test Normal file
View File

@ -0,0 +1,297 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ab
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1 VALUES('hello', 'world');
INSERT INTO t1 VALUES('one two', 'three four');
INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
}
do_execsql_test 1.1 {
SELECT * FROM t1 ORDER BY rowid DESC;
} { forty five {one two} {three four} hello world }
do_execsql_test 1.2 {
SELECT rowid FROM t1 ORDER BY rowid DESC;
} {45 2 1}
do_execsql_test 1.3 {
SELECT rowid FROM t1 ORDER BY rowid ASC;
} {1 2 45}
do_execsql_test 1.4 {
SELECT * FROM t1 WHERE rowid=2;
} {{one two} {three four}}
do_execsql_test 1.5 {
SELECT * FROM t1 WHERE rowid=2.01;
} {}
do_execsql_test 1.6 {
SELECT * FROM t1 WHERE rowid=1.99;
} {}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 VALUES('one');
INSERT INTO t1 VALUES('two');
INSERT INTO t1 VALUES('three');
}
do_catchsql_test 2.2 {
SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'
} {1 {fts5: syntax error near "AND"}}
do_execsql_test 2.3 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'two' } {2 two}
do_execsql_test 2.4 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'three' } {3 three}
do_execsql_test 2.5 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'one' } {1 one}
do_execsql_test 2.6 {
INSERT INTO t1 VALUES('a b c d e f g');
INSERT INTO t1 VALUES('b d e a a a i');
INSERT INTO t1 VALUES('x y z b c c c');
}
foreach {tn expr res} {
1 a {5 4}
2 b {6 5 4}
3 c {6 4}
4 d {5 4}
5 e {5 4}
6 f {4}
7 g {4}
8 x {6}
9 y {6}
10 z {6}
} {
do_execsql_test 2.7.$tn.1 {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC
} $res
do_execsql_test 2.7.$tn.2 {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC
} [lsort -integer $res]
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a,b);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {tn a b} {
1 {abashed abandons abase abash abaft} {abases abased}
2 {abasing abases abaft abated abandons} {abases abandoned}
3 {abatement abash abash abated abase} {abasements abashing}
4 {abaft abasements abase abasement abasing} {abasement abases}
5 {abaft abashing abatement abash abasements} {abandons abandoning}
6 {aback abate abasements abashes abandoned} {abasement abased}
7 {abandons abated abased aback abandoning} {abases abandoned}
8 {abashing abases abasement abaft abashing} {abashed abate}
9 {abash abase abate abashing abashed} {abandon abandoned}
10 {abate abandoning abandons abasement aback} {abandon abandoning}
} {
do_execsql_test 3.1.$tn.1 { INSERT INTO t1 VALUES($a, $b) }
do_execsql_test 3.1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
}
foreach {tn expr res} {
1 {abash} {9 5 3 1}
2 {abase} {9 4 3 1}
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {8 5}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
do_execsql_test 3.2.$tn {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid DESC
} $res
}
do_execsql_test 3.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)'
} {6}
foreach {tn expr res} {
1 {abash} {1 3 5 9}
2 {abase} {1 3 4 9}
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {5 8}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
do_execsql_test 3.4.$tn {
SELECT rowid FROM t1 WHERE t1 MATCH $expr
} $res
}
#-------------------------------------------------------------------------
# Documents with more than 2M tokens.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%);
}
foreach {tn doc} [list \
1 [string repeat {a x } 1500000] \
2 "[string repeat {a a } 1500000] x" \
] {
do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) }
}
do_execsql_test 4.3 {
SELECT rowid FROM s1 WHERE s1 MATCH 'x'
} {1 2}
if {[detail_is_full]} {
do_execsql_test 4.4 {
SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
} {1 2}
}
do_execsql_test 4.5 {
SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
} {1 2}
#-------------------------------------------------------------------------
# Check that a special case of segment promotion works. The case is where
# a new segment is written to level L, but the oldest segment within level
# (L-2) is larger than it.
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
proc rnddoc {n} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]]
}
set doc
}
db func rnddoc rnddoc
do_test 5.1 {
for {set i 1} {$i <= 65} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(10)) }
}
for {set i 1} {$i <= 63} {incr i} {
execsql { DELETE FROM s2 WHERE rowid = $i }
}
fts5_level_segs s2
} {0 8}
do_test 5.2 {
execsql {
INSERT INTO s2(s2, rank) VALUES('automerge', 8);
}
for {set i 0} {$i < 7} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(50)) }
}
fts5_level_segs s2
} {8 0 0}
# Test also the other type of segment promotion - when a new segment is written
# that is larger than segments immediately following it.
do_test 5.3 {
execsql {
DROP TABLE s2;
CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
for {set i 1} {$i <= 16} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
}
fts5_level_segs s2
} {0 1}
do_test 5.4 {
execsql { INSERT INTO s2 VALUES(rnddoc(160)) }
fts5_level_segs s2
} {2 0}
#-------------------------------------------------------------------------
#
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%);
BEGIN;
INSERT INTO s3 VALUES('a b c');
INSERT INTO s3 VALUES('A B C');
}
do_execsql_test 6.1.1 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a'
} {1 2}
do_execsql_test 6.1.2 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a' ORDER BY rowid DESC
} {2 1}
do_execsql_test 6.2 {
COMMIT;
}
do_execsql_test 6.3 {
SELECT rowid FROM s3 WHERE s3 MATCH 'a'
} {1 2}
do_test 6.4 {
db close
sqlite3 db test.db
execsql {
BEGIN;
INSERT INTO s3(s3) VALUES('optimize');
ROLLBACK;
}
} {}
#-------------------------------------------------------------------------
#
set doc [string repeat "a b c " 500]
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
INSERT INTO x1 VALUES($doc);
}
} ;# foreach_detail_mode...
finish_test

279
ext/fts5/test/fts5ac.test Normal file
View File

@ -0,0 +1,279 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ac
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
set data {
0 {p o q e z k z p n f y u z y n y} {l o o l v v k}
1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w}
2 {l s z j k i m p s} {l w e j t j e e i t w r o p o}
3 {x g y m y m h p} {k j j b r e y y a k y}
4 {q m a i y i z} {o w a g k x g j m w e u k}
5 {k o a w y b s z} {s g l m m l m g p}
6 {d a q i z h b l c p k j g k} {p x u j x t v c z}
7 {f d a g o c t i} {w f c x l d r k i j}
8 {y g w u b q p o m j y b p a e k} {r i d k y w o z q m a t p}
9 {r k o m c c j s x m x m x m q r} {y r c a q d z k n x n}
10 {k j q m g q a j d} {d d e z g w h c d o o g x d}
11 {j z u m o y q j f w e e w t r j w} {g m o r x n t n w i f g l z f}
12 {s y w a w d o h x m k} {c w k z b p o r a}
13 {u t h x e g s k n g i} {f j w g c s r}
14 {b f i c s u z t k} {c k q s j u i z o}
15 {n a f n u s w h y n s i q e w} {x g e g a s s h n}
16 {k s q e j n p} {t r j f t o e k k l m i}
17 {g d t u w r o p m n m n p h b o u} {h s w o s l j e}
18 {f l q y q q g e e x j r} {n b r r g e i r t x q k}
19 {f i r g o a w e p i l o a w} {e k r z t d g h g i b d i e m}
20 {l d u u f p y} {g o m m u x m g l j t t x x u}
21 {m c d k x i c z l} {m i a i e u h}
22 {w b f o c g x y j} {z d w x d f h i p}
23 {w u i u x t c h k i b} {b y k h b v r t g j}
24 {h f d j s w s b a p k} {a q y u z e y m m j q r}
25 {d i x y x x k i y f s d j h z p n} {l l q m e t c w g y h t s v g}
26 {g s q w t d k x g f m j p k y} {r m b x e l t d}
27 {j l s q u g y v e c l o} {m f l m m m h g x x l n c}
28 {c t j g v r s b z j} {l c f y d t q n}
29 {e x z y w i h l} {b n b x e y q e n u m}
30 {g y y h j b w r} {q b q f u s k c k g r}
31 {g u l x l b r c m z b u c} {k g t b x k x n t e z d h o}
32 {w g v l z f b z h p s c v h} {g e w v m h k r g w a r f q}
33 {c g n f u d o y o b} {e y o h x x y y i z s b h a j}
34 {v y h c q u u s q y x x k s q} {d n r m y k n t i r n w e}
35 {o u c x l e b t a} {y b a x y f z x r}
36 {x p h l j a a u u j h} {x o f s z m b c q p}
37 {k q t i c a q n m v v} {v r z e f m y o}
38 {r w t t t t r v v o e p g h} {l w x a g a u h y}
39 {o p v g v b a g o} {j t q c r b b g y z}
40 {f s o r o d t h q f x l} {r d b m k i f s t d l m y x j w}
41 {t m o t m f m f} {i p i q j v n v m b q}
42 {t x w a r l w d t b c o d o} {a h f h w z d n s}
43 {t u q c d g p q x j o l c x c} {m n t o z z j a y}
44 {v d i i k b f s z r v r z y} {g n q y s x x m b x c l w}
45 {p v v a c s z y e o l} {m v t u d k m k q b d c v z r}
46 {f y k l d r q w r s t r e} {h m v r r l r r t f q e x y}
47 {w l n l t y x} {n h s l a f c h u f l x x m v n o}
48 {t n v i k e b p z p d j j l i o} {i v z p g u e j s i k n h w d c}
49 {z v x p n l t a j c} {e j l e n c e t a d}
50 {w u b x u i v h a i y m m r p m s} {s r h d o g z y f f x e}
51 {d c c x b c a x g} {p r a j v u y}
52 {f w g r c o d l t u e z h i} {j l l s s b j m}
53 {p m t f k i x} {u v y a z g w v v m x h i}
54 {l c z g l o j i c d e b} {b f v y w u i b e i y}
55 {r h c x f x a d s} {z x y k f l r b q c v}
56 {v x x c y h z x b g m o q n c} {h n b i t g h a q b c o r u}
57 {d g l o h t b s b r} {n u e p t i m u}
58 {t d y e t d c w u o s w x f c h} {i o s v y b r d r}
59 {l b a p q n d r} {k d c c d n y q h g a o p e x}
60 {f r z v m p k r} {x x r i s b a g f c}
61 {s a z i e r f i w c n y v z t k s} {y y i r y n l s b w i e k n}
62 {n x p r e x q r m v i b y} {f o o z n b s r q j}
63 {y j s u j x o n r q t f} {f v k n v x u s o a d e f e}
64 {u s i l y c x q} {r k c h p c h b o s s u s p b}
65 {m p i o s h o} {s w h u n d m n q t y k b w c}
66 {l d f g m x x x o} {s w d d f b y j j h h t i y p j o}
67 {c b m h f n v w n h} {i r w i e x r w l z p x u g u l s}
68 {y a h u h i m a y q} {d d r x h e v q n z y c j}
69 {c x f d x o n p o b r t b l p l} {m i t k b x v f p t m l l y r o}
70 {u t l w w m s} {m f m o l t k o p e}
71 {f g q e l n d m z x q} {z s i i i m f w w f n g p e q}
72 {n l h a v u o d f j d e x} {v v s l f g d g r a j x i f z x}
73 {x v m v f i g q e w} {r y s j i k m j j e d g r n o i f}
74 {g d y n o h p s y q z j d w n h w} {x o d l t j i b r d o r y}
75 {p g b i u r b e q d v o a g w m k} {q y z s f q o h}
76 {u z a q u f i f f b} {b s p b a a d x r r i q f}
77 {w h h z t h p o a h h e e} {h w r p h k z v y f r x}
78 {c a r k i a p u x} {f w l p t e m l}
79 {q q u k o t r k z} {f b m c w p s s o z}
80 {t i g v y q s r x m r x z e f} {x o j w a u e y s j c b u p p r o}
81 {n j n h r l a r e o z w e} {v o r r j a v b}
82 {i f i d k w d n h} {o i d z i z l m w s b q v u}
83 {m d g q q b k b w f q q p p} {j m q f b y c i z k y q p l e a}
84 {m x o n y f g} {y c n x n q j i y c l h b r q z}
85 {v o z l n p c} {g n j n t b b x n c l d a g j v}
86 {z n a y f b t k k t d b z a v} {r p c n r u k u}
87 {b q t x z e c w} {q a o a l o a h i m j r}
88 {j f h o x x a z g b a f a m i b} {j z c z y x e x w t}
89 {t c t p r s u c q n} {z x l i k n f q l n t}
90 {w t d q j g m r f k n} {l e w f w w a l y q k i q t p c t}
91 {c b o k l i c b s j n m b l} {y f p q o w g}
92 {f y d j o q t c c q m f j s t} {f h e d y m o k}
93 {k x j r m a d o i z j} {r t t t f e b r x i v j v g o}
94 {s f e a e t i h h d q p z t q} {b k m k w h c}
95 {h b n j t k i h o q u} {w n g i t o k c a m y p f l x c p}
96 {f c x p y r b m o l m o a} {p c a q s u n n x d c f a o}
97 {u h h k m n k} {u b v n u a o c}
98 {s p e t c z d f n w f} {l s f j b l c e s h}
99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o}
}
foreach {tn2 sql} {
1 {}
2 {BEGIN}
} {
reset_db
fts5_aux_test_functions db
do_execsql_test 1.$tn2.0 {
CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%);
INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
}
execsql $sql
do_test 1.$tn2.1.1 {
foreach {id x y} $data {
execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
}
execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
} {}
#-------------------------------------------------------------------------
#
do_execsql_test 1.$tn2.integrity {
INSERT INTO xx(xx) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
foreach {tn expr} {
1.1 "a AND b"
1.2 "a OR b"
1.3 "o"
1.4 "b q"
1.5 "e a e"
1.6 "m d g q q b k b w f q q p p"
1.7 "l o o l v v k"
1.8 "a"
1.9 "b"
1.10 "c"
1.11 "no"
1.12 "L O O L V V K"
1.13 "a AND b AND c"
1.14 "x:a"
2.1 "x:a"
2.2 "y:a"
2.3 "x:b"
2.4 "y:b"
3.1 "{x}:a"
3.2 "{y}:a"
3.3 "{x}:b"
3.4 "{y}:b"
4.1 "{x y}:a"
4.2 "{y x}:a"
4.3 "{x x}:b"
4.4 "{y y}:b"
5.1 {{"x" "y"}:a}
5.2 {{"y" x}:a}
5.3 {{x "x"}:b}
5.4 {{"y" y}:b}
6.1 "b + q"
6.2 "e + a + e"
6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
6.4 "l + o + o + l + v + v + k"
6.5 "L + O + O + L + V + V + K"
7.1 "a+b AND c"
7.2 "d+c AND u"
7.3 "d+c AND u+d"
7.4 "a+b OR c"
7.5 "d+c OR u"
7.6 "d+c OR u+d"
8.1 "NEAR(a b)"
8.2 "NEAR(r c)"
8.2 { NEAR(r c, 5) }
8.3 { NEAR(r c, 3) }
8.4 { NEAR(r c, 2) }
8.5 { NEAR(r c, 0) }
8.6 { NEAR(a b c) }
8.7 { NEAR(a b c, 8) }
8.8 { x : NEAR(r c) }
8.9 { y : NEAR(r c) }
9.1 { NEAR(r c) }
9.2 { NEAR(r c, 5) }
9.3 { NEAR(r c, 3) }
9.4 { NEAR(r c, 2) }
9.5 { NEAR(r c, 0) }
9.6 { NEAR(a b c) }
9.7 { NEAR(a b c, 8) }
9.8 { x : NEAR(r c) }
9.9 { y : NEAR(r c) }
9.10 { x : "r c" }
9.11 { y : "r c" }
9.12 { a AND b }
9.13 { a AND b AND c }
9.14a { a }
9.14b { a OR b }
9.15 { a OR b AND c }
9.16 { c AND b OR a }
9.17 { c AND (b OR a) }
9.18 { c NOT (b OR a) }
9.19 { (c NOT b) OR (a AND d) }
} {
if {[fts5_expr_ok $expr xx]==0} {
do_test 1.$tn2.$tn.OMITTED { list } [list]
continue
}
set res [fts5_query_data $expr xx]
do_execsql_test 1.$tn2.$tn.[llength $res].asc {
SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx)
FROM xx WHERE xx match $expr
} $res
set res [fts5_query_data $expr xx DESC]
do_execsql_test 1.$tn2.$tn.[llength $res].desc {
SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx)
FROM xx WHERE xx match $expr ORDER BY 1 DESC
} $res
}
}
}
do_execsql_test 2.1 {
SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}
do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0}
do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2}
foreach {tn expr tclexpr} {
1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
} {
do_execsql_test 2.3.$tn {
SELECT fts5_expr_tcl($expr, 'N $x')
} [list $tclexpr]
}
finish_test

245
ext/fts5/test/fts5ad.test Normal file
View File

@ -0,0 +1,245 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# More specifically, the focus is on testing prefix queries, both with and
# without prefix indexes.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ad
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%);
INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');
INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is');
INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the');
} {}
foreach {tn match res} {
1 {c*} {1}
2 {i*} {3 2}
3 {t*} {3 1}
4 {r*} {3 1}
} {
do_execsql_test 1.$tn {
SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid DESC
} $res
}
foreach {tn match res} {
5 {c*} {1}
6 {i*} {2 3}
7 {t*} {1 3}
8 {r*} {1 3}
} {
do_execsql_test 1.$tn {
SELECT rowid FROM yy WHERE yy MATCH $match
} $res
}
foreach {T create} {
2 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
3 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
4 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
BEGIN;
}
5 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
BEGIN;
}
} {
do_test $T.1 {
execsql { DROP TABLE IF EXISTS t1 }
execsql $create
} {}
do_test $T.1 {
foreach {rowid a b} {
0 {fghij uvwxyz klmn pq uvwx} {klmn f fgh uv fghij klmno}
1 {uv f abcd abcd fghi} {pq klm uv uv fgh uv a}
2 {klmn klm pqrs fghij uv} {f k uvw ab abcd pqr uv}
3 {ab pqrst a fghi ab pqr fg} {k klmno a fg abcd}
4 {abcd pqrst uvwx a fgh} {f klmno fghij kl pqrst}
5 {uvwxyz k abcde u a} {uv k k kl klmn}
6 {uvwxyz k klmn pqrst uv} {fghi pqrs abcde u k}
7 {uvwxy klmn u p pqrst fgh} {p f fghi abcd uvw kl uv}
8 {f klmno pqrst uvwxy pqrst} {uv abcde klm pq pqr}
9 {f abcde a uvwxyz pqrst} {fghij abc k uvwx pqr fghij uvwxy}
10 {ab uv f fg pqrst uvwxy} {fgh p uv k abc klm uvw}
11 {pq klmno a uvw abcde uvwxyz} {fghij pq uvwxyz pqr fghi}
12 {fgh u pq fgh uvw} {uvw pqr f uvwxy uvwx}
13 {uvwx klmn f fgh abcd pqr} {uvw k fg uv klm abcd}
14 {ab uvwx pqrst pqr uvwxyz pqrs} {uvwxyz abcde ab ab uvw abcde}
15 {abc abcde uvwxyz abc kl k pqr} {klm k k klmno u fgh}
16 {fghi abcd fghij uv uvwxyz ab uv} {klmn pqr a uvw fghi}
17 {abc pqrst fghi uvwx uvw klmn fghi} {ab fg pqr pqrs p}
18 {pqr kl a fghij fgh fg kl} {pqr uvwxyz uvw abcd uvwxyz}
19 {fghi fghi pqr kl fghi f} {klmn u u klmno klmno}
20 {abc pqrst klmno kl pq uvwxy} {abc k fghi pqrs klm}
21 {a pqr uvwxyz uv fghi a fgh} {abc pqrs pqrst pq klm}
22 {klm abc uvwxyz klm pqrst} {fghij k pq pqr u klm fghij}
23 {p klm uv p a a} {uvwxy klmn uvw abcde pq}
24 {uv fgh fg pq uvwxy u uvwxy} {pqrs a uvw p uvwx uvwxyz fg}
25 {fghij fghi klmn abcd pq kl} {fghi abcde pqrs abcd fgh uvwxy}
26 {pq fgh a abc klmno klmn} {fgh p k p fg fghij}
27 {fg pq kl uvwx fghij pqrst klmn} {abcd uvw abcd fghij f fghij}
28 {uvw fghi p fghij pq fgh uvwx} {k fghij abcd uvwx pqr fghi}
29 {klm pq abcd pq f uvwxy} {pqrst p fghij pqr p}
30 {ab uvwx fg uvwx klmn klm} {klmn klmno fghij klmn klm}
31 {pq k pqr abcd a pqrs} {abcd abcd uvw a abcd klmno ab}
32 {pqrst u abc pq klm} {abc kl uvwxyz fghij u fghi p}
33 {f uvwxy u k f uvw uvwx} {pqrs uvw fghi fg pqrst klm}
34 {pqrs pq fghij uvwxyz pqr} {ab abc abc uvw f pq f}
35 {uvwxy ab uvwxy klmno kl pqrs} {abcde uvw pqrs uvwx k k}
36 {uvwxyz k ab abcde abc uvw} {uvw abcde uvw klmn uv klmn}
37 {k kl uv abcde uvwx fg u} {u abc uvwxy k fg abcd}
38 {fghi pqrst fghi pqr pqrst uvwx} {u uv uvwx fghi abcde}
39 {k pqrst k uvw fg pqrst fghij} {uvwxy ab kl klmn uvwxyz abcde}
40 {fg uvwxy pqrs klmn uvwxyz klm p} {k uv ab fghij fgh k pqrs}
41 {uvwx abc f pq uvwxy k} {ab uvwxyz abc f fghij}
42 {uvwxy klmno uvwxyz uvwxyz pqrst} {uv kl kl klmno k f abcde}
43 {abcde ab pqrs fg f fgh} {abc fghij fghi k k}
44 {uvw abcd a ab pqrst klmn fg} {pqrst u uvwx pqrst fghij f pqrst}
45 {uvwxy p kl uvwxyz ab pqrst fghi} {abc f pqr fg a k}
46 {u p f a fgh} {a kl pq uv f}
47 {pqrs abc fghij fg abcde ab a} {p ab uv pqrs kl fghi abcd}
48 {abcde uvwxy pqrst uv abc pqr uvwx} {uvwxy klm uvwxy uvwx k}
49 {fgh klm abcde klmno u} {a f fghij f uvwxyz abc u}
50 {uv uvw uvwxyz uvwxyz uv ab} {uvwx pq fg u k uvwxy}
51 {uvwxy pq p kl fghi} {pqrs fghi pqrs abcde uvwxyz ab}
52 {pqr p uvwxy kl pqrs klmno fghij} {ab abcde abc pqrst pqrs uv}
53 {fgh pqrst p a klmno} {ab ab pqrst pqr kl pqrst}
54 {abcd klm ab uvw a fg u} {f pqr f abcd uv}
55 {u fg uvwxyz k uvw} {abc pqrs f fghij fg pqrs uvwxy}
56 {klm fg p fghi fg a} {uv a fghi uvwxyz a fghi}
57 {uvwxy k abcde fgh f fghi} {f kl klmn f fghi klm}
58 {klm k fgh uvw fgh fghi} {klmno uvwx u pqrst u}
59 {fghi pqr pqrst p uvw fghij} {uv pqrst pqrs pq fghij klm}
60 {uvwx klm uvwxy uv klmn} {p a a abc klmn ab k}
61 {uvwxy uvwx klm uvwx klm} {pqrs ab ab uvwxyz fg}
62 {kl uv uv uvw fg kl k} {abcde uvw fgh uvwxy klm}
63 {a abc fgh u klm abcd} {fgh pqr uv klmn fghij}
64 {klmn k klmn klmno pqrs pqr} {fg kl abcde klmno uvwxy kl pq}
65 {uvwxyz klm fghi abc abcde kl} {uvwxy uvw uvwxyz uvwxyz pq pqrst}
66 {pq klm abc pqrst fgh f} {u abcde pqrst abcde fg}
67 {u pqrst kl u uvw klmno} {u pqr pqrs fgh u p}
68 {abc fghi uvwxy fgh k pq} {uv p uvwx uvwxyz ab}
69 {klmno f uvwxyz uvwxy klmn fg ab} {fgh kl a pqr abcd pqr}
70 {fghi pqrst pqrst uv a} {uvwxy k p uvw uvwx a}
71 {a fghij f p uvw} {klm fg abcd abcde klmno pqrs}
72 {uv uvwx uvwx uvw klm} {uv fghi klmno uvwxy uvw}
73 {kl uvwxy ab f pq klm u} {uvwxy klmn klm abcd pq fg k}
74 {uvw pqrst abcd uvwxyz ab} {fgh fgh klmn abc pq}
75 {uvwxyz klm pq abcd klmno pqr uvwxyz} {kl f a fg pqr klmn}
76 {uvw uvwxy pqr k pqrst kl} {uvwxy abc uvw uvw u}
77 {fgh klm u uvwxyz f uvwxy abcde} {uv abcde klmno u u ab}
78 {klmno abc pq pqr fgh} {p uv abcd fgh abc u k}
79 {fg pqr uvw pq uvwx} {uv uvw fghij pqrs fg p}
80 {abcd pqrs uvwx uvwxy uvwx} {u uvw pqrst pqr abcde pqrs kl}
81 {uvwxyz klm pq uvwxy fghij} {p pq klm fghij u a a}
82 {uvwx k uvwxyz klmno pqrst kl} {abcde p f pqrst abcd uvwxyz p}
83 {abcd abcde klm pqrst uvwxyz} {uvw pqrst u p uvwxyz a pqrs}
84 {k klm abc uv uvwxy klm klmn} {k abc pqr a abc p kl}
85 {klmn abcd pqrs p pq klm a} {klmn kl ab uvw pq}
86 {klmn a pqrs abc uvw pqrst} {a pqr kl klm a k f}
87 {pqrs ab uvwx uvwxy a pqr f} {fg klm uvwx pqr pqr}
88 {klmno ab k kl u uvwxyz} {uv kl uvw fghi uv uvw}
89 {pq fghi pqrst klmn uvwxy abc pqrs} {fg f f fg abc abcde klm}
90 {kl a k fghi uvwx fghi u} {ab uvw pqr fg a p abc}
91 {uvwx pqrs klmno ab fgh uvwx} {pqr uvwx abc kl f klmno kl}
92 {fghij pq pqrs fghij f pqrst} {u abcde fg pq pqr fgh k}
93 {fgh u pqrs abcde klmno abc} {abc fg pqrst pqr abcde}
94 {uvwx p abc f pqr p} {k pqrs kl klm abc fghi klm}
95 {kl p klmno uvwxyz klmn} {fghi ab a fghi pqrs kl}
96 {pqr fgh pq uvwx a} {uvw klm klmno fg uvwxy uvwx}
97 {fg abc uvwxyz fghi pqrst pq} {abc k a ab abcde f}
98 {uvwxy fghi uvwxy u abcde abcde uvw} {klmn uvwx pqrs uvw uvwxy abcde}
99 {pq fg fghi uvwx uvwx fghij uvwxy} {klmn klmn f abc fg a}
} {
execsql {
INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b);
}
}
} {}
proc prefix_query {prefixlist} {
set ret [list]
db eval {SELECT rowid, a, b FROM t1 ORDER BY rowid DESC} {
set bMatch 1
foreach pref $prefixlist {
if { [lsearch -glob $a $pref]<0 && [lsearch -glob $b $pref]<0 } {
set bMatch 0
break
}
}
if {$bMatch} { lappend ret $rowid }
}
return $ret
}
do_execsql_test $T.integrity {
INSERT INTO t1(t1) VALUES('integrity-check');
}
foreach {bAsc sql} {
1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC}
} {
foreach {tn prefix} {
1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*}
6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*}
11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*}
16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*}
21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*}
27 {x*}
28 {a f*} 29 {a* f*} 30 {a* fghij*}
} {
set res [prefix_query $prefix]
if {$bAsc} {
set res [lsort -integer -increasing $res]
}
set n [llength $res]
if {$T==5} breakpoint
do_execsql_test $T.$bAsc.$tn.$n $sql $res
}
}
catchsql COMMIT
}
}
finish_test

312
ext/fts5/test/fts5ae.test Normal file
View File

@ -0,0 +1,312 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ae
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_execsql_test 1.1 {
INSERT INTO t1 VALUES('hello', 'world');
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1}
do_execsql_test 1.2 {
INSERT INTO t1 VALUES('world', 'hello');
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2}
do_execsql_test 1.3 {
INSERT INTO t1 VALUES('world', 'world');
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2}
do_execsql_test 1.4.1 {
INSERT INTO t1 VALUES('hello', 'hello');
}
do_execsql_test 1.4.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2 4}
fts5_aux_test_functions db
#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
}
do_execsql_test 2.1 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'm' ORDER BY rowid;
} {
1 {0.0.5 0.1.0 0.1.2}
2 {0.0.7 0.1.5}
}
do_execsql_test 2.2 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'u OR q' ORDER BY rowid;
} {
1 {0.0.0}
2 {1.0.2 1.0.10}
}
if {[detail_is_full]} {
do_execsql_test 2.3 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
1 {0.1.3 0.1.7}
}
}
#-------------------------------------------------------------------------
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
INSERT INTO t3 VALUES( 'r c', '');
}
if {[detail_is_full]} {
do_execsql_test 3.1 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
}
do_execsql_test 3.2 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
2 {0.0.0 1.0.1}
}
}
do_execsql_test 3.3 {
INSERT INTO t3
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
SELECT rowid, fts5_test_poslist(t3)
FROM t3 WHERE t3 MATCH 'a OR b AND c';
} {
1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2}
3 0.0.5
}
#-------------------------------------------------------------------------
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t4
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}
do_execsql_test 4.1 {
SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
1 0.0.5
}
#-------------------------------------------------------------------------
# Test that the xColumnSize() and xColumnAvgsize() APIs work.
#
reset_db
fts5_aux_test_functions db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
INSERT INTO t5 VALUES('', 'a');
INSERT INTO t5 VALUES('a', '');
}
do_execsql_test 5.2 {
SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {1 0}
2 {0 1}
1 {4 6}
}
do_execsql_test 5.3 {
SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {a {}}
2 {{} a}
1 {{a b c d} {e f g h i j}}
}
do_execsql_test 5.4 {
SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {5 7}
2 {5 7}
1 {5 7}
}
do_execsql_test 5.5 {
INSERT INTO t5 VALUES('x y z', 'v w x y z');
SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
ORDER BY rowid DESC;
} {
3 {8 12}
2 {8 12}
1 {8 12}
}
#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
fts5_aux_test_functions db
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}
do_execsql_test 6.2 {
SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*'
} {
1 {{there are more} {things in heaven and earth}}
2 {{horatio than are} {dreamt of in your philosophy}}
}
#-------------------------------------------------------------------------
# Test the xQueryPhrase() API
#
reset_db
fts5_aux_test_functions db
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%);
}
do_test 7.2 {
foreach {x y} {
{q i b w s a a e l o} {i b z a l f p t e u}
{b a z t a l o x d i} {b p a d b f h d w y}
{z m h n p p u i e g} {v h d v b x j j c z}
{a g i m v a u c b i} {p k s o t l r t b m}
{v v c j o d a s c p} {f f v o k p o f o g}
} {
execsql {INSERT INTO t7 VALUES($x, $y)}
}
execsql { SELECT count(*) FROM t7 }
} {5}
foreach {tn q res} {
1 a {{4 2}}
2 b {{3 4}}
3 c {{2 1}}
4 d {{2 2}}
5 {a AND b} {{4 2} {3 4}}
6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
} {
do_execsql_test 7.3.$tn {
SELECT fts5_test_queryphrase(t7) FROM t7 WHERE t7 MATCH $q LIMIT 1
} [list $res]
}
do_execsql_test 7.4 {
SELECT fts5_test_rowcount(t7) FROM t7 WHERE t7 MATCH 'a';
} {5 5 5 5}
#do_execsql_test 7.4 {
# SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
#} {5 5 5 5}
#
#-------------------------------------------------------------------------
#
do_test 8.1 {
execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) }
foreach {rowid x y} {
0 {A o} {o o o C o o o o o o o o}
1 {o o B} {o o o C C o o o o o o o}
2 {A o o} {o o o o D D o o o o o o}
3 {o B} {o o o o o D o o o o o o}
4 {E o G} {H o o o o o o o o o o o}
5 {F o G} {I o J o o o o o o o o o}
6 {E o o} {H o J o o o o o o o o o}
7 {o o o} {o o o o o o o o o o o o}
9 {o o o} {o o o o o o o o o o o o}
} {
execsql { INSERT INTO t8(rowid, x, y) VALUES($rowid, $x, $y) }
}
} {}
foreach {tn q res} {
1 {a} {0 2}
2 {b} {3 1}
3 {c} {1 0}
4 {d} {2 3}
5 {g AND (e OR f)} {5 4}
6 {j AND (h OR i)} {5 6}
} {
do_execsql_test 8.2.$tn.1 {
SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY bm25(t8);
} $res
do_execsql_test 8.2.$tn.2 {
SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank;
} $res
do_execsql_test 8.2.$tn.3 {
SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank;
} $res
}
#-------------------------------------------------------------------------
# Test xPhraseCount() for some different queries.
#
do_test 9.1 {
execsql { CREATE VIRTUAL TABLE t9 USING fts5(x) }
foreach x {
"a b c" "d e f"
} {
execsql { INSERT INTO t9 VALUES($x) }
}
} {}
foreach {tn q cnt} {
1 {a AND b} 2
2 {a OR b} 2
3 {a OR b OR c} 3
4 {NEAR(a b)} 2
} {
do_execsql_test 9.2.$tn {
SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
} $cnt
}
}
finish_test

148
ext/fts5/test/fts5af.test Normal file
View File

@ -0,0 +1,148 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# More specifically, the tests in this file focus on the built-in
# snippet() function.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5af
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
proc do_snippet_test {tn doc match res} {
uplevel #0 [list set v1 $doc]
uplevel #0 [list set v2 $match]
do_execsql_test $tn.1 {
DELETE FROM t1;
INSERT INTO t1 VALUES($v1, NULL);
SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
} [list $res]
do_execsql_test $tn.2 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, $v1);
SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
} [list $res]
do_execsql_test $tn.3 {
DELETE FROM t1;
INSERT INTO t1 VALUES($v1, NULL);
SELECT snippet(t1, -1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2
ORDER BY rank DESC;
} [list $res]
}
foreach {tn doc res} {
1.1 {X o o o o o o} {[X] o o o o o o}
1.2 {o X o o o o o} {o [X] o o o o o}
1.3 {o o X o o o o} {o o [X] o o o o}
1.4 {o o o X o o o} {o o o [X] o o o}
1.5 {o o o o X o o} {o o o o [X] o o}
1.6 {o o o o o X o} {o o o o o [X] o}
1.7 {o o o o o o X} {o o o o o o [X]}
2.1 {X o o o o o o o} {[X] o o o o o o...}
2.2 {o X o o o o o o} {o [X] o o o o o...}
2.3 {o o X o o o o o} {o o [X] o o o o...}
2.4 {o o o X o o o o} {o o o [X] o o o...}
2.5 {o o o o X o o o} {...o o o [X] o o o}
2.6 {o o o o o X o o} {...o o o o [X] o o}
2.7 {o o o o o o X o} {...o o o o o [X] o}
2.8 {o o o o o o o X} {...o o o o o o [X]}
3.1 {X o o o o o o o o} {[X] o o o o o o...}
3.2 {o X o o o o o o o} {o [X] o o o o o...}
3.3 {o o X o o o o o o} {o o [X] o o o o...}
3.4 {o o o X o o o o o} {o o o [X] o o o...}
3.5 {o o o o X o o o o} {...o o o [X] o o o...}
3.6 {o o o o o X o o o} {...o o o [X] o o o}
3.7 {o o o o o o X o o} {...o o o o [X] o o}
3.8 {o o o o o o o X o} {...o o o o o [X] o}
3.9 {o o o o o o o o X} {...o o o o o o [X]}
4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
6.1 {X o o o X o o o} {[X] o o o [X] o o...}
6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
} {
do_snippet_test 1.$tn $doc X $res
}
if {[detail_is_full]} {
foreach {tn doc res} {
1.1 {X Y o o o o o} {[X Y] o o o o o}
1.2 {o X Y o o o o} {o [X Y] o o o o}
1.3 {o o X Y o o o} {o o [X Y] o o o}
1.4 {o o o X Y o o} {o o o [X Y] o o}
1.5 {o o o o X Y o} {o o o o [X Y] o}
1.6 {o o o o o X Y} {o o o o o [X Y]}
2.1 {X Y o o o o o o} {[X Y] o o o o o...}
2.2 {o X Y o o o o o} {o [X Y] o o o o...}
2.3 {o o X Y o o o o} {o o [X Y] o o o...}
2.4 {o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o X Y} {...o o o o o [X Y]}
3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
} {
do_snippet_test 2.$tn $doc "X + Y" $res
}
}
} ;# foreach_detail_mode
finish_test

145
ext/fts5/test/fts5ag.test Normal file
View File

@ -0,0 +1,145 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ag
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# This file attempts to verify that the extension APIs work with
# "ORDER BY rank" queries. This is done by comparing the results of
# the fts5_test() function when run with queries of the form:
#
# ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC]
#
# and
#
# ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC]
#
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%);
}
do_test 1.1 {
foreach {x y z} {
{j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w}
{m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v}
{m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n}
{j e m v k p e c j m} {m p v z d x l n i a} {v p u p m t p q i f}
{v r w l e e t d z p} {c s b w k m n k o u} {w g y f v w v w v p}
{k d g o u j p z n o} {t g e q l z i g b j} {f i q q j y h b g h}
{j s w x o t j b t m} {v a v v r t x c q a} {r t k x w u l h a g}
{j y b i u d e m d w} {y s o j h i n a u p} {n a g b u c w e b m}
{b c k s c w j p w b} {m o c o w o b d q q} {n t y o y z y r z e}
{p n q l e l h z q c} {n s e i h c v b b u} {m p d i t a o o f f}
{k c o n v e z l b m} {s m n i n s d e s u} {t a u e q d a o u c}
{h d t o i a g b b p} {k x c i g f g b b k} {x f i v n a n n j i}
{f z k r b u s k z e} {n z v z w l e r h t} {t i s v v a v p n s}
{k f e c t z r e f d} {f m g r c w q k b v} {v y s y f r b f e f}
{z r c t d q q h x b} {u c g z n z u v s s} {y t n f f x b f d x}
{u n p n u t i m e j} {p j j d m f k p m z} {d o l v c o e a h w}
{h o q w t f v i c y} {c q u n r z s l l q} {z x a q w s b w s y}
{y m s x k i m n x c} {b i a n v h z n k a} {w l q p b h h g d y}
{z v s j f p v l f w} {c s b i z e k i g c} {x b v d w j f e d z}
{r k k j e o m k g b} {h b d c h m y b t u} {u j s h k z c u d y}
{v h i v s y z i k l} {d t m w q w c a z p} {r s e s x v d w k b}
{u r e q j y h o o s} {x x z r x y t f j s} {k n h x i i u e c v}
{q l f d a p w l q o} {y z q w j o p b o v} {s u h z h f d f n l}
{q o e o x x l g q i} {j g m h q q w c d b} {o m d h w a g b f n}
{m x k t s s y l v a} {j x t c a u w b w g} {n f j b v x y p u t}
{u w k a q b u w k w} {a h j u o w f s k p} {j o f s h y t j h g}
{x v b l m t l m h l} {t p y i y i q b q a} {k o o z w a c h c f}
{j g c d k w b d t v} {a k v c m a v h v p} {i c a i j g h l j h}
{l m v l c z j b p b} {z p z f l n k i b a} {j v q k g i x g i b}
{m c i w u z m i s z} {i z r f n l q z k w} {x n b p b q r g i z}
{d g i o o x l f x d} {r t m f b n q y c b} {i u g k w x n m p o}
{t o s i q d z x d t} {v a k s q z j c o o} {z f n n r l y w v v}
{w k h d t l j g n n} {r z m v y b l n c u} {v b v s c l n k g v}
{m a g r a b u u n z} {u y l h v w v k b f} {x l p g i s j f x v}
{v s g x k z a k a r} {l t g v j q l k p l} {f h n a x t v s t y}
{z u v u x p s j y t} {g b q e e g l n w g} {e n p j i g j f u r}
{q z l t w o l m p e} {t s g h r p r o t z} {y b f a o n u m z g}
{d t w n y b o g f o} {d a j e r l g g s h} {d z e l w q l t h f}
{f l u w q v x j a h} {f n u l l d m h h w} {d x c c e r o d q j}
{b y f q s q f u l g} {u z w l f d b i a g} {m v q b g u o z e z}
{h z p t s e x i v m} {l h q m e o x x x j} {e e d n p r m g j f}
{k h s g o n s d a x} {u d t t s j o v h a} {z r b a e u v o e s}
{m b b g a f c p a t} {w c m j o d b l g e} {f p j p m o s y v j}
{c r n h d w c a b l} {s g e u s d n j b g} {b o n a x a b x y l}
{r h u x f c d z n o} {x y l g u m i i w d} {t f h b z v r s r g}
{t i o r b v g g p a} {d x l u q k m o s u} {j f h t u n z u k m}
{g j t y d c n j y g} {w e s k v c w i g t} {g a h r g v g h r o}
{e j l a q j g i n h} {d z k c u p n u p p} {t u e e v z v r r g}
{l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j}
{i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a}
} {
execsql { INSERT INTO t1 VALUES($x, $y, $z) }
}
set {} {}
} {}
fts5_aux_test_functions db
proc do_fts5ag_test {tn E} {
set q1 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
set q2 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}
set res [execsql $q1]
set expected [execsql $q2]
uplevel [list do_test $tn.1 [list set {} $res] $expected]
append q1 " DESC"
append q2 " DESC"
set res [execsql $q1]
set expected [execsql $q2]
uplevel [list do_test $tn.2 [list set {} $res] $expected]
}
foreach {tn expr} {
2.1 a
2.2 b
2.3 c
2.4 d
3.0 {a AND b}
3.1 {a OR b}
3.2 {b OR c AND d}
} {
do_fts5ag_test $tn $expr
}
if {[detail_is_full]} {
foreach {tn expr} {
4.1 {"m m"}
4.2 {e + s}
4.3 {NEAR(c d)}
} {
do_fts5ag_test $tn $expr
}
}
} ;# foreach_detail_mode
finish_test

170
ext/fts5/test/fts5ah.test Normal file
View File

@ -0,0 +1,170 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ah
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
#-------------------------------------------------------------------------
# This file contains tests for very large doclists.
#
set Y [list]
set W [list]
do_test 1.0 {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) }
execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
set v {w w w w w w w w w w w w w w w w w w w w}
execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) }
for {set i 1} {$i <= 10000} {incr i} {
set v {x x x x x x x x x x x x x x x x x x x x}
if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
execsql { INSERT INTO t1 VALUES($v) }
}
set v {w w w w w w w w w w w w w w w w w w w w}
execsql { INSERT INTO t1 VALUES($v) }
} {}
do_execsql_test 1.1.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w'
} [lsort -integer -incr $W]
do_execsql_test 1.1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'x* AND w*'
} [lsort -integer -incr $W]
do_execsql_test 1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x'
} [lsort -integer -incr $Y]
do_execsql_test 1.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
proc reads {} {
db one {SELECT t1 FROM t1 WHERE t1 MATCH '*reads'}
}
proc execsql_reads {sql} {
set nRead [reads]
execsql $sql
expr [reads] - $nRead
}
do_test 1.4 {
set nRead [reads]
execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
set nReadX [expr [reads] - $nRead]
#puts -nonewline "(nReadX=$nReadX)"
if {[detail_is_full]} { set expect 1000 }
if {[detail_is_col]} { set expect 250 }
if {[detail_is_none]} { set expect 80 }
expr $nReadX>$expect
} {1}
do_test 1.5 {
set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }]
set bwd [execsql_reads {
SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC
}]
expr {$bwd < $fwd + 12}
} {1}
foreach {tn q res} "
1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x' } [list $W]
2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } [list $W]
3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W]
4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y]
" {
if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue
if {[detail_is_full]} { set ratio 8 }
if {[detail_is_col]} { set ratio 4 }
if {[detail_is_none]} { set ratio 2 }
do_test 1.6.$tn.1 {
set n [execsql_reads $q]
#puts -nonewline "(n=$n nReadX=$nReadX)"
expr {$n < ($nReadX / $ratio)}
} {1}
do_test 1.6.$tn.2 {
set n [execsql_reads "$q ORDER BY rowid DESC"]
#puts -nonewline "(n=$n nReadX=$nReadX)"
expr {$n < ($nReadX / $ratio)}
} {1}
do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res]
do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res]
}
#-------------------------------------------------------------------------
# Now test that adding range constraints on the rowid field reduces the
# number of pages loaded from disk.
#
foreach {tn fraction tail cnt} {
1 0.6 {rowid > 5000} 5000
2 0.2 {rowid > 9000} 1000
3 0.2 {rowid < 1000} 999
4 0.2 {rowid BETWEEN 4000 AND 5000} 1001
5 0.6 {rowid >= 5000} 5001
6 0.2 {rowid >= 9000} 1001
7 0.2 {rowid <= 1000} 1000
8 0.6 {rowid > '5000'} 5000
9 0.2 {rowid > '9000'} 1000
10 0.1 {rowid = 444} 1
} {
set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail"
set n [execsql_reads $q]
set ret [llength [execsql $q]]
# Because the position lists for 'x' are quite long in this db, the
# advantage is a bit smaller in detail=none mode. Update $fraction to
# reflect this.
if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] }
do_test "1.7.$tn.asc.(n=$n ret=$ret)" {
expr {$n < ($fraction*$nReadX) && $ret==$cnt}
} {1}
set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC"
set n [execsql_reads $q]
set ret [llength [execsql $q]]
do_test "1.7.$tn.desc.(n=$n ret=$ret)" {
expr {$n < 2*$fraction*$nReadX && $ret==$cnt}
} {1}
}
do_execsql_test 1.8.1 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text';
} {10000}
do_execsql_test 1.8.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
} {10000}
} ;# foreach_detail_mode
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
finish_test

58
ext/fts5/test/fts5ai.test Normal file
View File

@ -0,0 +1,58 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, it tests transactions and savepoints
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ai
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%);
} {}
do_execsql_test 1.1 {
BEGIN;
INSERT INTO t1 VALUES('a b c');
INSERT INTO t1 VALUES('d e f');
SAVEPOINT one;
INSERT INTO t1 VALUES('g h i');
SAVEPOINT two;
INSERT INTO t1 VALUES('j k l');
ROLLBACK TO one;
INSERT INTO t1 VALUES('m n o');
SAVEPOINT two;
INSERT INTO t1 VALUES('p q r');
RELEASE one;
SAVEPOINT one;
INSERT INTO t1 VALUES('s t u');
ROLLBACK TO one;
COMMIT;
}
do_execsql_test 1.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
finish_test

69
ext/fts5/test/fts5aj.test Normal file
View File

@ -0,0 +1,69 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, this tests that, provided the amount of data remains
# constant, the FTS index does not grow indefinitely as rows are inserted
# and deleted,
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aj
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc doc {} {
set dict [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
set res [list]
for {set i 0} {$i < 20} {incr i} {
lappend res [lindex $dict [expr int(rand() * 26)]]
}
set res
}
proc structure {} {
set val [db one {SELECT fts5_decode(rowid,block) FROM t1_data WHERE rowid=10}]
foreach lvl [lrange $val 1 end] {
lappend res [expr [llength $lvl]-2]
}
set res
}
expr srand(0)
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
}
for {set iTest 0} {$iTest < 50000} {incr iTest} {
if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } }
set new [doc]
execsql { INSERT INTO t1 VALUES($new) }
if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] }
if {0==($iTest % 1000)} {
set sz [db one {SELECT count(*) FROM t1_data}]
set s [structure]
do_execsql_test 1.$iTest.$sz.{$s} {
INSERT INTO t1(t1) VALUES('integrity-check')
}
}
}
do_execsql_test 2.0 { INSERT INTO t1(t1) VALUES('integrity-check') }
finish_test

150
ext/fts5/test/fts5ak.test Normal file
View File

@ -0,0 +1,150 @@
# 2014 November 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, the auxiliary function "highlight".
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ak
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
INSERT INTO ft1 VALUES('i d d a g i b g d d');
INSERT INTO ft1 VALUES('h d b j c c g a c a');
INSERT INTO ft1 VALUES('e j a e f h b f h h');
INSERT INTO ft1 VALUES('j f h d g h i b d f');
INSERT INTO ft1 VALUES('d c j d c j b c g e');
INSERT INTO ft1 VALUES('i a d e g j g d a a');
INSERT INTO ft1 VALUES('j f c e d a h j d b');
INSERT INTO ft1 VALUES('i c c f a d g h j e');
INSERT INTO ft1 VALUES('i d i g c d c h b f');
INSERT INTO ft1 VALUES('g d a e h a b c f j');
CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%);
INSERT INTO ft2 VALUES('a b c d e f g h i j');
}
do_execsql_test 1.2 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e';
} {
{[e] j a [e] f h b f h h}
{d c j d c j b c g [e]}
{i a d [e] g j g d a a}
{j f c [e] d a h j d b}
{i c c f a d g h j [e]}
{g d a [e] h a b c f j}
}
do_execsql_test 1.3 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e'
} {
{[e] j a [e] f h b f h h}
{d c j d c j b c g [e]}
{i a d [e] g j g d a a}
{j f c [e] d a h j d b}
{i c c f a d g h j [e]}
{g d a [e] h a b c f j}
}
do_execsql_test 1.4 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
} {
{a b c [d] e [f] g h i j}
}
do_execsql_test 1.5 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
} {
{a b c [d] e [f] g h i j}
}
#-------------------------------------------------------------------------
# Tests below this point require detail=full.
#-------------------------------------------------------------------------
if {[detail_is_full]==0} continue
do_execsql_test 2.1 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
} {
{[h d] b j c c g a c a}
{j f [h d] g h i b d f}
}
do_execsql_test 2.2 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
} {
{i [d d] a g i b g [d d]}
}
do_execsql_test 2.3 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d';
} {
{i [d d] a g i b g [d d]}
}
do_execsql_test 2.4 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e'
} {{a [b c d e] f g h i j}}
do_execsql_test 2.5 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g'
} {
{a [b c d] [e f g] h i j}
}
do_execsql_test 2.6 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c'
} {
{a [b c d] e f g h i j}
}
do_execsql_test 2.7 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
} {
{a [b c d e] f g h i j}
}
#-------------------------------------------------------------------------
# The example from the docs.
#
do_execsql_test 3.1 {
-- Assuming this:
CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%);
INSERT INTO ft VALUES('a b c x c d e');
INSERT INTO ft VALUES('a b c c d e');
INSERT INTO ft VALUES('a b c d e');
-- The following SELECT statement returns these three rows:
-- '[a b c] x [c d e]'
-- '[a b c] [c d e]'
-- '[a b c d e]'
SELECT highlight(ft, 0, '[', ']') FROM ft WHERE ft MATCH 'a+b+c AND c+d+e';
} {
{[a b c] x [c d e]}
{[a b c] [c d e]}
{[a b c d e]}
}
}
finish_test

300
ext/fts5/test/fts5al.test Normal file
View File

@ -0,0 +1,300 @@
# 2014 November 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# Specifically, this function tests the %_config table.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5al
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
SELECT * FROM ft1_config;
} {version 4}
do_execsql_test 1.2 {
INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
SELECT * FROM ft1_config;
} {pgsz 32 version 4}
do_execsql_test 1.3 {
INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
SELECT * FROM ft1_config;
} {pgsz 64 version 4}
#--------------------------------------------------------------------------
# Test the logic for parsing the rank() function definition.
#
foreach {tn defn} {
1 "fname()"
2 "fname(1)"
3 "fname(1,2)"
4 "fname(null,NULL,nUlL)"
5 " fname ( null , NULL , nUlL ) "
6 "fname('abc')"
7 "fname('a''bc')"
8 "fname('''abc')"
9 "fname('abc''')"
7 "fname( 'a''bc' )"
8 "fname('''abc' )"
9 "fname( 'abc''' )"
10 "fname(X'1234ab')"
11 "myfunc(1.2)"
12 "myfunc(-1.0)"
13 "myfunc(.01,'abc')"
} {
do_execsql_test 2.1.$tn {
INSERT INTO ft1(ft1, rank) VALUES('rank', $defn);
}
}
foreach {tn defn} {
1 ""
2 "fname"
3 "fname(X'234ab')"
4 "myfunc(-1.,'abc')"
} {
do_test 2.2.$tn {
catchsql { INSERT INTO ft1(ft1, rank) VALUES('rank', $defn) }
} {1 {SQL logic error or missing database}}
}
#-------------------------------------------------------------------------
# Assorted tests of the tcl interface for creating extension functions.
#
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1 VALUES('q w e r t y');
INSERT INTO t1 VALUES('y t r e w q');
}
proc argtest {cmd args} { return $args }
sqlite3_fts5_create_function db argtest argtest
do_execsql_test 3.2.1 {
SELECT argtest(t1, 123) FROM t1 WHERE t1 MATCH 'q'
} {123 123}
do_execsql_test 3.2.2 {
SELECT argtest(t1, 123, 456) FROM t1 WHERE t1 MATCH 'q'
} {{123 456} {123 456}}
proc rowidtest {cmd} { $cmd xRowid }
sqlite3_fts5_create_function db rowidtest rowidtest
do_execsql_test 3.3.1 {
SELECT rowidtest(t1) FROM t1 WHERE t1 MATCH 'q'
} {1 2}
proc insttest {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
lappend res [$cmd xInst $i]
}
set res
}
sqlite3_fts5_create_function db insttest insttest
do_execsql_test 3.4.1 {
SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q'
} {
{{0 0 0}}
{{0 0 5}}
}
if {[detail_is_full]} {
do_execsql_test 3.4.2 {
SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
} {
{{1 0 1}}
{{0 0 2} {1 0 4}}
}
}
proc coltest {cmd} {
list [$cmd xColumnSize 0] [$cmd xColumnText 0]
}
sqlite3_fts5_create_function db coltest coltest
do_execsql_test 3.5.1 {
SELECT coltest(t1) FROM t1 WHERE t1 MATCH 'q'
} {
{6 {q w e r t y}}
{6 {y t r e w q}}
}
#-------------------------------------------------------------------------
# Tests for remapping the "rank" column.
#
# 4.1.*: Mapped to a function with no arguments.
# 4.2.*: Mapped to a function with one or more arguments.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e');
INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q');
INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t');
INSERT INTO t2 VALUES('h j g t r e l s g s', 'k q k c i i c k n s');
INSERT INTO t2 VALUES('b l k h d n n n m i', 'p t i a r b t q o l');
INSERT INTO t2 VALUES('k r i l j b g i p a', 't q c h a i m g n l');
INSERT INTO t2 VALUES('a e c q n m o m d g', 'l c t g i s q g q e');
INSERT INTO t2 VALUES('b o j h f o g b p e', 'r t l h s b g i c p');
INSERT INTO t2 VALUES('s q k f q b j g h f', 'n m a o p e i e k t');
INSERT INTO t2 VALUES('o q g g q c o k a b', 'r t k p t f t h p c');
}
proc firstinst {cmd} {
foreach {p c o} [$cmd xInst 0] {}
expr $c*100 + $o
}
sqlite3_fts5_create_function db firstinst firstinst
do_execsql_test 4.1.1 {
SELECT rowid, firstinst(t2) FROM t2 WHERE t2 MATCH 'a' ORDER BY rowid ASC
} {
1 0 2 4 3 6 5 103
6 9 7 0 9 102 10 8
}
do_execsql_test 4.1.2 {
SELECT rowid, rank FROM t2
WHERE t2 MATCH 'a' AND rank MATCH 'firstinst()'
ORDER BY rowid ASC
} {
1 0 2 4 3 6 5 103
6 9 7 0 9 102 10 8
}
do_execsql_test 4.1.3 {
SELECT rowid, rank FROM t2
WHERE t2 MATCH 'a' AND rank MATCH 'firstinst()'
ORDER BY rank DESC
} {
5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0
}
do_execsql_test 4.1.4 {
INSERT INTO t2(t2, rank) VALUES('rank', 'firstinst()');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rowid ASC
} {
1 0 2 4 3 6 5 103
6 9 7 0 9 102 10 8
}
do_execsql_test 4.1.5 {
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rank DESC
} {
5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0
}
do_execsql_test 4.1.6 {
INSERT INTO t2(t2, rank) VALUES('rank', 'firstinst ( ) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'a' ORDER BY rank DESC
} {
5 103 9 102 6 9 10 8 3 6 2 4 1 0 7 0
}
proc rowidplus {cmd ival} {
expr [$cmd xRowid] + $ival
}
sqlite3_fts5_create_function db rowidplus rowidplus
if {[detail_is_full]} {
do_execsql_test 4.2.1 {
INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
} {
10 110
}
do_execsql_test 4.2.2 {
INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
} {
10 121
}
do_execsql_test 4.2.3 {
SELECT rowid, rank FROM t2
WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
} {
10 122
}
}
proc rowidmod {cmd imod} {
expr [$cmd xRowid] % $imod
}
sqlite3_fts5_create_function db rowidmod rowidmod
do_execsql_test 4.3.1 {
CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%);
INSERT INTO t3 VALUES('a one');
INSERT INTO t3 VALUES('a two');
INSERT INTO t3 VALUES('a three');
INSERT INTO t3 VALUES('a four');
INSERT INTO t3 VALUES('a five');
INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()');
}
do_execsql_test 4.3.2 {
SELECT * FROM t3
WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)'
ORDER BY rank ASC
} {
{a four} {a one} {a five} {a two} {a three}
}
do_execsql_test 4.3.3 {
SELECT *, rank FROM t3
WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)'
ORDER BY rank ASC
} {
{a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2
}
do_execsql_test 4.3.4 {
SELECT * FROM t3('a', 'rowidmod(4)') ORDER BY rank ASC;
} {
{a four} {a one} {a five} {a two} {a three}
}
do_execsql_test 4.3.5 {
SELECT *, rank FROM t3('a', 'rowidmod(3)') ORDER BY rank ASC
} {
{a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2
}
do_catchsql_test 4.4.3 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)'
} {1 {no such function: xyz}}
do_catchsql_test 4.4.4 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}
} ;# foreach_detail_mode
finish_test

103
ext/fts5/test/fts5alter.test Normal file
View File

@ -0,0 +1,103 @@
# 2015 Jun 10
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The tests in this file focus on renaming FTS5 tables using the
# "ALTER TABLE ... RENAME TO ..." command
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5alter
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Test renaming regular, contentless and columnsize=0 FTS5 tables.
#
do_execsql_test 1.1.0 {
CREATE VIRTUAL TABLE "a x" USING fts5(a, x);
INSERT INTO "a x" VALUES('a a a', 'x x x');
ALTER TABLE "a x" RENAME TO "x y";
}
do_execsql_test 1.1.1 {
SELECT * FROM "x y";
SELECT rowid FROM "x y" WHERE "x y" MATCH 'a'
} {{a a a} {x x x} 1}
do_execsql_test 1.2.0 {
CREATE VIRTUAL TABLE "one/two" USING fts5(one, columnsize=0);
INSERT INTO "one/two"(rowid, one) VALUES(456, 'd d d');
ALTER TABLE "one/two" RENAME TO "three/four";
}
do_execsql_test 1.2.1 {
SELECT * FROM "three/four";
SELECT rowid FROM "three/four" WHERE "three/four" MATCH 'd'
} {{d d d} 456}
do_execsql_test 1.3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(val, content='');
INSERT INTO t1(rowid, val) VALUES(-1, 'drop table');
INSERT INTO t1(rowid, val) VALUES(-2, 'drop view');
ALTER TABLE t1 RENAME TO t2;
}
do_execsql_test 1.3.1 {
SELECT rowid, * FROM t2;
SELECT rowid FROM t2 WHERE t2 MATCH 'table'
} {-2 {} -1 {} -1}
#-------------------------------------------------------------------------
# Test renaming an FTS5 table within a transaction.
#
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE zz USING fts5(a);
INSERT INTO zz(rowid, a) VALUES(-56, 'a b c');
BEGIN;
INSERT INTO zz(rowid, a) VALUES(-22, 'a b c');
ALTER TABLE zz RENAME TO yy;
SELECT rowid FROM yy WHERE yy MATCH 'a + b + c';
COMMIT;
} {-56 -22}
do_execsql_test 2.2 {
BEGIN;
ALTER TABLE yy RENAME TO ww;
INSERT INTO ww(rowid, a) VALUES(-11, 'a b c');
SELECT rowid FROM ww WHERE ww MATCH 'a + b + c';
} {-56 -22 -11}
do_execsql_test 2.3 {
ROLLBACK;
SELECT rowid FROM yy WHERE yy MATCH 'a + b + c';
} {-56 -22}
#-------------------------------------------------------------------------
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE abc USING fts5(a);
INSERT INTO abc(rowid, a) VALUES(1, 'a');
BEGIN;
INSERT INTO abc(rowid, a) VALUES(2, 'a');
}
breakpoint
do_execsql_test 3.2 {
SELECT rowid FROM abc WHERE abc MATCH 'a';
} {1 2}
do_execsql_test 3.3 {
COMMIT;
SELECT rowid FROM abc WHERE abc MATCH 'a';
} {1 2}
finish_test

345
ext/fts5/test/fts5auto.test Normal file
View File

@ -0,0 +1,345 @@
# 2015 May 30
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file contains automatically generated tests for various types
# of MATCH expressions.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5auto
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
set data {
-4026076
{n x w k b p x b n t t d s} {f j j s p j o}
{w v i y r} {i p y s}
{a o q v e n q r} {q v g u c y a z y}
3995120
{c} {e e w d t}
{x c p f w r s m l r b f d} {g g u e}
{s n u t d v p d} {b k v p m f}
-2913881
{k m} {a}
{w r j z n s l} {m j i w d t w e l}
{z n c} {v f b m}
174082
{j} {q l w u k e q v r i}
{j l} {u v w r s p e l}
{p i k j k q c t g u s} {g u y s m h q k g t e s o r}
3207399
{e t} {}
{p} {y v r b e k h d e v}
{t m w z b g q t s d d h} {o n v u i t o y k j}
182399
{} {m o s o x d y f a x j z}
{x n z r c d} {n r x i r}
{s v s} {a u}
768994
{e u t q v z q k j p u f j p} {y c b}
{p s d} {k n w p m p p}
{u o x s d} {f s g r d b d r m m m z y}
3931037
{c j p x e} {c n k t h z o i}
{} {r r p j k x w q}
{o r d z d} {x}
3105748
{p x r u} {x i s w o t o g x m z i w}
{q x m z} {h c j w b l y w x c o}
{m b k v} {t v q i s a d x}
-2501642
{o u d n w o m o o s n t r h} {k p e u y p e z d j r y g}
{v b b h d d q y j q j} {a m w d t}
{y e f n} {a k x i x}
-1745680
{z u w j f d b f} {j w i c g u d w e}
{m f p v m a s p v c o s} {s c r z o t w l b e a q}
{m k q} {k b a v o}
-932328
{r v i u m q d r} {f z u v h c m r f g}
{r x r} {k p i d h h w h z u a x}
{k m j p} {h l j a e u c i q x x f x g}
-3923818
{t t p b n u i h e c k} {m z}
{v u d c} {v y y j s g}
{o a f k k q p h g x e n z x} {h d w c o l}
-2145922
{z z l f a l g e d c d h} {j b j p k o o u b q}
{d i g q t f d r h k} {n w g j c x r p t y f l c t}
{d o c u k f o} {r y s x z s p p h g t p y c}
4552917
{j w j y h l k u} {n a}
{y h w c n k} {b}
{w} {z l r t s i m v c y}
2292008
{q v q j w y y x u t} {r q z n h a b o}
{d q y} {y v o e j}
{} {a b h c d l p d x}
1407892
{n j j u q d o a u c f} {r d b w o q n g}
{d e v w s} {v d v o u o x s l s j z y}
{j y w h i f g i h m} {v n z b n y}
-4412544
{g h h r s} {h e r e}
{n q s} {o p z r m l l t}
{p} {f s u o b j}
1209110
{o a a z t t u h j} {z z i r k r}
{i c x q w g v o x z i z p} {q o g k i n z x e d v w v}
{p f v b g f e d n p u c y k} {q z z a i p a a s r e z}
3448977
{i v} {l u x t b o k}
{f h u v p} {k a o y j}
{d m k c j} {v c e r u e f i t}
-4703774
{d h v w u z r e h x o l t} {p s f y w y r q d a m w}
{c h g c g j j f t b i c q} {s e}
{c t q j g f} {v n r w y r a g e j d}
2414151
{s o o s d s k q b f q v p e} {j r o b t o p d l o o x}
{d d k t v e} {}
{t v o d w} {w e q w h y c y y i j b a m}
-3342407
{m c h n e p d o c r w n t} {j d k s p q l}
{t g s r w x j l r z r} {h}
{r q v x i r a n h s} {m y p b v w r a u o g q r}
-993951
{l n p u o j d x t u u c o j} {k r n a r e k v i t o e}
{q f t t a a c z v f} {o n m p v f o e n}
{h z h i p s b j z h} {i t w m k c u g n i}
1575251
{} {z s i j d o x j a r t}
{h g j u j n v e n z} {p z j n n f}
{s q q f d w r l y i z d o m} {b a n d h t b y g h d}
4263668
{q g t h f s} {s g x p f q z i s o f l i}
{q k} {w v h a x n a r b}
{m j a h o b i x k r w z q u} {m t r g j o e q t m p u l}
2487819
{m w g x r n e u t s r} {b x a t u u j c r n}
{j} {w f j r e e y l p}
{o u h b} {o c a c a b v}
167966
{o d b s d o a u m o x y} {c}
{r w d o b v} {z e b}
{i n z a f g z o} {m u b a g}
1948599
{n r g q d j s} {n k}
{l b p d v t k h y y} {u m k e c}
{t b n y o t b} {j w c i r x x}
2941631
{l d p l b g f} {e k e}
{p j} {m c s w t b k n l d x}
{f o v y v l} {c w p s w j w c u t y}
3561104
{d r j j r j i g p} {u}
{g r j q} {z l p d s n f c h t d c v z}
{w r c f s x z y} {g f o k g g}
-2223281
{y e t j j z f p o m m z} {h k o g o}
{m x a t} {l q x l}
{r w k d l s y b} {q g k b}
-4502874
{k k b x k l f} {r}
{} {q m z b k h k u n e z}
{z q g y m y u} {}
1757599
{d p z j y u r} {z p l q w j t j}
{n i r x r y j} {}
{h} {w t d q c x z z x e e}
-4809589
{} {z p x u h i i n g}
{w q s u d b f x n} {l y k b b r x t i}
{n d v j q o t o d p z e} {u r y u v u c}
1068408
{y e} {e g s k e w t p v o b k}
{z c m s} {r u r u h n h b p q g b}
{j k b l} {m c d t s r s q a d b o f}
-1972554
{m s w} {d k v s a r k p a r i v}
{g j z k p} {y k c v r e u o q f i b a}
{i p i} {c z w c y b n z i v}
-2052385
{} {x e u f f g n c i x n e i e}
{} {p s w d x p g}
{} {s j a h n}
2805981
{m x g c w o e} {k g u y r y i u e g g}
{f k j v t x p h x k u} {w i}
{b l f z f v t n} {i u d o d p h s m u}
2507621
{} {u b n l x f n j t}
{u r x l h} {h r l m r}
{d y e n b s q v t k n q q} {x l t v w h a s k}
-3138375
{e o f j y x u w v e w z} {r d q g k n n v r c z n e w}
{l y i q z k j p u f q s k} {c i l l i m a a g a z r x f}
{a v k h m q z b y n z} {q g w c y r r o a}
-457971
{j x a w e c s h f l f} {q}
{j f v j u m d q r v v} {x n v a w}
{i e h d h f u w t t z} {v s u l s v o v i k n e}
2265221
{z t c y w n y r t} {n b a x s}
{q w a v} {a b s d x i g w t e z h}
{t l} {j k r w f f y j o k u}
-3941280
{r x t o z} {f j n z k}
{t x e b t d b k w i s} {j t y h i h}
{y q g n g s u v c z j z n g} {n n g t l p h}
2084745
{z d z d} {j}
{o e k t b k a z l w} {o p i h k c x}
{c r b t i j f} {z e n m}
1265843
{} {j s g j j x u y}
{u q t f} {g o g}
{w o j e d} {w q n a c t q x j}
-2941116
{i n c u o} {f b}
{o m s q d o z a q} {f s v o b b}
{o a z c h r} {j e w h b f z}
-1265441
{p g z q v a o a x a} {s t h}
{w i p o c} {s n d g f z w q o d v v l j}
{y f b i a s v} {u m o z k k s t s d p b l p}
-1989158
{r i c n} {r e w w i n z}
{q u s y b w u g y g f o} {y}
{d} {j x i b x u y d c p v a h}
2391989
{b n w x w f q h p i} {e u b b i n a i o c d g}
{v a z o i e n l x l r} {r u f o r k w m d w}
{k s} {r f e j q p w}
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f);
} {}
fts5_aux_test_functions db
proc do_auto_test {tn tbl expr} {
foreach order {asc desc} {
set res [fts5_poslist_data $expr $tbl $order]
set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
set ::autotest_expr $expr
do_execsql_test $testname [subst -novar {
SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl]
WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
}] $res
}
}
#-------------------------------------------------------------------------
#
for {set fold 0} {$fold < 3} {incr fold} {
switch $fold {
0 { set map {} }
1 { set map {
a a b a c b d b e c f c g d h d
i e j e k f l f m g g g o h p h
q i r i s j t j u k v k w l x l
y m z m
}}
2 { set map {
a a b a c a d a e a f a g a h a
i b j b k b l b m b g b o b p b
q c r c s c t c u c v c w c x c
}}
}
execsql {
BEGIN;
DELETE FROM tt;
}
foreach {rowid a b c d e f} [string map $map $data] {
if {$rowid==-4703774} {
execsql {
INSERT INTO tt(rowid, a, b, c, d, e, f)
VALUES($rowid, $a, $b, $c, $d, $e, $f)
}
}
}
execsql COMMIT
foreach {tn expr} {
A.1 { {a} : x }
A.2 { {a b} : x }
A.3 { {a b f} : x }
A.4 { {f a b} : x }
A.5 { {f a b} : x y }
A.6 { {f a b} : x + y }
A.7 { {c a b} : x + c }
A.8 { {c d} : "l m" }
A.9 { {c e} : "l m" }
A.10 { {a b c a b c a b c f f e} : "l m" }
B.1 { a NOT b }
B.2 { a NOT a:b }
B.3 { a OR (b AND c) }
B.4 { a OR (b AND {a b c}:c) }
B.5 { a OR "b c" }
B.6 { a OR b OR c }
C.1 { a OR (b AND "b c") }
C.2 { a OR (b AND "z c") }
} {
do_auto_test 3.$fold.$tn tt $expr
}
}
proc replace_elems {list args} {
set ret $list
foreach {idx elem} $args {
set ret [lreplace $ret $idx $idx $elem]
}
set ret
}
#-------------------------------------------------------------------------
#
set bigdoc [string trim [string repeat "a " 1000]]
do_test 4.0 {
set a [replace_elems $bigdoc 50 x 950 x]
set b [replace_elems $bigdoc 20 y 21 x 887 x 888 y]
set c [replace_elems $bigdoc 1 z 444 z 789 z]
execsql {
CREATE VIRTUAL TABLE yy USING fts5(c1, c2, c3);
INSERT INTO yy(rowid, c1, c2, c3) VALUES(-56789, $a, $b, $c);
INSERT INTO yy(rowid, c1, c2, c3) VALUES(250, $a, $b, $c);
}
} {}
foreach {tn expr} {
1 x
2 y
3 z
4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
} {
do_auto_test 4.$tn yy $expr
}
finish_test

250
ext/fts5/test/fts5aux.test Normal file
View File

@ -0,0 +1,250 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the auxiliary function APIs.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5aux
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc inst {cmd i} {
$cmd xInst $i
}
sqlite3_fts5_create_function db inst inst
proc colsize {cmd i} {
$cmd xColumnSize $i
}
sqlite3_fts5_create_function db colsize colsize
proc totalsize {cmd i} {
$cmd xColumnTotalSize $i
}
sqlite3_fts5_create_function db totalsize totalsize
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE f1 USING fts5(a, b);
INSERT INTO f1 VALUES('one two', 'two one zero');
INSERT INTO f1 VALUES('one one', 'one one one');
}
do_catchsql_test 1.1 {
SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_catchsql_test 1.2 {
SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 0 1}}}
do_catchsql_test 1.3 {
SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two';
} {0 {{0 1 0}}}
do_catchsql_test 1.4 {
SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_catchsql_test 2.1 {
SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two';
} {1 SQLITE_RANGE}
do_execsql_test 2.2 {
SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero';
} {2 3}
do_execsql_test 2.3 {
SELECT colsize(f1, -1) FROM f1 WHERE f1 MATCH 'zero';
} {5}
do_execsql_test 2.4.1 {
SELECT totalsize(f1, -1) FROM f1 WHERE f1 MATCH 'zero';
} {10}
do_execsql_test 2.4.2 {
SELECT totalsize(f1, 0) FROM f1 WHERE f1 MATCH 'zero';
} {4}
do_execsql_test 2.4.3 {
SELECT totalsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero';
} {6}
do_catchsql_test 2.4.4 {
SELECT totalsize(f1, 2) FROM f1 WHERE f1 MATCH 'zero';
} {1 SQLITE_RANGE}
#-------------------------------------------------------------------------
# Test the xSet and xGetAuxdata APIs with a NULL destructor.
#
proc prevrowid {add cmd} {
set res [$cmd xGetAuxdataInt 0]
set r [$cmd xRowid]
$cmd xSetAuxdataInt $r
return [expr $res + $add]
}
sqlite3_fts5_create_function db prevrowid [list prevrowid 0]
sqlite3_fts5_create_function db prevrowid1 [list prevrowid 1]
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE e5 USING fts5(x);
INSERT INTO e5 VALUES('a b c');
INSERT INTO e5 VALUES('d e f');
INSERT INTO e5 VALUES('a b c');
INSERT INTO e5 VALUES('d e f');
INSERT INTO e5 VALUES('a b c');
}
do_execsql_test 3.1 {
SELECT prevrowid(e5) || '+' || rowid FROM e5 WHERE e5 MATCH 'c'
} {0+1 1+3 3+5}
do_execsql_test 3.2 {
SELECT prevrowid(e5) || '+' || prevrowid1(e5) || '+' || rowid
FROM e5 WHERE e5 MATCH 'e'
} {0+1+2 2+3+4}
#-------------------------------------------------------------------------
# Test that if the xQueryPhrase callback returns other than SQLITE_OK,
# the query is abandoned. And that if it returns an error code other than
# SQLITE_DONE, the error is propagated back to the caller.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE e7 USING fts5(x);
INSERT INTO e7 VALUES('a x a');
INSERT INTO e7 VALUES('b x b');
INSERT INTO e7 VALUES('c x c');
INSERT INTO e7 VALUES('d x d');
INSERT INTO e7 VALUES('e x e');
}
proc xCallback {rowid code cmd} {
set r [$cmd xRowid]
lappend ::cb $r
if {$r==$rowid} { return $code }
return ""
}
proc phrasequery {cmd code} {
set ::cb [list]
$cmd xQueryPhrase 1 [list xCallback [$cmd xRowid] $code]
set ::cb
}
sqlite3_fts5_create_function db phrasequery phrasequery
do_execsql_test 4.1 {
SELECT phrasequery(e7, 'SQLITE_OK') FROM e7 WHERE e7 MATCH 'c x'
} {{1 2 3 4 5}}
do_execsql_test 4.2 {
SELECT phrasequery(e7, 'SQLITE_DONE') FROM e7 WHERE e7 MATCH 'c x'
} {{1 2 3}}
do_catchsql_test 4.3 {
SELECT phrasequery(e7, 'SQLITE_ERROR') FROM e7 WHERE e7 MATCH 'c x'
} {1 SQLITE_ERROR}
#-------------------------------------------------------------------------
# Auxiliary function calls with many cursors in the global cursor list.
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE e9 USING fts5(y);
INSERT INTO e9(rowid, y) VALUES(1, 'i iii');
INSERT INTO e9(rowid, y) VALUES(2, 'ii iv');
INSERT INTO e9(rowid, y) VALUES(3, 'ii');
INSERT INTO e9(rowid, y) VALUES(4, 'i iv');
INSERT INTO e9(rowid, y) VALUES(5, 'iii');
}
proc my_rowid {cmd} { $cmd xRowid }
sqlite3_fts5_create_function db my_rowid my_rowid
foreach {var q} {
s1 i
s2 ii
s3 iii
s4 iv
} {
set sql "SELECT my_rowid(e9) FROM e9 WHERE e9 MATCH '$q'"
set $var [sqlite3_prepare db $sql -1 dummy]
}
do_test 5.1.1 { sqlite3_step $s1 ; sqlite3_column_int $s1 0 } 1
do_test 5.1.2 { sqlite3_step $s2 ; sqlite3_column_int $s2 0 } 2
do_test 5.1.3 { sqlite3_step $s3 ; sqlite3_column_int $s3 0 } 1
do_test 5.1.4 { sqlite3_step $s4 ; sqlite3_column_int $s4 0 } 2
do_test 5.2.1 { sqlite3_step $s1 ; sqlite3_column_int $s1 0 } 4
do_test 5.2.2 { sqlite3_step $s2 ; sqlite3_column_int $s2 0 } 3
do_test 5.2.3 { sqlite3_step $s3 ; sqlite3_column_int $s3 0 } 5
do_test 5.2.4 { sqlite3_step $s4 ; sqlite3_column_int $s4 0 } 4
sqlite3_finalize $s1
sqlite3_finalize $s2
sqlite3_finalize $s3
sqlite3_finalize $s4
#-------------------------------------------------------------------------
# Passing an invalid first argument to an auxiliary function is detected.
#
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE e11 USING fts5(y, z);
INSERT INTO e11(rowid, y, z) VALUES(1, 'a b', 45);
INSERT INTO e11(rowid, y, z) VALUES(2, 'b c', 46);
}
do_catchsql_test 6.1 {
SELECT my_rowid(z) FROM e11 WHERE e11 MATCH 'b'
} {1 {no such cursor: 45}}
do_catchsql_test 6.2 {
SELECT my_rowid(y) FROM e11 WHERE e11 MATCH 'b'
} {1 {no such cursor: 0}}
#-------------------------------------------------------------------------
# Test passing an out-of-range phrase number to xPhraseSize (should
# return 0).
#
proc my_phrasesize {cmd iPhrase} { $cmd xPhraseSize $iPhrase }
sqlite3_fts5_create_function db my_phrasesize my_phrasesize
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('a b c');
}
do_execsql_test 7.2 {
SELECT
my_phrasesize(t1, -1),
my_phrasesize(t1, 0),
my_phrasesize(t1, 1),
my_phrasesize(t1, 2)
FROM t1 WHERE t1 MATCH 'a OR b+c'
} {0 1 2 0}
#-------------------------------------------------------------------------
#
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE x1 USING fts5(a);
}
foreach {tn lRow res} {
4 {"a a a" "b" "a d"} {"[a] [a] [a]" "[a] d"}
1 {"b d" "a b"} {"[b] [d]" "[a] b"}
2 {"d b" "a d"} {"[d] [b]" "[a] d"}
3 {"a a d"} {"[a] [a] d"}
} {
execsql { DELETE FROM x1 }
foreach row $lRow { execsql { INSERT INTO x1 VALUES($row) } }
breakpoint
do_execsql_test 8.$tn {
SELECT highlight(x1, 0, '[', ']') FROM x1 WHERE x1 MATCH 'a OR (b AND d)';
} $res
}
finish_test

115
ext/fts5/test/fts5auxdata.test Normal file
View File

@ -0,0 +1,115 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the fts5 xSetAuxdata() and xGetAuxdata() APIs.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5auxdata
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE f1 USING fts5(a, b);
INSERT INTO f1(rowid, a, b) VALUES(1, 'a', 'b1');
INSERT INTO f1(rowid, a, b) VALUES(2, 'a', 'b2');
INSERT INTO f1(rowid, a, b) VALUES(3, 'a', 'b3');
INSERT INTO f1(rowid, a, b) VALUES(4, 'a', 'b4');
INSERT INTO f1(rowid, a, b) VALUES(5, 'a', 'b5');
}
proc aux_function_1 {cmd tn} {
switch [$cmd xRowid] {
1 {
do_test $tn.1 [list $cmd xGetAuxdata 0 ] {}
$cmd xSetAuxdata "one"
}
2 {
do_test $tn.2 [list $cmd xGetAuxdata 0 ] {one}
$cmd xSetAuxdata "two"
}
3 {
do_test $tn.3 [list $cmd xGetAuxdata 0 ] {two}
}
4 {
do_test $tn.4 [list $cmd xGetAuxdata 1 ] {two}
}
5 {
do_test $tn.5 [list $cmd xGetAuxdata 0 ] {}
}
}
}
sqlite3_fts5_create_function db aux_function_1 aux_function_1
db eval {
SELECT aux_function_1(f1, 1) FROM f1 WHERE f1 MATCH 'a'
ORDER BY rowid ASC
}
proc aux_function_2 {cmd tn inst} {
if {$inst == "A"} {
switch [$cmd xRowid] {
1 {
do_test $tn.1.$inst [list $cmd xGetAuxdata 0 ] {}
$cmd xSetAuxdata "one $inst"
}
2 {
do_test $tn.2.$inst [list $cmd xGetAuxdata 0 ] "one $inst"
$cmd xSetAuxdata "two $inst"
}
3 {
do_test $tn.3.$inst [list $cmd xGetAuxdata 0 ] "two $inst"
}
4 {
do_test $tn.4.$inst [list $cmd xGetAuxdata 1 ] "two $inst"
}
5 {
do_test $tn.5.$inst [list $cmd xGetAuxdata 0 ] {}
}
}
} else {
switch [$cmd xRowid] {
1 {
do_test $tn.1.$inst [list $cmd xGetAuxdata 0 ] "one A"
}
2 {
do_test $tn.2.$inst [list $cmd xGetAuxdata 0 ] "two A"
}
3 {
do_test $tn.3.$inst [list $cmd xGetAuxdata 0 ] "two A"
}
4 {
do_test $tn.4.$inst [list $cmd xGetAuxdata 0 ] {}
}
5 {
do_test $tn.5.$inst [list $cmd xGetAuxdata 0 ] {}
}
}
}
}
sqlite3_fts5_create_function db aux_function_2 aux_function_2
db eval {
SELECT aux_function_2(f1, 2, 'A'), aux_function_2(f1, 2, 'B')
FROM f1 WHERE f1 MATCH 'a'
ORDER BY rowid ASC
}
finish_test

64
ext/fts5/test/fts5bigpl.test Normal file
View File

@ -0,0 +1,64 @@
# 2015 April 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on really large position lists. Those that require
# 4 or 5 byte position-list size varints. Because of the amount of memory
# required, these tests only run on 64-bit platforms.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5bigpl
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
if { $tcl_platform(wordSize)<8 } {
finish_test
return
}
do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x) }
do_test 1.1 {
foreach t {a b c d e f g h i j} {
set doc [string repeat "$t " 1200000]
execsql { INSERT INTO t1 VALUES($doc) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
do_test 1.2 {
execsql { DELETE FROM t1 }
foreach t {"a b" "b a" "c d" "d c"} {
set doc [string repeat "$t " 600000]
execsql { INSERT INTO t1 VALUES($doc) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
# 5-byte varint. This test takes 30 seconds or so on a 2014 workstation.
# The generated database is roughly 635MiB.
#
do_test 2.1...slow {
execsql { DELETE FROM t1 }
foreach t {a} {
set doc [string repeat "$t " 150000000]
execsql { INSERT INTO t1 VALUES($doc) }
}
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
finish_test

67
ext/fts5/test/fts5bigtok.test Normal file
View File

@ -0,0 +1,67 @@
# 2016 Jan 19
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5bigtok
proc rndterm {} {
set L [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
set l [lindex $L [expr int(rand() * [llength $L])]]
string repeat $l [expr int(rand() * 5) + 60]
}
proc rnddoc {n} {
set res [list]
for {set i 0} {$i < $n} {incr i} {
lappend res [rndterm]
}
set res
}
foreach_detail_mode $::testprefix {
db func rnddoc rnddoc
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, row);
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
INSERT INTO t1 SELECT rnddoc(3) FROM s;
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 )
INSERT INTO t1 SELECT rnddoc(3) FROM s;
}
foreach v [db eval {SELECT term FROM t1vocab}] {
set res [db eval {SELECT rowid FROM t1($v)}]
do_execsql_test 1.[string range $v 0 0] {
SELECT rowid FROM t1($v) ORDER BY rowid DESC
} [lsort -integer -decr $res]
}
do_execsql_test 2.0 {
INSERT INTO t1(t1) VALUES('optimize');
}
foreach v [db eval {SELECT term FROM t1vocab}] {
set res [db eval {SELECT rowid FROM t1($v)}]
do_execsql_test 2.[string range $v 0 0] {
SELECT rowid FROM t1($v) ORDER BY rowid DESC
} [lsort -integer -decr $res]
}
}
finish_test

151
ext/fts5/test/fts5columnsize.test Normal file
View File

@ -0,0 +1,151 @@
# 2015 Jun 10
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on fts5 tables with the columnsize=0 option.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5columnsize
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Check that the option can be parsed and that the %_docsize table is
# only created if it is set to true.
#
foreach {tn outcome stmt} {
1 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0) }
2 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=1) }
3 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='0') }
4 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='1') }
5 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='') }
6 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=2) }
7 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0, columnsize=1) }
8 1 { CREATE VIRTUAL TABLE t1 USING fts5(x) }
9 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=11) }
} {
execsql {
DROP TABLE IF EXISTS t1;
}
if {$outcome==2} {
do_catchsql_test 1.$tn.1 $stmt {1 {malformed columnsize=... directive}}
} else {
do_execsql_test 1.$tn.2 $stmt
do_execsql_test 1.$tn.3 {
SELECT count(*) FROM sqlite_master WHERE name = 't1_docsize'
} $outcome
}
}
#-------------------------------------------------------------------------
# Run tests on a table with no %_content or %_docsize backing store.
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x, columnsize=0, content='');
}
do_catchsql_test 2.1 {
INSERT INTO t2 VALUES('a b c d e f');
} {1 {datatype mismatch}}
do_execsql_test 2.2 {
INSERT INTO t2(rowid, x) VALUES(1, 'c d e f');
INSERT INTO t2(rowid, x) VALUES(2, 'c d e f g h');
INSERT INTO t2(rowid, x) VALUES(3, 'a b c d e f g h');
} {}
do_execsql_test 2.3 {
SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
SELECT rowid FROM t2 WHERE t2 MATCH 'h';
} {3 :: 1 2 3 :: 2 3}
do_execsql_test 2.4 {
INSERT INTO t2(t2, rowid, x) VALUES('delete', 2, 'c d e f g h');
SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
SELECT rowid FROM t2 WHERE t2 MATCH 'h';
} {3 :: 1 3 :: 3}
do_execsql_test 2.5 {
INSERT INTO t2(t2) VALUES('delete-all');
SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
SELECT rowid FROM t2 WHERE t2 MATCH 'h';
} {:: ::}
do_execsql_test 2.6 {
INSERT INTO t2(rowid, x) VALUES(1, 'o t t f');
INSERT INTO t2(rowid, x) VALUES(2, 'f s s e');
INSERT INTO t2(rowid, x) VALUES(3, 'n t e t');
}
do_catchsql_test 2.7.1 {
SELECT rowid FROM t2
} {1 {t2: table does not support scanning}}
do_catchsql_test 2.7.2 {
SELECT rowid FROM t2 WHERE rowid=2
} {1 {t2: table does not support scanning}}
do_catchsql_test 2.7.3 {
SELECT rowid FROM t2 WHERE rowid BETWEEN 1 AND 3
} {1 {t2: table does not support scanning}}
do_execsql_test 2.X {
DROP TABLE t2
}
#-------------------------------------------------------------------------
# Test the xColumnSize() API
#
fts5_aux_test_functions db
do_execsql_test 3.1.0 {
CREATE VIRTUAL TABLE t3 USING fts5(x, y UNINDEXED, z, columnsize=0);
INSERT INTO t3 VALUES('a a', 'b b b', 'c');
INSERT INTO t3 VALUES('x a x', 'b b b y', '');
}
do_execsql_test 3.1.1 {
SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a'
} {
1 {2 0 1} 2 {3 0 0}
}
do_execsql_test 3.1.2 {
INSERT INTO t3 VALUES(NULL, NULL, 'a a a a');
DELETE FROM t3 WHERE rowid = 1;
SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a'
} {
2 {3 0 0} 3 {0 0 4}
}
do_execsql_test 3.2.0 {
CREATE VIRTUAL TABLE t4 USING fts5(x, y UNINDEXED, z, columnsize=0, content='');
INSERT INTO t4(rowid, x, y, z) VALUES(1, 'a a', 'b b b', 'c');
INSERT INTO t4(rowid, x, y, z) VALUES(2, 'x a x', 'b b b y', '');
}
do_execsql_test 3.2.1 {
SELECT rowid, fts5_test_columnsize(t4) FROM t4 WHERE t4 MATCH 'a'
} {
1 {-1 0 -1} 2 {-1 0 -1}
}
#-------------------------------------------------------------------------
# Test the integrity-check
#
do_execsql_test 4.1.1 {
CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0);
INSERT INTO t5 VALUES('1 2 3 4');
INSERT INTO t5 VALUES('2 4 6 8');
}
breakpoint
do_execsql_test 4.1.2 {
INSERT INTO t5(t5) VALUES('integrity-check');
}
finish_test

251
ext/fts5/test/fts5config.test Normal file
View File

@ -0,0 +1,251 @@
# 2015 Jan 13
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on the code in fts5_config.c, which is largely concerned
# with parsing the various configuration and CREATE TABLE options.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5config
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Try different types of quote characters.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5('a', "b", [c], `d`);
PRAGMA table_info = t1;
} {
0 a {} 0 {} 0
1 b {} 0 {} 0
2 c {} 0 {} 0
3 d {} 0 {} 0
}
#-------------------------------------------------------------------------
# Syntax errors in the prefix= option.
#
foreach {tn opt} {
1 {prefix=x}
2 {prefix='x'}
3 {prefix='$'}
4 {prefix='1,2,'}
5 {prefix=',1'}
6 {prefix='1,2,3...'}
7 {prefix='1,2,3xyz'}
} {
set res [list 1 {malformed prefix=... directive}]
do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
}
#-------------------------------------------------------------------------
# Syntax errors in the 'rank' option.
#
foreach {tn val} {
1 "f1(xyz)"
2 "f1(zyx)"
3 "f1(nzz)"
4 "f1(x'!!')"
5 "f1(x':;')"
6 "f1(x'[]')"
7 "f1(x'{}')"
8 "f1('abc)"
} {
do_catchsql_test 3.$tn {
INSERT INTO t1(t1, rank) VALUES('rank', $val);
} {1 {SQL logic error or missing database}}
}
#-------------------------------------------------------------------------
# The parsing of SQL literals specified as part of 'rank' options.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE zzz USING fts5(one);
INSERT INTO zzz VALUES('a b c');
}
proc first {cmd A} { return $A }
sqlite3_fts5_create_function db first first
foreach {tn arg} {
1 "123"
2 "'01234567890ABCDEF'"
3 "x'0123'"
4 "x'ABCD'"
5 "x'0123456789ABCDEF'"
6 "x'0123456789abcdef'"
7 "22.5"
8 "-91.5"
9 "-.5"
10 "''''"
11 "+.5"
} {
set func [string map {' ''} "first($arg)"]
do_execsql_test 4.1.$tn "
INSERT INTO zzz(zzz, rank) VALUES('rank', '$func');
SELECT rank IS $arg FROM zzz WHERE zzz MATCH 'a + b + c'
" 1
}
do_execsql_test 4.2 {
INSERT INTO zzz(zzz, rank) VALUES('rank', 'f1()');
} {}
#-------------------------------------------------------------------------
# Misquoting in tokenize= and other options.
#
do_catchsql_test 5.1 {
CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii");
} {1 {parse error in tokenize directive}}
breakpoint
do_catchsql_test 5.2 {
CREATE VIRTUAL TABLE xx USING fts5(x, [y[]);
} {0 {}}
do_catchsql_test 5.3 {
CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
} {1 {unrecognized token: "]"}}
#-------------------------------------------------------------------------
# Errors in prefix= directives.
#
do_catchsql_test 6.2 {
CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001');
} {1 {prefix length out of range (max 999)}}
do_catchsql_test 6.3 {
CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000');
} {1 {prefix length out of range (max 999)}}
do_catchsql_test 6.4 {
CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1 , 1000000');
} {1 {prefix length out of range (max 999)}}
#-------------------------------------------------------------------------
# Duplicate tokenize= and other options.
#
do_catchsql_test 7.1 {
CREATE VIRTUAL TABLE abc USING fts5(a, tokenize=porter, tokenize=ascii);
} {1 {multiple tokenize=... directives}}
do_catchsql_test 7.2 {
CREATE VIRTUAL TABLE abc USING fts5(a, content=porter, content=ascii);
} {1 {multiple content=... directives}}
do_catchsql_test 7.3 {
CREATE VIRTUAL TABLE abc USING fts5(a, content_rowid=porter, content_rowid=a);
} {1 {multiple content_rowid=... directives}}
#-------------------------------------------------------------------------
# Unrecognized option.
#
do_catchsql_test 8.0 {
CREATE VIRTUAL TABLE abc USING fts5(a, nosuchoption=123);
} {1 {unrecognized option: "nosuchoption"}}
do_catchsql_test 8.1 {
CREATE VIRTUAL TABLE abc USING fts5(a, "nosuchoption"=123);
} {1 {parse error in ""nosuchoption"=123"}}
#-------------------------------------------------------------------------
# Errors in:
#
# 9.1.* 'pgsz' options.
# 9.2.* 'automerge' options.
# 9.3.* 'crisismerge' options.
# 9.4.* a non-existant option.
# 9.5.* 'hashsize' options.
#
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE abc USING fts5(a, b);
} {}
do_catchsql_test 9.1.1 {
INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.1.2 {
INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.1.3 {
INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.2.1 {
INSERT INTO abc(abc, rank) VALUES('automerge', -5);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.2.2 {
INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.2.3 {
INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
} {1 {SQL logic error or missing database}}
do_execsql_test 9.2.4 {
INSERT INTO abc(abc, rank) VALUES('automerge', 1);
} {}
do_catchsql_test 9.3.1 {
INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.3.2 {
INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
} {1 {SQL logic error or missing database}}
do_execsql_test 9.3.3 {
INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
} {}
do_execsql_test 9.3.4 {
INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
} {}
do_catchsql_test 9.4.1 {
INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.5.1 {
INSERT INTO abc(abc, rank) VALUES('hashsize', 'not an integer');
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.5.2 {
INSERT INTO abc(abc, rank) VALUES('hashsize', -500000);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.5.3 {
INSERT INTO abc(abc, rank) VALUES('hashsize', 500000);
} {0 {}}
#-------------------------------------------------------------------------
# Too many prefix indexes. Maximum allowed is 31.
#
foreach {tn spec} {
1 {prefix="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
} {
set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)"
do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
}
#-------------------------------------------------------------------------
# errors in the detail= option.
#
foreach {tn opt} {
1 {detail=x}
2 {detail='x'}
3 {detail='$'}
4 {detail='1,2,'}
5 {detail=',1'}
6 {detail=''}
} {
set res [list 1 {malformed detail=... directive}]
do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
}
do_catchsql_test 12.1 {
INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
} {1 {SQL logic error or missing database}}
finish_test

70
ext/fts5/test/fts5conflict.test Normal file
View File

@ -0,0 +1,70 @@
# 2015 October 27
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5conflict
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE TABLE t1(x INTEGER PRIMARY KEY, a, b);
CREATE VIRTUAL TABLE ft USING fts5(a, b, content=t1, content_rowid=x);
}
do_execsql_test 1.1 {
REPLACE INTO ft(rowid, a, b) VALUES(1, 'a b c', 'a b c');
REPLACE INTO t1 VALUES(1, 'a b c', 'a b c');
}
do_execsql_test 1.2 {
INSERT INTO ft(ft) VALUES('integrity-check');
}
do_execsql_test 2.0 {
CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c);
CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content=tbl, content_rowid=a);
CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN
INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
END;
CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN
INSERT INTO fts_idx(fts_idx, rowid, b, c)
VALUES('delete', old.a, old.b, old.c);
END;
CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN
INSERT INTO fts_idx(fts_idx, rowid, b, c)
VALUES('delete', old.a, old.b, old.c);
INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
END;
}
do_execsql_test 2.1 {
PRAGMA recursive_triggers = 1;
INSERT INTO tbl VALUES(1, 'x y z', '1 2 3');
INSERT INTO tbl VALUES(10, 'x y z', '1 2 3');
INSERT INTO tbl VALUES(100, 'x 1 z', '1 y 3');
UPDATE tbl SET b = '1 2 x' WHERE rowid=10;
REPLACE INTO tbl VALUES(1, '4 5 6', '3 2 1');
DELETE FROM tbl WHERE a=100;
INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
}
finish_test

258
ext/fts5/test/fts5content.test Normal file
View File

@ -0,0 +1,258 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests for the content= and content_rowid= options.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5content
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Contentless tables
#
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE f1 USING fts5(a, b, content='');
INSERT INTO f1(rowid, a, b) VALUES(1, 'one', 'o n e');
INSERT INTO f1(rowid, a, b) VALUES(2, 'two', 't w o');
INSERT INTO f1(rowid, a, b) VALUES(3, 'three', 't h r e e');
}
do_execsql_test 1.2 {
SELECT rowid FROM f1 WHERE f1 MATCH 'o';
} {1 2}
do_execsql_test 1.3 {
INSERT INTO f1(a, b) VALUES('four', 'f o u r');
SELECT rowid FROM f1 WHERE f1 MATCH 'o';
} {1 2 4}
do_execsql_test 1.4 {
SELECT rowid, a, b FROM f1 WHERE f1 MATCH 'o';
} {1 {} {} 2 {} {} 4 {} {}}
do_execsql_test 1.5 {
SELECT rowid, highlight(f1, 0, '[', ']') FROM f1 WHERE f1 MATCH 'o';
} {1 {} 2 {} 4 {}}
do_execsql_test 1.6 {
SELECT rowid, highlight(f1, 0, '[', ']') IS NULL FROM f1 WHERE f1 MATCH 'o';
} {1 1 2 1 4 1}
do_execsql_test 1.7 {
SELECT rowid, snippet(f1, -1, '[', ']', '...', 5) IS NULL
FROM f1 WHERE f1 MATCH 'o';
} {1 1 2 1 4 1}
do_execsql_test 1.8 {
SELECT rowid, snippet(f1, 1, '[', ']', '...', 5) IS NULL
FROM f1 WHERE f1 MATCH 'o';
} {1 1 2 1 4 1}
do_execsql_test 1.9 {
SELECT rowid FROM f1;
} {1 2 3 4}
do_execsql_test 1.10 {
SELECT * FROM f1;
} {{} {} {} {} {} {} {} {}}
do_execsql_test 1.11 {
SELECT rowid, a, b FROM f1 ORDER BY rowid ASC;
} {1 {} {} 2 {} {} 3 {} {} 4 {} {}}
do_execsql_test 1.12 {
SELECT a IS NULL FROM f1;
} {1 1 1 1}
do_catchsql_test 1.13 {
DELETE FROM f1 WHERE rowid = 2;
} {1 {cannot DELETE from contentless fts5 table: f1}}
do_catchsql_test 1.14 {
UPDATE f1 SET a = 'a b c' WHERE rowid = 2;
} {1 {cannot UPDATE contentless fts5 table: f1}}
do_execsql_test 1.15 {
INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o');
} {}
do_execsql_test 1.16 {
SELECT rowid FROM f1 WHERE f1 MATCH 'o';
} {1 4}
do_execsql_test 1.17 {
SELECT rowid FROM f1;
} {1 3 4}
#-------------------------------------------------------------------------
# External content tables
#
reset_db
do_execsql_test 2.1 {
-- Create a table. And an external content fts5 table to index it.
CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c);
CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content='tbl', content_rowid='a');
-- Triggers to keep the FTS index up to date.
CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN
INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
END;
CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN
INSERT INTO fts_idx(fts_idx, rowid, b, c)
VALUES('delete', old.a, old.b, old.c);
END;
CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN
INSERT INTO fts_idx(fts_idx, rowid, b, c)
VALUES('delete', old.a, old.b, old.c);
INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
END;
}
do_execsql_test 2.2 {
INSERT INTO tbl VALUES(1, 'one', 'o n e');
INSERT INTO tbl VALUES(NULL, 'two', 't w o');
INSERT INTO tbl VALUES(3, 'three', 't h r e e');
}
do_execsql_test 2.3 {
INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
}
do_execsql_test 2.4 {
DELETE FROM tbl WHERE rowid=2;
INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
}
do_execsql_test 2.5 {
UPDATE tbl SET c = c || ' x y z';
INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
}
do_execsql_test 2.6 {
SELECT * FROM fts_idx WHERE fts_idx MATCH 't AND x';
} {three {t h r e e x y z}}
do_execsql_test 2.7 {
SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx
WHERE fts_idx MATCH 't AND x';
} {{[t] h r e e [x] y z}}
#-------------------------------------------------------------------------
# Quick tests of the 'delete-all' command.
#
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE t3 USING fts5(x, content='');
INSERT INTO t3 VALUES('a b c');
INSERT INTO t3 VALUES('d e f');
}
do_execsql_test 3.2 {
SELECT count(*) FROM t3_docsize;
SELECT count(*) FROM t3_data;
} {2 4}
do_execsql_test 3.3 {
INSERT INTO t3(t3) VALUES('delete-all');
SELECT count(*) FROM t3_docsize;
SELECT count(*) FROM t3_data;
} {0 2}
do_execsql_test 3.4 {
INSERT INTO t3 VALUES('a b c');
INSERT INTO t3 VALUES('d e f');
SELECT rowid FROM t3 WHERE t3 MATCH 'e';
} {2}
do_execsql_test 3.5 {
SELECT rowid FROM t3 WHERE t3 MATCH 'c';
} {1}
do_execsql_test 3.6 {
SELECT count(*) FROM t3_docsize;
SELECT count(*) FROM t3_data;
} {2 4}
do_execsql_test 3.7 {
CREATE VIRTUAL TABLE t4 USING fts5(x);
} {}
do_catchsql_test 3.8 {
INSERT INTO t4(t4) VALUES('delete-all');
} {1 {'delete-all' may only be used with a contentless or external content fts5 table}}
#-------------------------------------------------------------------------
# Test an external content table with a more interesting schema.
#
do_execsql_test 4.1 {
CREATE TABLE x2(a, "key col" PRIMARY KEY, b, c) WITHOUT ROWID;
INSERT INTO x2 VALUES('a b', 1, 'c d' , 'e f');
INSERT INTO x2 VALUES('x y', -40, 'z z' , 'y x');
CREATE VIRTUAL TABLE t2 USING fts5(a, c, content=x2, content_rowid='key col');
INSERT INTO t2(t2) VALUES('rebuild');
}
do_execsql_test 4.2 { SELECT rowid FROM t2 } {-40 1}
do_execsql_test 4.3 { SELECT rowid FROM t2 WHERE t2 MATCH 'c'} {}
do_execsql_test 4.4 { SELECT rowid FROM t2 WHERE t2 MATCH 'a'} {1}
do_execsql_test 4.5 { SELECT rowid FROM t2 WHERE t2 MATCH 'x'} {-40}
do_execsql_test 4.6 { INSERT INTO t2(t2) VALUES('integrity-check') } {}
do_execsql_test 4.7 {
DELETE FROM x2 WHERE "key col" = 1;
INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f');
INSERT INTO t2(t2) VALUES('integrity-check');
}
do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}
#-------------------------------------------------------------------------
# Test that if the 'rowid' field of a 'delete' is not an integer, no
# changes are made to the FTS index.
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=);
INSERT INTO t5(rowid, a, b) VALUES(-1, 'one', 'two');
INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four');
INSERT INTO t5(rowid, a, b) VALUES( 1, 'five', 'six');
}
set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}]
do_execsql_test 5.1 {
INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four');
SELECT md5sum(id, block) FROM t5_data;
} $::checksum
#-------------------------------------------------------------------------
# Check that a contentless table can be dropped.
#
reset_db
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE xx USING fts5(x, y, content="");
SELECT name FROM sqlite_master;
} {xx xx_data xx_idx xx_docsize xx_config}
do_execsql_test 6.2 {
DROP TABLE xx;
SELECT name FROM sqlite_master;
} {}
finish_test

99
ext/fts5/test/fts5corrupt.test Normal file
View File

@ -0,0 +1,99 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests that the FTS5 'integrity-check' command detects
# inconsistencies (corruption) in the on-disk backing tables.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
do_test 1.1 {
db transaction {
for {set i 1} {$i < 200} {incr i} {
set doc [list [string repeat x $i] [string repeat y $i]]
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
}
fts5_level_segs t1
} {1}
db_save
do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
set segid [lindex [fts5_level_segids t1] 0]
do_test 1.3 {
execsql {
DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
}
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
do_test 1.4 {
db_restore_and_reopen
execsql {
UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
rowid = fts5_rowid('segment', $segid, 4);
}
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
db_restore_and_reopen
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
#--------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x);
INSERT INTO t2(t2, rank) VALUES('pgsz', 64);
}
db func rnddoc fts5_rnddoc
do_test 2.1 {
for {set i 0} {$i < 500} {incr i} {
execsql { INSERT INTO t2 VALUES(rnddoc(50)) }
}
execsql { INSERT INTO t2(t2) VALUES('integrity-check') }
} {}
#--------------------------------------------------------------------
# A mundane test - missing row in the %_content table.
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(x);
INSERT INTO t3 VALUES('one o');
INSERT INTO t3 VALUES('two e');
INSERT INTO t3 VALUES('three o');
INSERT INTO t3 VALUES('four e');
INSERT INTO t3 VALUES('five o');
}
do_execsql_test 3.1 {
SELECT * FROM t3 WHERE t3 MATCH 'o'
} {{one o} {three o} {five o}}
do_catchsql_test 3.1 {
DELETE FROM t3_content WHERE rowid = 3;
SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {database disk image is malformed}}
finish_test

272
ext/fts5/test/fts5corrupt2.test Normal file
View File

@ -0,0 +1,272 @@
# 2015 Apr 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests that FTS5 handles corrupt databases (i.e. internal
# inconsistencies in the backing tables) correctly. In this case
# "correctly" means without crashing.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
sqlite3_fts5_may_be_corrupt 1
# Create a simple FTS5 table containing 100 documents. Each document
# contains 10 terms, each of which start with the character "x".
#
expr srand(0)
db func rnddoc fts5_rnddoc
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
set mask [expr 31 << 31]
if 1 {
# Test 1:
#
# For each page in the t1_data table, open a transaction and DELETE
# the t1_data entry. Then run:
#
# * an integrity-check, and
# * unless the deleted block was a b-tree node, a query for "t1 MATCH 'x*'"
#
# and check that the corruption is detected in both cases. The
# rollback the transaction.
#
# Test 2:
#
# Same thing, except instead of deleting a row from t1_data, replace its
# blob content with integer value 14.
#
foreach {tno stmt} {
1 { DELETE FROM t1_data WHERE rowid=$rowid }
2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid }
} {
set tn 0
foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] {
incr tn
#if {$tn!=224} continue
do_test 1.$tno.$tn.1.$rowid {
execsql { BEGIN }
execsql $stmt
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
if {($rowid & $mask)==0} {
# Node is a leaf node, not a b-tree node.
do_catchsql_test 1.$tno.$tn.2.$rowid {
SELECT rowid FROM t1 WHERE t1 MATCH 'x*'
} {1 {database disk image is malformed}}
}
do_execsql_test 1.$tno.$tn.3.$rowid {
ROLLBACK;
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
}
}
# Using the same database as the 1.* tests.
#
# Run N-1 tests, where N is the number of bytes in the rightmost leaf page
# of the fts index. For test $i, truncate the rightmost leafpage to $i
# bytes. Then test both the integrity-check detects the corruption.
#
# Also tested is that "MATCH 'x*'" does not crash and sometimes reports
# corruption. It may not report the db as corrupt because truncating the
# final leaf to some sizes may create a valid leaf page.
#
set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
set all [db eval {SELECT rowid FROM t1}]
for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
do_execsql_test 2.$i.1 {
BEGIN;
UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
}
do_catchsql_test 2.$i.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_test 2.$i.3 {
set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}]
expr {
$res=="1 {database disk image is malformed}"
|| $res=="0 {$all}"
}
} 1
do_execsql_test 2.$i.4 {
ROLLBACK;
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
}
#-------------------------------------------------------------------------
# Test that corruption in leaf page headers is detected by queries that use
# doclist-indexes.
#
set doc "A B C D E F G H I J "
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE x3 USING fts5(tt);
INSERT INTO x3(x3, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000)
INSERT INTO x3
SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
}
foreach {tn hdr} {
1 "\x00\x00\x00\x00"
2 "\xFF\xFF\xFF\xFF"
3 "\x44\x45"
} {
set tn2 0
set nCorrupt 0
set nCorrupt2 0
foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
do_test 3.$tn.$tn2.1 {
execsql BEGIN
set fd [db incrblob main x3_data block $rowid]
fconfigure $fd -encoding binary -translation binary
set existing [read $fd [string length $hdr]]
seek $fd 0
puts -nonewline $fd $hdr
close $fd
set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}]
if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
set {} 1
} {1}
if {($tn2 % 10)==0 && $existing != $hdr} {
do_test 3.$tn.$tn2.2 {
catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
}
execsql ROLLBACK
}
do_test 3.$tn.x { expr $nCorrupt>0 } 1
}
#--------------------------------------------------------------------
#
set doc "A B C D E F G H I J "
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x4 USING fts5(tt);
INSERT INTO x4(x4, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)
INSERT INTO x4
SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
}
foreach {tn nCut} {
1 1
2 10
} {
set tn2 0
set nCorrupt 0
foreach rowid [db eval {SELECT rowid FROM x4_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
do_test 4.$tn.$tn2 {
execsql {
BEGIN;
UPDATE x4_data SET block = substr(block, 1, length(block)-$nCut)
WHERE id = $rowid;
}
set res [catchsql {
SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC
}]
if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
set {} 1
} {1}
execsql ROLLBACK
}
# do_test 4.$tn.x { expr $nCorrupt>0 } 1
}
}
set doc [string repeat "A B C " 1000]
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE x5 USING fts5(tt);
INSERT INTO x5(x5, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10)
INSERT INTO x5 SELECT $doc FROM ii;
}
foreach {tn hdr} {
1 "\x00\x01"
} {
set tn2 0
set nCorrupt 0
foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
do_test 5.$tn.$tn2 {
execsql BEGIN
set fd [db incrblob main x5_data block $rowid]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd $hdr
close $fd
catchsql { INSERT INTO x5(x5) VALUES('integrity-check') }
set {} {}
} {}
execsql ROLLBACK
}
}
#--------------------------------------------------------------------
reset_db
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE x5 USING fts5(tt);
INSERT INTO x5 VALUES('a');
INSERT INTO x5 VALUES('a a');
INSERT INTO x5 VALUES('a a a');
INSERT INTO x5 VALUES('a a a a');
UPDATE x5_docsize SET sz = X'' WHERE id=3;
}
proc colsize {cmd i} {
$cmd xColumnSize $i
}
sqlite3_fts5_create_function db colsize colsize
do_catchsql_test 6.2 {
SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a'
} {1 SQLITE_CORRUPT_VTAB}
sqlite3_fts5_may_be_corrupt 0
finish_test

408
ext/fts5/test/fts5corrupt3.test Normal file
View File

@ -0,0 +1,408 @@
# 2015 Apr 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests that FTS5 handles corrupt databases (i.e. internal
# inconsistencies in the backing tables) correctly. In this case
# "correctly" means without crashing.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt3
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
sqlite3_fts5_may_be_corrupt 1
proc create_t1 {} {
expr srand(0)
db func rnddoc fts5_rnddoc
db eval {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
}
if 1 {
# Create a simple FTS5 table containing 100 documents. Each document
# contains 10 terms, each of which start with the character "x".
#
do_test 1.0 { create_t1 } {}
do_test 1.1 {
# Pick out the rowid of the right-most b-tree leaf in the new segment.
set rowid [db one {
SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
}]
set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
set {} {}
} {}
for {set i 0} {$i < $L} {incr i} {
do_test 1.2.$i {
catchsql {
BEGIN;
UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid;
INSERT INTO t1(t1) VALUES('integrity-check');
}
} {1 {database disk image is malformed}}
catchsql ROLLBACK
}
#-------------------------------------------------------------------------
# Test that trailing bytes appended to the averages record are ignored.
#
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x);
INSERT INTO t2 VALUES(rnddoc(10));
INSERT INTO t2 VALUES(rnddoc(10));
SELECT length(block) FROM t2_data WHERE id=1;
} {2}
do_execsql_test 2.2 {
UPDATE t2_data SET block = block || 'abcd' WHERE id=1;
SELECT length(block) FROM t2_data WHERE id=1;
} {6}
do_execsql_test 2.2 {
INSERT INTO t2 VALUES(rnddoc(10));
SELECT length(block) FROM t2_data WHERE id=1;
} {2}
#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}
do_execsql_test 3.1 {
SELECT count(*) FROM t1_data;
} {105}
proc do_3_test {tn} {
set i 0
foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] {
incr i
do_test $tn.$i {
db eval BEGIN
db eval {DELETE FROM t1_data WHERE rowid = $::rowid}
list [
catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg
] $msg
} {1 {database disk image is malformed}}
catch { db eval ROLLBACK }
}
}
do_3_test 3.2
do_execsql_test 3.3 {
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 SELECT x FROM t1;
INSERT INTO t1(t1) VALUES('optimize');
} {}
do_3_test 3.4
do_test 3.5 {
execsql {
DELETE FROM t1;
INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
}
for {set i 0} {$i < 1000} {incr i} {
set rnd [expr int(rand() * 1000)]
set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]]
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
} {}
do_3_test 3.6
do_test 3.7 {
execsql {
INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
INSERT INTO t1 SELECT x FROM t1;
INSERT INTO t1(t1) VALUES('optimize');
}
} {}
do_3_test 3.8
do_test 3.9 {
execsql {
DELETE FROM t1;
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
for {set i 0} {$i < 100} {incr i} {
set rnd [expr int(rand() * 100)]
set doc "x[string repeat $rnd 20]"
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
} {}
do_3_test 3.10
#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
do_test 4.0 {
execsql {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
for {set i 0} {$i < 100} {incr i} {
set rnd [expr int(rand() * 100)]
set doc "x[string repeat $rnd 20]"
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
execsql { INSERT INTO t1(t1) VALUES('optimize') }
} {}
set nErr 0
for {set i 1} {1} {incr i} {
set struct [db one {SELECT block FROM t1_data WHERE id=10}]
binary scan $struct c* var
set end [lindex $var end]
if {$end<=$i} break
lset var end [expr $end - $i]
set struct [binary format c* $var]
db eval {
BEGIN;
UPDATE t1_data SET block = $struct WHERE id=10;
}
do_test 4.1.$i {
incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
set {} {}
} {}
catch { db eval ROLLBACK }
}
do_test 4.1.x { expr $nErr>45 } 1
#-------------------------------------------------------------------------
#
# The first argument passed to this command must be a binary blob
# containing an FTS5 leaf page. This command returns a copy of this
# blob, with the pgidx of the leaf page replaced by a single varint
# containing value $iVal.
#
proc rewrite_pgidx {blob iVal} {
binary scan $blob SS off1 szLeaf
if {$iVal<0 || $iVal>=128} {
error "$iVal out of range!"
} else {
set pgidx [binary format c $iVal]
}
binary format a${szLeaf}a* $blob $pgidx
}
reset_db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 40);
BEGIN;
INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj');
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1(x1) VALUES('optimize');
COMMIT;
}
#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b }
#
db func rewrite_pgidx rewrite_pgidx
set i 0
foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
foreach val {2 100} {
do_test 5.2.$val.[incr i] {
catchsql {
BEGIN;
UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid;
SELECT rowid FROM x1 WHERE x1 MATCH 'xa*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xb*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xc*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xd*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xe*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xf*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xg*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xh*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xi*';
}
set {} {}
} {}
catch { db eval ROLLBACK }
}
}
#------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('bbbbb ccccc');
SELECT quote(block) FROM t1_data WHERE rowid>100;
} {X'000000180630626262626201020201056363636363010203040A'}
do_execsql_test 6.1.1 {
UPDATE t1_data SET block =
X'000000180630626262626201020201056161616161010203040A'
WHERE rowid>100;
}
do_catchsql_test 6.1.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------
reset_db
do_execsql_test 6.2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 VALUES('aa bb cc dd ee');
SELECT pgno, quote(term) FROM t1_idx;
} {2 X'' 4 X'3064'}
do_execsql_test 6.2.1 {
UPDATE t1_idx SET term = X'3065' WHERE pgno=4;
}
do_catchsql_test 6.2.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------
reset_db
do_execsql_test 6.3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('abc abcdef abcdefghi');
SELECT quote(block) FROM t1_data WHERE id>100;
} {X'0000001C043061626301020204036465660102030703676869010204040808'}
do_execsql_test 6.3.1 {
BEGIN;
UPDATE t1_data SET block =
X'0000001C043061626301020204036465660102035003676869010204040808'
------------------------------------------^^---------------------
WHERE id>100;
}
do_catchsql_test 6.3.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 6.3.3 {
ROLLBACK;
BEGIN;
UPDATE t1_data SET block =
X'0000001C043061626301020204036465660102030750676869010204040808'
--------------------------------------------^^-------------------
WHERE id>100;
}
do_catchsql_test 6.3.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 6.3.4 {
ROLLBACK;
BEGIN;
UPDATE t1_data SET block =
X'0000001C043061626301020204036465660102030707676869010204040850'
--------------------------------------------------------------^^-
WHERE id>100;
}
do_catchsql_test 6.3.5 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 6.3.6 {
ROLLBACK;
BEGIN;
UPDATE t1_data SET block =
X'0000001C503061626301020204036465660102030707676869010204040808'
----------^^-----------------------------------------------------
WHERE id>100;
}
do_catchsql_test 6.3.5 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#------------------------------------------------------------------------
#
reset_db
proc rnddoc {n} {
set map [list a b c d]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc "x[lindex $map [expr int(rand()*4)]]"
}
set doc
}
db func rnddoc rnddoc
do_test 7.0 {
execsql {
CREATE VIRTUAL TABLE t5 USING fts5(x);
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5(t5) VALUES('optimize');
}
} {}
do_test 7.1 {
foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
db eval BEGIN
db eval {DELETE FROM t5_data WHERE rowid = $i}
set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
if {$r != "1 {database disk image is malformed}"} { error $r }
db eval ROLLBACK
}
} {}
}
#------------------------------------------------------------------------
# Corruption within the structure record.
#
reset_db
do_execsql_test 8.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
INSERT INTO t1 VALUES('one', 'two');
}
do_test 9.1.1 {
set blob "12345678" ;# cookie
append blob "0105" ;# 1 level, total of 5 segments
append blob "06" ;# write counter
append blob "0002" ;# first level has 0 segments merging, 2 other.
append blob "450108" ;# first segment
execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
} {}
do_catchsql_test 9.1.2 {
SELECT * FROM t1('one AND two');
} {1 {database disk image is malformed}}
do_test 9.2.1 {
set blob "12345678" ;# cookie
append blob "0205" ;# 2 levels, total of 5 segments
append blob "06" ;# write counter
append blob "0001" ;# first level has 0 segments merging, 1 other.
append blob "450108" ;# first segment
execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
} {}
do_catchsql_test 9.2.2 {
SELECT * FROM t1('one AND two');
} {1 {database disk image is malformed}}
sqlite3_fts5_may_be_corrupt 0
finish_test

244
ext/fts5/test/fts5detail.test Normal file
View File

@ -0,0 +1,244 @@
# 2015 December 18
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5detail
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
fts5_aux_test_functions db
#--------------------------------------------------------------------------
# Simple tests.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8
INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9
INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10
}
do_execsql_test 1.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
foreach {tn match res} {
1 "a:a" {9}
2 "b:g" {1 4 6}
3 "c:h" {1 3 6 8 9 10}
} {
do_execsql_test 1.2.$tn.1 {
SELECT rowid FROM t1($match);
} $res
do_execsql_test 1.2.$tn.2 {
SELECT rowid FROM t1($match || '*');
} $res
}
do_catchsql_test 1.3.1 {
SELECT rowid FROM t1('h + d');
} {1 {fts5: phrase queries are not supported (detail!=full)}}
do_catchsql_test 1.3.2 {
SELECT rowid FROM t1('NEAR(h d)');
} {1 {fts5: NEAR queries are not supported (detail!=full)}}
#-------------------------------------------------------------------------
# integrity-check with both detail= and prefix= options.
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
INSERT INTO t2(a) VALUES('aa ab');
}
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r}
do_execsql_test 2.1 {
INSERT INTO t2(t2) VALUES('integrity-check');
}
do_execsql_test 2.2 {
SELECT fts5_test_poslist(t2) FROM t2('aa');
} {0.0.0}
do_execsql_test 2.3 {
SELECT fts5_test_collist(t2) FROM t2('aa');
} {0.0}
set ::pc 0
#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
#exit
#-------------------------------------------------------------------------
# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
# work with detail=col tables.
#
set data {
1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc}
3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca}
4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca}
5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb}
6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb}
7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc}
8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb}
9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb}
10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa}
11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac}
12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc}
13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc}
14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa}
15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc}
16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc}
17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca}
18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc}
19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc}
20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc}
21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb}
22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc}
23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc}
24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb}
25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc}
26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa}
27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac}
28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa}
29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac}
30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac}
31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba}
32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc}
33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa}
34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac}
35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb}
36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa}
37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb}
38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac}
39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
}
set data {
1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
}
proc matchdata {expr {bAsc 1}} {
set tclexpr [db one {
SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
}]
set res [list]
#puts "$expr -> $tclexpr"
foreach {id x y z} $::data {
set cols [list $x $y $z]
set ::pc 0
#set hits [lsort -command instcompare [eval $tclexpr]]
set hits [eval $tclexpr]
if {[llength $hits]>0} {
lappend res [list $id $hits]
}
}
if {$bAsc} {
set res [lsort -integer -increasing -index 0 $res]
} else {
set res [lsort -integer -decreasing -index 0 $res]
}
return [concat {*}$res]
}
foreach {tn tbl} {
1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) }
} {
reset_db
fts5_aux_test_functions db
execsql $tbl
foreach {id x y z} $data {
execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) }
}
foreach {tn2 expr} {
1 aaa 2 ccc 3 bab 4 aac
5 aa* 6 cc* 7 ba* 8 aa*
9 a* 10 b* 11 c*
} {
set res [matchdata $expr]
do_execsql_test 3.$tn.$tn2.1 {
SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
} $res
do_execsql_test 3.$tn.$tn2.2 {
SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
} $res
}
}
#-------------------------------------------------------------------------
# Simple tests for detail=none tables.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
}
do_catchsql_test 4.1 {
SELECT * FROM t4('a:a')
} {1 {fts5: column queries are not supported (detail=none)}}
#-------------------------------------------------------------------------
# Test that for the same content detail=none uses less space than
# detail=col, and that detail=col uses less space than detail=full
#
reset_db
do_test 5.1 {
foreach {tbl detail} {t1 none t2 col t3 full} {
execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)"
foreach {rowid x y z} $::data {
execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)"
}
}
} {}
do_execsql_test 5.2 {
SELECT
(SELECT sum(length(block)) from t1_data) <
(SELECT sum(length(block)) from t2_data)
} {1}
do_execsql_test 5.3 {
SELECT
(SELECT sum(length(block)) from t2_data) <
(SELECT sum(length(block)) from t3_data)
} {1}
finish_test

200
ext/fts5/test/fts5dlidx.test Normal file
View File

@ -0,0 +1,200 @@
# 2015 April 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on uses of doclist-index records.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5dlidx
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
if { $tcl_platform(wordSize)<8 } {
finish_test
return
}
foreach_detail_mode $testprefix {
proc do_fb_test {tn sql res} {
set res2 [lsort -integer -decr $res]
uplevel [list do_execsql_test $tn.1 $sql $res]
uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}
# This test populates the FTS5 table with $nEntry entries. Rows are
# numbered from 0 to ($nEntry-1). The rowid for row $i is:
#
# ($iFirst + $i*$nStep)
#
# Each document is of the form "a b c a b c a b c...". If the row number ($i)
# is an integer multiple of $spc1, then an "x" token is appended to the
# document. If it is *also* a multiple of $spc2, a "y" token is also appended.
#
proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
do_execsql_test $tn.0 { DELETE FROM t1 }
set xdoc [list]
set ydoc [list]
execsql BEGIN
for {set i 0} {$i < $nEntry} {incr i} {
set rowid [expr $i * $nStep]
set doc [string trim [string repeat "a b c " 100]]
if {($i % $spc1)==0} {
lappend xdoc $rowid
append doc " x"
if {($i % $spc2)==0} {
lappend ydoc $rowid
append doc " y"
}
}
execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
}
execsql COMMIT
breakpoint
do_test $tn.1 {
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
if {[detail_is_full]} {
do_fb_test $tn.5.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
do_fb_test $tn.5.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
}
}
foreach {tn pgsz} {
1 32
2 200
} {
do_execsql_test $tn.0 {
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
}
do_dlidx_test1 1.$tn.1 10 100 10000 0 1000
do_dlidx_test1 1.$tn.2 10 10 10000 0 128
do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970
do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000
do_dlidx_test1 1.$tn.5 10 10 200 0 [expr 1<<55]
do_dlidx_test1 1.$tn.6 10 10 30 0 [expr 1<<58]
}
proc do_dlidx_test2 {tn nEntry iFirst nStep} {
set str [string repeat "a " 500]
execsql {
BEGIN;
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
INSERT INTO t1 VALUES('b a');
WITH iii(ii, i) AS (
SELECT 1, $iFirst UNION ALL
SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry
)
INSERT INTO t1(rowid,x) SELECT i, $str FROM iii;
COMMIT;
}
do_execsql_test $tn.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
} {1}
breakpoint
do_execsql_test $tn.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
} {1}
}
do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
#--------------------------------------------------------------------
#
reset_db
set ::vocab [list \
IteratorpItercurrentlypointstothefirstrowidofadoclist \
Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
pageIfthecurrenttermisthelasttermonthepageloadthe \
doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
IteratorpItercurrentlypointstothefirstrowidofadoclist \
Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
pageIfthecurrenttermisthelasttermonthepageloadthe \
doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
]
proc rnddoc {} {
global vocab
set nVocab [llength $vocab]
set ret [list]
for {set i 0} {$i < 64} {incr i} {
lappend ret [lindex $vocab [expr $i % $nVocab]]
}
set ret
}
db func rnddoc rnddoc
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%);
INSERT INTO abc(abc, rank) VALUES('pgsz', 32);
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc VALUES ( rnddoc() );
INSERT INTO abc SELECT rnddoc() FROM abc;
INSERT INTO abc SELECT rnddoc() FROM abc;
}
do_execsql_test 3.2 {
SELECT rowid FROM abc WHERE abc
MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist'
ORDER BY rowid DESC;
} {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
do_execsql_test 3.2 {
INSERT INTO abc(abc) VALUES('integrity-check');
INSERT INTO abc(abc) VALUES('optimize');
INSERT INTO abc(abc) VALUES('integrity-check');
}
set v [lindex $vocab 0]
set i 0
foreach v $vocab {
do_execsql_test 3.3.[incr i] {
SELECT rowid FROM abc WHERE abc MATCH $v
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}
} ;# foreach_detail_mode
finish_test

47
ext/fts5/test/fts5doclist.test Normal file
View File

@ -0,0 +1,47 @@
# 2015 April 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This test is focused on edge cases in the doclist format.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5doclist
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Create a table with 1000 columns. Then add some large documents to it.
# All text is in the right most column of the table.
#
do_test 1.0 {
set cols [list]
for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" }
execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])"
} {}
db func rnddoc fts5_rnddoc
do_execsql_test 1.1 {
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii;
}
do_execsql_test 1.2 {
INSERT INTO ccc(ccc) VALUES('integrity-check');
}
finish_test

99
ext/fts5/test/fts5ea.test Normal file
View File

@ -0,0 +1,99 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# Test the fts5 expression parser directly using the fts5_expr() SQL
# test function.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ea
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc do_syntax_error_test {tn expr err} {
set ::se_expr $expr
do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]
}
proc do_syntax_test {tn expr res} {
set ::se_expr $expr
do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
}
foreach {tn expr res} {
1 {abc} {"abc"}
2 {abc def} {"abc" AND "def"}
3 {abc*} {"abc" *}
4 {"abc def ghi" *} {"abc" + "def" + "ghi" *}
5 {one AND two} {"one" AND "two"}
6 {one+two} {"one" + "two"}
7 {one AND two OR three} {("one" AND "two") OR "three"}
8 {one OR two AND three} {"one" OR ("two" AND "three")}
9 {NEAR(one two)} {NEAR("one" "two", 10)}
10 {NEAR("one three"* two, 5)} {NEAR("one" + "three" * "two", 5)}
11 {a OR b NOT c} {"a" OR ("b" NOT "c")}
12 "\x20one\x20two\x20three" {"one" AND "two" AND "three"}
13 "\x09one\x0Atwo\x0Dthree" {"one" AND "two" AND "three"}
14 {"abc""def"} {"abc" + "def"}
} {
do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}
foreach {tn expr res} {
1 {c1:abc}
{c1 : "abc"}
2 {c2 : NEAR(one two) c1:"hello world"}
{c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"}
} {
do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res]
}
foreach {tn expr err} {
1 {AND} {fts5: syntax error near "AND"}
2 {abc def AND} {fts5: syntax error near ""}
3 {abc OR AND} {fts5: syntax error near "AND"}
4 {(a OR b) abc} {fts5: syntax error near "abc"}
5 {NEaR (a b)} {fts5: syntax error near "NEaR"}
6 {NEa (a b)} {fts5: syntax error near "NEa"}
7 {(a OR b) NOT c)} {fts5: syntax error near ")"}
8 {nosuch: a nosuch2: b} {no such column: nosuch}
9 {addr: a nosuch2: b} {no such column: nosuch2}
10 {NOT} {fts5: syntax error near "NOT"}
11 {a AND "abc} {unterminated string}
12 {NEAR(a b, xyz)} {expected integer, got "xyz"}
13 {NEAR(a b, // )} {fts5: syntax error near "/"}
14 {NEAR(a b, "xyz" )} {expected integer, got ""xyz""}
} {
do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
}
#-------------------------------------------------------------------------
# Experiment with a tokenizer that considers " to be a token character.
#
do_execsql_test 4.0 {
SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"');
} {{"a" AND """"}}
#-------------------------------------------------------------------------
# Experiment with a tokenizer that considers " to be a token character.
#
do_catchsql_test 5.0 {
SELECT fts5_expr('abc | def');
} {1 {fts5: syntax error near "|"}}
finish_test

69
ext/fts5/test/fts5eb.test Normal file
View File

@ -0,0 +1,69 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5eb
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc do_syntax_error_test {tn expr err} {
set ::se_expr $expr
do_catchsql_test $tn {SELECT fts5_expr($se_expr)} [list 1 $err]
}
proc do_syntax_test {tn expr res} {
set ::se_expr $expr
do_execsql_test $tn {SELECT fts5_expr($se_expr)} [list $res]
}
foreach {tn expr res} {
1 {abc} {"abc"}
2 {abc ""} {"abc"}
3 {""} {}
4 {abc OR ""} {"abc"}
5 {abc NOT ""} {"abc"}
6 {abc AND ""} {"abc"}
7 {"" OR abc} {"abc"}
8 {"" NOT abc} {"abc"}
9 {"" AND abc} {"abc"}
10 {abc + "" + def} {"abc" + "def"}
11 {abc "" def} {"abc" AND "def"}
12 {r+e OR w} {"r" + "e" OR "w"}
13 {a AND b NOT c} {"a" AND ("b" NOT "c")}
14 {a OR b NOT c} {"a" OR ("b" NOT "c")}
15 {a NOT b AND c} {("a" NOT "b") AND "c"}
16 {a NOT b OR c} {("a" NOT "b") OR "c"}
17 {a AND b OR c} {("a" AND "b") OR "c"}
18 {a OR b AND c} {"a" OR ("b" AND "c")}
} {
do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}
do_catchsql_test 2.1 {
SELECT fts5_expr()
} {1 {wrong number of arguments to function fts5_expr}}
do_catchsql_test 2.1 {
SELECT fts5_expr_tcl()
} {1 {wrong number of arguments to function fts5_expr_tcl}}
finish_test

354
ext/fts5/test/fts5fault1.test Normal file
View File

@ -0,0 +1,354 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault1
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
# Simple tests:
#
# 1: CREATE VIRTUAL TABLE
# 2: INSERT statement
# 3: DELETE statement
# 4: MATCH expressions
#
#
faultsim_save_and_close
do_faultsim_test 1 -faults ioerr-t* -prep {
faultsim_restore_and_reopen
} -body {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
} -test {
faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
}
faultsim_save_and_close
do_faultsim_test 2 -prep {
faultsim_restore_and_reopen
} -body {
execsql {
INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
} -test {
faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
INSERT INTO t1 VALUES('a b c', 'a bc def ghij klmno');
}
faultsim_save_and_close
do_faultsim_test 3 -prep {
faultsim_restore_and_reopen
} -body {
execsql { DELETE FROM t1 }
} -test {
faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
}
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b);
INSERT INTO t2 VALUES('m f a jj th q gi ar', 'hj n h h sg j i m');
INSERT INTO t2 VALUES('nr s t g od j kf h', 'sb h aq rg op rb n nl');
INSERT INTO t2 VALUES('do h h pb p p q fr', 'c rj qs or cr a l i');
INSERT INTO t2 VALUES('lk gp t i lq mq qm p', 'h mr g f op ld aj h');
INSERT INTO t2 VALUES('ct d sq kc qi k f j', 'sn gh c of g s qt q');
INSERT INTO t2 VALUES('d ea d d om mp s ab', 'dm hg l df cm ft pa c');
INSERT INTO t2 VALUES('tc dk c jn n t sr ge', 'a a kn bc n i af h');
INSERT INTO t2 VALUES('ie ii d i b sa qo rf', 'a h m aq i b m fn');
INSERT INTO t2 VALUES('gs r fo a er m h li', 'tm c p gl eb ml q r');
INSERT INTO t2 VALUES('k fe fd rd a gi ho kk', 'ng m c r d ml rm r');
}
faultsim_save_and_close
foreach {tn expr res} {
1 { dk } 7
2 { m f } 1
3 { f* } {1 3 4 5 6 8 9 10}
4 { m OR f } {1 4 5 8 9 10}
5 { sn + gh } {5}
6 { "sn gh" } {5}
7 { NEAR(r a, 5) } {9}
8 { m* f* } {1 4 6 8 9 10}
9 { m* + f* } {1 8}
10 { c NOT p } {5 6 7 10}
} {
do_faultsim_test 4.$tn -prep {
faultsim_restore_and_reopen
} -body "
execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' }
" -test "
faultsim_test_result {[list 0 $res]} {1 {vtable constructor failed: t2}}
"
}
#-------------------------------------------------------------------------
# The following tests use a larger database populated with random data.
#
# The database page size is set to 512 bytes and the FTS5 page size left
# at the default 1000 bytes. This means that reading a node may require
# pulling an overflow page from disk, which is an extra opportunity for
# an error to occur.
#
reset_db
do_execsql_test 5.0.1 {
PRAGMA main.page_size = 512;
CREATE VIRTUAL TABLE x1 USING fts5(a, b);
PRAGMA main.page_size;
} {512}
proc rnddoc {n} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc [string map $map [format %.3d [expr int(rand()*1000)]]]
}
set doc
}
db func rnddoc rnddoc
do_execsql_test 5.0.2 {
WITH r(a, b) AS (
SELECT rnddoc(6), rnddoc(6) UNION ALL
SELECT rnddoc(6), rnddoc(6) FROM r
)
INSERT INTO x1 SELECT * FROM r LIMIT 10000;
}
set res [db one {
SELECT count(*) FROM x1 WHERE x1.a LIKE '%abc%' OR x1.b LIKE '%abc%'}
]
do_faultsim_test 5.1 -faults oom* -body {
execsql { SELECT count(*) FROM x1 WHERE x1 MATCH 'abc' }
} -test {
faultsim_test_result [list 0 $::res]
}
do_faultsim_test 5.2 -faults oom* -body {
execsql { SELECT count(*) FROM x1 WHERE x1 MATCH 'abcd' }
} -test {
faultsim_test_result [list 0 0]
}
proc test_astar {a b} {
return [expr { [regexp {a[^ ][^ ]} $a] || [regexp {a[^ ][^ ]} $b] }]
}
db func test_astar test_astar
set res [db one { SELECT count(*) FROM x1 WHERE test_astar(a, b) } ]
do_faultsim_test 5.3 -faults oom* -body {
execsql { SELECT count(*) FROM x1 WHERE x1 MATCH 'a*' }
} -test {
faultsim_test_result [list 0 $::res]
}
do_faultsim_test 5.4 -faults oom* -prep {
db close
sqlite3 db test.db
} -body {
execsql { INSERT INTO x1 VALUES('a b c d', 'e f g h') }
} -test {
faultsim_test_result [list 0 {}]
}
do_faultsim_test 5.5.1 -faults oom* -body {
execsql {
SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=1
}
} -test {
faultsim_test_result [list 0 1]
}
do_faultsim_test 5.5.2 -faults oom* -body {
execsql {
SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=10
}
} -test {
faultsim_test_result [list 0 1]
}
do_faultsim_test 5.5.3 -faults oom* -body {
execsql {
SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = (
SELECT min(rowid) FROM x1_data WHERE rowid>20
)
}
} -test {
faultsim_test_result [list 0 1]
}
do_faultsim_test 5.5.4 -faults oom* -body {
execsql {
SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = (
SELECT max(rowid) FROM x1_data
)
}
} -test {
faultsim_test_result [list 0 1]
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('automerge', 0);
INSERT INTO x1 VALUES('a b c'); -- 1
INSERT INTO x1 VALUES('a b c'); -- 2
INSERT INTO x1 VALUES('a b c'); -- 3
INSERT INTO x1 VALUES('a b c'); -- 4
INSERT INTO x1 VALUES('a b c'); -- 5
INSERT INTO x1 VALUES('a b c'); -- 6
INSERT INTO x1 VALUES('a b c'); -- 7
INSERT INTO x1 VALUES('a b c'); -- 8
INSERT INTO x1 VALUES('a b c'); -- 9
INSERT INTO x1 VALUES('a b c'); -- 10
INSERT INTO x1 VALUES('a b c'); -- 11
INSERT INTO x1 VALUES('a b c'); -- 12
INSERT INTO x1 VALUES('a b c'); -- 13
INSERT INTO x1 VALUES('a b c'); -- 14
INSERT INTO x1 VALUES('a b c'); -- 15
SELECT count(*) FROM x1_data;
} {17}
faultsim_save_and_close
do_faultsim_test 6.1 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO x1 VALUES('d e f') }
} -test {
faultsim_test_result [list 0 {}]
if {$testrc==0} {
set nCnt [db one {SELECT count(*) FROM x1_data}]
if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" }
}
}
do_faultsim_test 6.2 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO x1(x1, rank) VALUES('pgsz', 32) }
} -test {
faultsim_test_result [list 0 {}]
}
do_faultsim_test 6.3 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO x1(x1) VALUES('integrity-check') }
} -test {
faultsim_test_result [list 0 {}]
}
do_faultsim_test 6.4 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO x1(x1) VALUES('optimize') }
} -test {
faultsim_test_result [list 0 {}]
}
#-------------------------------------------------------------------------
#
do_faultsim_test 7.0 -faults oom* -prep {
catch { db close }
} -body {
sqlite3 db test.db
} -test {
faultsim_test_result [list 0 {}] {1 {}} {1 {initialization of fts5 failed: }}
}
#-------------------------------------------------------------------------
# A prefix query against a large document set.
#
proc rnddoc {n} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
}
set doc
}
reset_db
db func rnddoc rnddoc
do_test 8.0 {
execsql { CREATE VIRTUAL TABLE x1 USING fts5(a) }
set ::res [list]
for {set i 1} {$i<100} {incr i 1} {
execsql { INSERT INTO x1 VALUES( rnddoc(50) ) }
lappend ::res $i
}
} {}
do_faultsim_test 8.1 -faults oom* -prep {
} -body {
execsql {
SELECT rowid FROM x1 WHERE x1 MATCH 'x*'
}
} -test {
faultsim_test_result [list 0 $::res]
}
#-------------------------------------------------------------------------
# Segment promotion.
#
do_test 9.0 {
reset_db
db func rnddoc fts5_rnddoc
execsql {
CREATE VIRTUAL TABLE s2 USING fts5(x);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
for {set i 1} {$i <= 16} {incr i} {
execsql { INSERT INTO s2 VALUES(rnddoc(5)) }
}
fts5_level_segs s2
} {0 1}
set insert_doc [db one {SELECT rnddoc(160)}]
faultsim_save_and_close
do_faultsim_test 9.1 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO s2 VALUES($::insert_doc) }
} -test {
faultsim_test_result {0 {}}
if {$testrc==0} {
set ls [fts5_level_segs s2]
if {$ls != "2 0"} { error "fts5_level_segs says {$ls}" }
}
}
finish_test

140
ext/fts5/test/fts5fault2.test Normal file
View File

@ -0,0 +1,140 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault2
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
set doc [string trim [string repeat "x y z " 200]]
do_execsql_test 1.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, x);
CREATE VIRTUAL TABLE x1 USING fts5(x, content='t1', content_rowid='a');
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
WITH input(a,b) AS (
SELECT 1, $doc UNION ALL
SELECT a+1, ($doc || CASE WHEN (a+1)%100 THEN '' ELSE ' xyz' END)
FROM input WHERE a < 1000
)
INSERT INTO t1 SELECT * FROM input;
INSERT INTO x1(x1) VALUES('rebuild');
}
do_faultsim_test 1.1 -faults oom-* -prep {
} -body {
execsql { SELECT rowid FROM x1 WHERE x1 MATCH 'z AND xyz' }
} -test {
faultsim_test_result {0 {100 200 300 400 500 600 700 800 900 1000}}
}
do_faultsim_test 1.2 -faults oom-* -prep {
} -body {
execsql { SELECT rowid FROM x1 WHERE x1 MATCH 'z + xyz' ORDER BY 1 DESC}
} -test {
faultsim_test_result {0 {1000 900 800 700 600 500 400 300 200 100}}
}
#-------------------------------------------------------------------------
# OOM within a query that accesses the in-memory hash table.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE "a b c" USING fts5(a, b, c);
INSERT INTO "a b c" VALUES('one two', 'x x x', 'three four');
INSERT INTO "a b c" VALUES('nine ten', 'y y y', 'two two');
}
do_faultsim_test 2.1 -faults oom-trans* -prep {
execsql {
BEGIN;
INSERT INTO "a b c" VALUES('one one', 'z z z', 'nine ten');
}
} -body {
execsql { SELECT rowid FROM "a b c" WHERE "a b c" MATCH 'one' }
} -test {
faultsim_test_result {0 {1 3}}
catchsql { ROLLBACK }
}
#-------------------------------------------------------------------------
# OOM within an 'optimize' operation that writes multiple pages to disk.
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE zzz USING fts5(z);
INSERT INTO zzz(zzz, rank) VALUES('pgsz', 32);
INSERT INTO zzz VALUES('a b c d');
INSERT INTO zzz SELECT 'c d e f' FROM zzz;
INSERT INTO zzz SELECT 'e f g h' FROM zzz;
INSERT INTO zzz SELECT 'i j k l' FROM zzz;
INSERT INTO zzz SELECT 'l k m n' FROM zzz;
INSERT INTO zzz SELECT 'o p q r' FROM zzz;
}
faultsim_save_and_close
do_faultsim_test 3.1 -faults oom-trans* -prep {
faultsim_restore_and_reopen
execsql { SELECT rowid FROM zzz }
} -body {
execsql { INSERT INTO zzz(zzz) VALUES('optimize') }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM within an 'integrity-check' operation.
#
reset_db
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE zzz USING fts5(z);
INSERT INTO zzz(zzz, rank) VALUES('pgsz', 32);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<10)
INSERT INTO zzz SELECT rnddoc(10) || ' xccc' FROM ii;
}
do_faultsim_test 4.1 -faults oom-trans* -prep {
} -body {
execsql { INSERT INTO zzz(zzz) VALUES('integrity-check') }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while parsing a tokenize=option
#
reset_db
faultsim_save_and_close
do_faultsim_test 5.0 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
CREATE VIRTUAL TABLE uio USING fts5(a, b,
tokenize="porter 'ascii'",
content="another table",
content_rowid="somecolumn"
);
}
} -test {
faultsim_test_result {0 {}}
}
finish_test

113
ext/fts5/test/fts5fault3.test Normal file
View File

@ -0,0 +1,113 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault3
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# An OOM while resuming a partially completed segment merge.
#
db func rnddoc fts5_rnddoc
do_test 1.0 {
expr srand(0)
execsql {
CREATE VIRTUAL TABLE xx USING fts5(x);
INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
INSERT INTO xx(xx, rank) VALUES('automerge', 16);
}
for {set i 0} {$i < 10} {incr i} {
execsql {
BEGIN;
INSERT INTO xx(x) VALUES(rnddoc(20));
INSERT INTO xx(x) VALUES(rnddoc(20));
INSERT INTO xx(x) VALUES(rnddoc(20));
COMMIT
}
}
execsql {
INSERT INTO xx(xx, rank) VALUES('automerge', 2);
INSERT INTO xx(xx, rank) VALUES('merge', 50);
}
} {}
faultsim_save_and_close
do_faultsim_test 1 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO xx(xx, rank) VALUES('merge', 1) }
} -test {
faultsim_test_result [list 0 {}]
}
#-------------------------------------------------------------------------
# An OOM while flushing an unusually large term to disk.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE xx USING fts5(x);
INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
}
faultsim_save_and_close
set doc "a long term abcdefghijklmnopqrstuvwxyz "
append doc "and then abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz "
append doc [string repeat "abcdefghijklmnopqrstuvwxyz" 10]
do_faultsim_test 2 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO xx(x) VALUES ($::doc) }
} -test {
faultsim_test_result [list 0 {}]
}
#-------------------------------------------------------------------------
# An OOM while flushing an unusually large term to disk.
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE xx USING fts5(x);
}
faultsim_save_and_close
set doc [fts5_rnddoc 1000]
do_faultsim_test 3.1 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO xx(x) VALUES ($::doc) }
} -test {
faultsim_test_result [list 0 {}]
}
set doc [string repeat "abc " 100]
do_faultsim_test 3.2 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
execsql { INSERT INTO xx(x) VALUES ($::doc) }
} -test {
faultsim_test_result [list 0 {}]
}
finish_test

398
ext/fts5/test/fts5fault4.test Normal file
View File

@ -0,0 +1,398 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault4
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# An OOM while dropping an fts5 table.
#
db func rnddoc fts5_rnddoc
do_test 1.0 {
execsql { CREATE VIRTUAL TABLE xx USING fts5(x) }
} {}
faultsim_save_and_close
do_faultsim_test 1 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM xx }
} -body {
execsql { DROP TABLE xx }
} -test {
faultsim_test_result [list 0 {}]
}
#-------------------------------------------------------------------------
# An OOM while "reseeking" an FTS cursor.
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE jj USING fts5(j);
INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s');
INSERT INTO jj(rowid, j) VALUES(202, 't w t f s');
INSERT INTO jj(rowid, j) VALUES(303, 'w t f');
INSERT INTO jj(rowid, j) VALUES(404, 't');
}
faultsim_save_and_close
do_faultsim_test 3 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM jj }
} -body {
set res [list]
db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } {
lappend res $rowid
if {$rowid==303} {
execsql { DELETE FROM jj WHERE rowid=404 }
}
}
set res
} -test {
faultsim_test_result [list 0 {101 202 303}]
}
#-------------------------------------------------------------------------
# An OOM within a special "*reads" query.
#
reset_db
db func rnddoc fts5_rnddoc
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
INSERT INTO x1 SELECT rnddoc(5) FROM ii;
}
set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
do_faultsim_test 4 -faults oom-* -body {
db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
faultsim_test_result {0 {0 {} 4}}
}
#-------------------------------------------------------------------------
# An OOM within a query that uses a custom rank function.
#
reset_db
do_execsql_test 5.0 {
PRAGMA encoding='utf16';
CREATE VIRTUAL TABLE x2 USING fts5(x);
INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3
INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6
INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2
INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5
INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1
}
proc rowidmod {cmd mod} {
set row [$cmd xRowid]
expr {$row % $mod}
}
sqlite3_fts5_create_function db rowidmod rowidmod
do_faultsim_test 5.1 -faults oom-* -body {
db eval {
SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND
rank MATCH "rowidmod('7')" ORDER BY rank
}
} -test {
faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}}
}
proc rowidprefix {cmd prefix} {
set row [$cmd xRowid]
set {} "${row}-${prefix}"
}
sqlite3_fts5_create_function db rowidprefix rowidprefix
set str [string repeat abcdefghijklmnopqrstuvwxyz 10]
do_faultsim_test 5.2 -faults oom-* -body {
db eval "
SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND
rank MATCH 'rowidprefix(''$::str'')'
LIMIT 1
"
} -test {
faultsim_test_result "0 {10-$::str {a b c}}"
}
#-------------------------------------------------------------------------
# OOM errors within auxiliary functions.
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE x3 USING fts5(xxx);
INSERT INTO x3 VALUES('a b c d c b a');
INSERT INTO x3 VALUES('a a a a a a a');
INSERT INTO x3 VALUES('a a a a a a a');
}
do_faultsim_test 6.1 -faults oom-t* -body {
db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' }
} -test {
faultsim_test_result {0 {{a b *c* d *c* b a}}}
}
proc firstinst {cmd} {
foreach {p c o} [$cmd xInst 0] {}
expr $c*100 + $o
}
sqlite3_fts5_create_function db firstinst firstinst
do_faultsim_test 6.2 -faults oom-t* -body {
db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' }
} -test {
faultsim_test_result {0 2} {1 SQLITE_NOMEM}
}
proc previc {cmd} {
set res [$cmd xGetAuxdataInt 0]
$cmd xSetAuxdataInt [$cmd xInstCount]
return $res
}
sqlite3_fts5_create_function db previc previc
do_faultsim_test 6.2 -faults oom-t* -body {
db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' }
} -test {
faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
# OOM error when querying for a phrase with many tokens.
#
reset_db
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE tt USING fts5(x, y);
INSERT INTO tt VALUES('f b g b c b', 'f a d c c b'); -- 1
INSERT INTO tt VALUES('d a e f e d', 'f b b d e e'); -- 2
INSERT INTO tt VALUES('f b g a d c', 'e f c f a d'); -- 3
INSERT INTO tt VALUES('f f c d g f', 'f a e b g b'); -- 4
INSERT INTO tt VALUES('a g b d a g', 'e g a e a c'); -- 5
INSERT INTO tt VALUES('c d b d e f', 'f g e g e e'); -- 6
INSERT INTO tt VALUES('e g f f b c', 'f c e f g f'); -- 7
INSERT INTO tt VALUES('e g c f c e', 'f e e a f g'); -- 8
INSERT INTO tt VALUES('e a e b e e', 'd c c f f f'); -- 9
INSERT INTO tt VALUES('f a g g c c', 'e g d g c e'); -- 10
INSERT INTO tt VALUES('c d b a e f', 'f g e h e e'); -- 11
CREATE VIRTUAL TABLE tt2 USING fts5(o);
INSERT INTO tt2(rowid, o) SELECT rowid, x||' '||y FROM tt;
INSERT INTO tt2(rowid, o) VALUES(12, 'a b c d e f g h i j k l');
}
do_faultsim_test 7.2 -faults oom-* -body {
db eval { SELECT rowid FROM tt WHERE tt MATCH 'f+g+e+g+e+e' }
} -test {
faultsim_test_result {0 6} {1 SQLITE_NOMEM}
}
do_faultsim_test 7.3 -faults oom-* -body {
db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d e f)' }
} -test {
faultsim_test_result {0 11} {1 SQLITE_NOMEM}
}
do_faultsim_test 7.4 -faults oom-t* -body {
db eval { SELECT rowid FROM tt2 WHERE tt2 MATCH '"g c f c e f e e a f"' }
} -test {
faultsim_test_result {0 8} {1 SQLITE_NOMEM}
}
do_faultsim_test 7.5 -faults oom-* -body {
db eval {SELECT rowid FROM tt2 WHERE tt2 MATCH 'NEAR(a b c d e f g h i j k)'}
} -test {
faultsim_test_result {0 12} {1 SQLITE_NOMEM}
}
do_faultsim_test 7.6 -faults oom-* -body {
db eval {SELECT rowid FROM tt WHERE tt MATCH 'y: "c c"'}
} -test {
faultsim_test_result {0 {1 9}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE tt USING fts5(x);
INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
BEGIN;
INSERT INTO tt(rowid, x) VALUES(1, 'a b c d x x');
WITH ii(i) AS (SELECT 2 UNION ALL SELECT i+1 FROM ii WHERE i<99)
INSERT INTO tt(rowid, x) SELECT i, 'a b c x x d' FROM ii;
INSERT INTO tt(rowid, x) VALUES(100, 'a b c d x x');
COMMIT;
}
do_faultsim_test 8.1 -faults oom-t* -body {
db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d, 2)' }
} -test {
faultsim_test_result {0 {1 100}} {1 SQLITE_NOMEM}
}
do_faultsim_test 8.2 -faults oom-t* -body {
db eval { SELECT count(*) FROM tt WHERE tt MATCH 'a OR d' }
} -test {
faultsim_test_result {0 100} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
# Fault in NOT query.
#
reset_db
do_execsql_test 9.0 {
CREATE VIRTUAL TABLE tt USING fts5(x);
INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
BEGIN;
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<200)
INSERT INTO tt(rowid, x)
SELECT i, CASE WHEN (i%50)==0 THEN 'a a a a a a' ELSE 'a x a x a x' END
FROM ii;
COMMIT;
}
do_faultsim_test 9.1 -faults oom-* -body {
db eval { SELECT rowid FROM tt WHERE tt MATCH 'a NOT x' }
} -test {
faultsim_test_result {0 {50 100 150 200}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
# OOM in fts5_expr() SQL function.
#
do_faultsim_test 10.1 -faults oom-t* -body {
db one { SELECT fts5_expr('a AND b NEAR(a b)') }
} -test {
faultsim_test_result {0 {"a" AND "b" AND NEAR("a" "b", 10)}}
}
do_faultsim_test 10.2 -faults oom-t* -body {
db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') }
} -test {
set res {AND [ns -col 0 -- {a b c}] [ns -- {b}] [ns -near 10 -- {a} {b}]}
faultsim_test_result [list 0 $res]
}
do_faultsim_test 10.3 -faults oom-t* -body {
db one { SELECT fts5_expr('x:a', 'x') }
} -test {
faultsim_test_result {0 {x : "a"}}
}
#-------------------------------------------------------------------------
# OOM while configuring 'rank' option.
#
reset_db
do_execsql_test 11.0 {
CREATE VIRTUAL TABLE ft USING fts5(x);
}
do_faultsim_test 11.1 -faults oom-t* -body {
db eval { INSERT INTO ft(ft, rank) VALUES('rank', 'bm25(10.0, 5.0)') }
} -test {
faultsim_test_result {0 {}} {1 {disk I/O error}}
}
#-------------------------------------------------------------------------
# OOM while creating an fts5vocab table.
#
reset_db
do_execsql_test 12.0 {
CREATE VIRTUAL TABLE ft USING fts5(x);
}
faultsim_save_and_close
do_faultsim_test 12.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM sqlite_master }
} -body {
db eval { CREATE VIRTUAL TABLE vv USING fts5vocab(ft, 'row') }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while querying an fts5vocab table.
#
reset_db
do_execsql_test 13.0 {
CREATE VIRTUAL TABLE ft USING fts5(x);
INSERT INTO ft VALUES('a b');
CREATE VIRTUAL TABLE vv USING fts5vocab(ft, 'row');
}
faultsim_save_and_close
do_faultsim_test 13.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM vv }
} -body {
db eval { SELECT * FROM vv }
} -test {
faultsim_test_result {0 {a 1 1 b 1 1}}
}
#-------------------------------------------------------------------------
# OOM in multi-column token query.
#
reset_db
do_execsql_test 13.0 {
CREATE VIRTUAL TABLE ft USING fts5(x, y, z);
INSERT INTO ft(ft, rank) VALUES('pgsz', 32);
INSERT INTO ft VALUES(
'x x x x x x x x x x x x x x x x',
'y y y y y y y y y y y y y y y y',
'z z z z z z z z x x x x x x x x'
);
INSERT INTO ft SELECT * FROM ft;
INSERT INTO ft SELECT * FROM ft;
INSERT INTO ft SELECT * FROM ft;
INSERT INTO ft SELECT * FROM ft;
}
faultsim_save_and_close
do_faultsim_test 13.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM ft }
} -body {
db eval { SELECT rowid FROM ft WHERE ft MATCH '{x z}: x' }
} -test {
faultsim_test_result {0 {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}}
}
#-------------------------------------------------------------------------
# OOM in an "ALTER TABLE RENAME TO"
#
reset_db
do_execsql_test 14.0 {
CREATE VIRTUAL TABLE "tbl one" USING fts5(x, y, z);
}
faultsim_save_and_close
do_faultsim_test 14.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM "tbl one" }
} -body {
db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" }
} -test {
faultsim_test_result {0 {}}
}
finish_test

133
ext/fts5/test/fts5fault5.test Normal file
View File

@ -0,0 +1,133 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault5
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# OOM while creating an FTS5 table.
#
do_faultsim_test 1.1 -faults oom-t* -prep {
db eval { DROP TABLE IF EXISTS abc }
} -body {
db eval { CREATE VIRTUAL TABLE abc USING fts5(x,y) }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while writing a multi-tier doclist-index. And while running
# integrity-check on the same.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE tt USING fts5(x);
INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
}
faultsim_save_and_close
do_faultsim_test 2.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM tt }
} -body {
set str [string repeat "abc " 50]
db eval {
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
INSERT INTO tt(rowid, x) SELECT i, $str FROM ii;
}
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 2.2 -faults oom-t* -body {
db eval { INSERT INTO tt(tt) VALUES('integrity-check') }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while scanning fts5vocab tables.
#
reset_db
do_test 3.0 {
execsql {
CREATE VIRTUAL TABLE tt USING fts5(x);
CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
CREATE VIRTUAL TABLE tt2 USING fts5(x, detail=col);
CREATE VIRTUAL TABLE tv2 USING fts5vocab(tt2, 'col');
INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
INSERT INTO tt2(tt2, rank) VALUES('pgsz', 32);
BEGIN;
}
for {set i 0} {$i < 20} {incr i} {
set str [string repeat "$i " 50]
execsql { INSERT INTO tt VALUES($str) }
execsql { INSERT INTO tt2 VALUES($str) }
}
execsql COMMIT
} {}
do_faultsim_test 3.1 -faults oom-t* -body {
db eval {
SELECT term FROM tv;
}
} -test {
faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}}
}
do_faultsim_test 3.2 -faults oom-t* -body {
db eval {
SELECT term FROM tv WHERE term BETWEEN '1' AND '2';
}
} -test {
faultsim_test_result {0 {1 10 11 12 13 14 15 16 17 18 19 2}}
}
breakpoint
do_execsql_test 3.3.0 {
SELECT * FROM tv2;
} {
0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {}
14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {}
2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {}
9 x 1 {}
}
do_faultsim_test 3.3 -faults oom-t* -body {
db eval {
SELECT * FROM tv2;
}
} -test {
faultsim_test_result [list 0 [list \
0 x 1 {} 1 x 1 {} 10 x 1 {} 11 x 1 {} 12 x 1 {} 13 x 1 {} \
14 x 1 {} 15 x 1 {} 16 x 1 {} 17 x 1 {} 18 x 1 {} 19 x 1 {} \
2 x 1 {} 3 x 1 {} 4 x 1 {} 5 x 1 {} 6 x 1 {} 7 x 1 {} 8 x 1 {} \
9 x 1 {}
]]
}
finish_test

295
ext/fts5/test/fts5fault6.test Normal file
View File

@ -0,0 +1,295 @@
# 2014 June 17
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault6
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# OOM while rebuilding an FTS5 table.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE tt USING fts5(a, b);
INSERT INTO tt VALUES('c d c g g f', 'a a a d g a');
INSERT INTO tt VALUES('c d g b f d', 'b g e c g c');
INSERT INTO tt VALUES('c c f d e d', 'c e g d b c');
INSERT INTO tt VALUES('e a f c e f', 'g b a c d g');
INSERT INTO tt VALUES('c g f b b d', 'g c d c f g');
INSERT INTO tt VALUES('d a g a b b', 'g c g g c e');
INSERT INTO tt VALUES('e f a b c e', 'f d c d c c');
INSERT INTO tt VALUES('e c a g c d', 'b b g f f b');
INSERT INTO tt VALUES('g b d d e b', 'f f b d a c');
INSERT INTO tt VALUES('e a d a e d', 'c e a e f g');
}
faultsim_save_and_close
do_faultsim_test 1.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
db eval { INSERT INTO tt(tt) VALUES('rebuild') }
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 1.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
db eval { REPLACE INTO tt(rowid, a, b) VALUES(6, 'x y z', 'l l l'); }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM within a special delete.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE tt USING fts5(a, content="");
INSERT INTO tt VALUES('c d c g g f');
INSERT INTO tt VALUES('c d g b f d');
INSERT INTO tt VALUES('c c f d e d');
INSERT INTO tt VALUES('e a f c e f');
INSERT INTO tt VALUES('c g f b b d');
INSERT INTO tt VALUES('d a g a b b');
INSERT INTO tt VALUES('e f a b c e');
INSERT INTO tt VALUES('e c a g c d');
INSERT INTO tt VALUES('g b d d e b');
INSERT INTO tt VALUES('e a d a e d');
}
faultsim_save_and_close
do_faultsim_test 2.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
db eval { INSERT INTO tt(tt, rowid, a) VALUES('delete', 3, 'c d g b f d'); }
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 2.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
db eval { INSERT INTO tt(tt) VALUES('delete-all') }
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 2.3 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
db eval { INSERT INTO tt VALUES('x y z') }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM in the ASCII tokenizer with very large tokens.
#
# Also the unicode tokenizer.
#
set t1 [string repeat wxyz 20]
set t2 [string repeat wxyz 200]
set t3 [string repeat wxyz 2000]
set doc "$t1 $t2 $t3"
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE xyz USING fts5(c, tokenize=ascii, content="");
CREATE VIRTUAL TABLE xyz2 USING fts5(c, content="");
}
faultsim_save_and_close
do_faultsim_test 3.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM xyz }
} -body {
db eval { INSERT INTO xyz VALUES($::doc) }
} -test {
faultsim_test_result {0 {}}
}
do_faultsim_test 3.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval { SELECT * FROM xyz2 }
} -body {
db eval { INSERT INTO xyz2 VALUES($::doc) }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# OOM while initializing a unicode61 tokenizer.
#
reset_db
faultsim_save_and_close
do_faultsim_test 4.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
db eval {
CREATE VIRTUAL TABLE yu USING fts5(x, tokenize="unicode61 separators abc");
}
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
#
# 5.2.* OOM while running a query that includes synonyms and matchinfo().
#
# 5.3.* OOM while running a query that returns a row containing instances
# of more than 4 synonyms for a single term.
#
proc mit {blob} {
set scan(littleEndian) i*
set scan(bigEndian) I*
binary scan $blob $scan($::tcl_platform(byteOrder)) r
return $r
}
proc tcl_tokenize {tflags text} {
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
if {$tflags=="query" && [string length $w]==1} {
for {set i 2} {$i < 7} {incr i} {
sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
}
}
}
}
proc tcl_create {args} { return "tcl_tokenize" }
reset_db
sqlite3_fts5_create_tokenizer db tcl tcl_create
db func mit mit
sqlite3_fts5_register_matchinfo db
do_test 5.0 {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) }
execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
foreach {rowid text} {
1 {aaaa cc b aaaaa cc aa}
2 {aa aa bb a bbb}
3 {bb aaaaa aaaaa b aaaa aaaaa}
4 {aa a b aaaa aa}
5 {aa b ccc aaaaa cc}
6 {aa aaaaa bbbb cc aaa}
7 {aaaaa aa aa ccccc bb}
8 {ccc bbbbb ccccc bbb c}
9 {cccccc bbbb a aaa cccc c}
20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee}
21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e}
22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd}
23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff}
24 {eee dd ee dddd dddd eeeeee e eee fff ffff}
25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f}
26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee}
27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d}
28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd}
29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee}
30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee}
31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd}
32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff}
33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd}
34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff}
35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff}
36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff}
37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee}
38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee}
39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd}
40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f}
41 {ddd ddd fff fffff ee fffff f fff ddddd fffff}
42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee}
43 {d dd fff fffff d f fff e dddd ee ee}
44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee}
45 {eeee f eeeee ee e ffff f ddd e fff}
46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee}
47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee}
48 {ffff ffff e dddd ffffff dd dd dddd f fffff}
49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee}
50 {x e}
} {
execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) }
}
} {}
set res [list {*}{
1 {3 24 8 2 12 6}
5 {2 24 8 2 12 6}
6 {3 24 8 1 12 6}
7 {3 24 8 1 12 6}
9 {2 24 8 3 12 6}
}]
do_execsql_test 5.1.1 {
SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
} $res
do_execsql_test 5.1.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f'
} 29
faultsim_save_and_close
do_faultsim_test 5.2 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
sqlite3_fts5_register_matchinfo db
db func mit mit
} -body {
db eval {
SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
}
} -test {
faultsim_test_result [list 0 $::res]
}
do_faultsim_test 5.3 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
} -body {
db eval {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
}
} -test {
faultsim_test_result {0 29}
}
do_faultsim_test 5.4 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
} -body {
db eval {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
}
} -test {
faultsim_test_result {0 1}
}
#-------------------------------------------------------------------------
catch { db close }
breakpoint
do_faultsim_test 6 -faults oom* -prep {
sqlite_orig db test.db
sqlite3_db_config_lookaside db 0 0 0
} -test {
faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }}
if {$testrc==0} {
db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) }
}
db close
}
finish_test

119
ext/fts5/test/fts5fault7.test Normal file
View File

@ -0,0 +1,119 @@
# 2015 September 3
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault7
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
if 1 {
#-------------------------------------------------------------------------
# Test fault-injection on a query that uses xColumnSize() on columnsize=0
# table.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0);
INSERT INTO t1 VALUES('a b c d e f g');
INSERT INTO t1 VALUES('a b c d');
INSERT INTO t1 VALUES('a b c d e f g h i j');
}
fts5_aux_test_functions db
do_faultsim_test 1 -faults oom* -body {
execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' }
} -test {
faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
}
}
#-------------------------------------------------------------------------
# Test fault-injection when a segment is promoted.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a);
INSERT INTO t2(t2, rank) VALUES('automerge', 0);
INSERT INTO t2(t2, rank) VALUES('crisismerge', 4);
INSERT INTO t2(t2, rank) VALUES('pgsz', 40);
INSERT INTO t2 VALUES('a b c');
INSERT INTO t2 VALUES('d e f');
INSERT INTO t2 VALUES('f e d');
INSERT INTO t2 VALUES('c b a');
INSERT INTO t2 VALUES('a b c');
INSERT INTO t2 VALUES('d e f');
INSERT INTO t2 VALUES('f e d');
INSERT INTO t2 VALUES('c b a');
} {}
faultsim_save_and_close
do_faultsim_test 1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval {
BEGIN;
INSERT INTO t2 VALUES('c d c g g f');
INSERT INTO t2 VALUES('c d g b f d');
INSERT INTO t2 VALUES('c c f d e d');
INSERT INTO t2 VALUES('e a f c e f');
INSERT INTO t2 VALUES('c g f b b d');
INSERT INTO t2 VALUES('d a g a b b');
INSERT INTO t2 VALUES('e f a b c e');
INSERT INTO t2 VALUES('e c a g c d');
INSERT INTO t2 VALUES('g b d d e b');
INSERT INTO t2 VALUES('e a d a e d');
}
} -body {
db eval COMMIT
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# Test fault-injection when a segment is promoted.
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE xy USING fts5(x);
INSERT INTO xy(rowid, x) VALUES(1, '1 2 3');
INSERT INTO xy(rowid, x) VALUES(2, '2 3 4');
INSERT INTO xy(rowid, x) VALUES(3, '3 4 5');
}
faultsim_save_and_close
do_faultsim_test 2.1 -faults oom-* -prep {
faultsim_restore_and_reopen
} -body {
db eval { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2 }
} -test {
faultsim_test_result {0 {}}
}
# Test fault-injection when an empty expression is parsed.
#
do_faultsim_test 2.2 -faults oom-* -body {
db eval { SELECT * FROM xy('""') }
} -test {
faultsim_test_result {0 {}}
}
finish_test

60
ext/fts5/test/fts5fault8.test Normal file
View File

@ -0,0 +1,60 @@
# 2015 September 3
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault8
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
INSERT INTO t1 VALUES('a b a b', NULL);
INSERT INTO t1 VALUES(NULL, '1 2 1 2');
}
do_faultsim_test 1 -faults oom-* -body {
execsql {
SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2'
}
} -test {
faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
{1 SQLITE_NOMEM}
}
do_faultsim_test 2 -faults oom-* -body {
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} -test {
faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}
if {[detail_is_none]==0} {
do_faultsim_test 3 -faults oom-* -body {
execsql { SELECT rowid FROM t1('b:2') }
} -test {
faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM}
}
}
} ;# foreach_detail_mode...
finish_test

156
ext/fts5/test/fts5fault9.test Normal file
View File

@ -0,0 +1,156 @@
# 2015 September 3
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault9
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH seq(s) AS ( SELECT 1 UNION ALL SELECT s+1 FROM seq WHERE s<50)
INSERT INTO t1 SELECT 'x x x y y y', 'a b c d e f' FROM seq;
}
do_faultsim_test 1 -faults oom-* -body {
execsql { SELECT count(*) FROM t1('x AND y') }
} -test {
faultsim_test_result {0 50}
}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
INSERT INTO t2 VALUES('abc cba', 'cba abc');
INSERT INTO t2 VALUES('abc cba', 'cba abc');
INSERT INTO t2 VALUES('abc cba', 'cba abc');
INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
INSERT INTO t2 VALUES('axy cyx', 'cyx axy');
}
do_faultsim_test 2 -faults oom-* -body {
execsql { SELECT count(*) FROM t2('a* AND c*') }
} -test {
faultsim_test_result {0 6}
}
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(a, detail=%DETAIL%);
INSERT INTO t3 VALUES('a x x a x a a a');
INSERT INTO t3 VALUES('x a a x a x x x');
}
do_faultsim_test 3.1 -faults oom-* -body {
execsql { SELECT highlight(t3, 0, '[', ']') FROM t3('a') }
} -test {
faultsim_test_result {0 {{[a] x x [a] x [a] [a] [a]} {x [a] [a] x [a] x x x}}}
}
do_faultsim_test 3.2 -faults oom-t* -body {
execsql { SELECT fts5_test_poslist2(t3) FROM t3('x') }
} -test {
faultsim_test_result \
{0 {{0.0.1 0.0.2 0.0.4} {0.0.0 0.0.3 0.0.5 0.0.6 0.0.7}}} \
{1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
# Test OOM injection with the xPhraseFirstColumn() API and a tokenizer
# uses query synonyms.
#
fts5_tclnum_register db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(x, y, z, detail=%DETAIL%, tokenize=tclnum);
INSERT INTO t4 VALUES('one two three', '1 2 3', 'i ii iii');
INSERT INTO t4 VALUES('1 2 3', 'i ii iii', 'one two three');
INSERT INTO t4 VALUES('i ii iii', 'one two three', 'i ii iii');
INSERT INTO t4 VALUES('a1 a2 a3', 'a4 a5 a6', 'a7 a8 a9');
INSERT INTO t4 VALUES('b1 b2 b3', 'b4 b5 b6', 'b7 b8 b9');
INSERT INTO t4 VALUES('c1 c2 c3', 'c4 c5 c6', 'c7 c8 c9');
}
do_faultsim_test 4.1 -faults oom-t* -body {
execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
} -test {
faultsim_test_result \
{0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
}
do_faultsim_test 4.2 -faults oom-t* -body {
execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
} -test {
faultsim_test_result \
{0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
# An OOM within an "ORDER BY rank" query.
#
db func rnddoc fts5_rnddoc
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=%DETAIL%);
INSERT INTO xx VALUES ('def', 'abc ' || rnddoc(10));
INSERT INTO xx VALUES ('def', 'abc abc' || rnddoc(9));
INSERT INTO xx VALUES ('def', 'abc abc abc' || rnddoc(8));
} {}
faultsim_save_and_close
do_faultsim_test 5 -faults oom-* -prep {
faultsim_restore_and_reopen
execsql { SELECT * FROM xx }
} -body {
execsql { SELECT rowid FROM xx('abc AND def') ORDER BY rank }
} -test {
faultsim_test_result [list 0 {3 2 1}]
}
set doc [string repeat "xyz " 500]
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE yy USING fts5(y, detail=%DETAIL%);
INSERT INTO yy(yy, rank) VALUES('pgsz', 64);
INSERT INTO yy VALUES ($doc);
INSERT INTO yy VALUES ('1 2 3');
INSERT INTO yy VALUES ('xyz');
UPDATE yy SET y = y WHERE rowid = 1;
UPDATE yy SET y = y WHERE rowid = 1;
UPDATE yy SET y = y WHERE rowid = 1;
UPDATE yy SET y = y WHERE rowid = 1;
} {}
do_faultsim_test 6 -faults oom-* -body {
execsql { SELECT rowid FROM yy('xyz') }
} -test {
faultsim_test_result [list 0 {1 3}]
}
} ;# foreach_detail_mode...
finish_test

64
ext/fts5/test/fts5faultA.test Normal file
View File

@ -0,0 +1,64 @@
# 2016 February 2
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5faultA
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE o1 USING fts5(a, detail=%DETAIL%);
INSERT INTO o1(o1, rank) VALUES('pgsz', 32);
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 )
INSERT INTO o1 SELECT 'A B C' FROM s;
INSERT INTO o1 VALUES('A X C');
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 )
INSERT INTO o1 SELECT 'A B C' FROM s;
}
do_faultsim_test 1 -faults oom* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT rowid FROM o1('a NOT b') }
} -test {
faultsim_test_result {0 301}
}
}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE o2 USING fts5(a);
INSERT INTO o2 VALUES('A B C');
WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 )
INSERT INTO o2 SELECT group_concat('A B C ') FROM s;
}
do_faultsim_test 2 -faults oom* -prep {
sqlite3 db test.db
} -body {
execsql { SELECT rowid FROM o2('a+b+c NOT xyz') }
} -test {
faultsim_test_result {0 {1 2}}
}
finish_test

43
ext/fts5/test/fts5full.test Normal file
View File

@ -0,0 +1,43 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Test that SQLITE_FULL is returned if the FTS5 table cannot find a free
# segid to use. In practice this can only really happen when automerge and
# crisismerge are both disabled.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5full
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE x8 USING fts5(i);
INSERT INTO x8(x8, rank) VALUES('automerge', 0);
INSERT INTO x8(x8, rank) VALUES('crisismerge', 100000);
}
db func rnddoc fts5_rnddoc
do_test 1.1 {
list [catch {
for {set i 0} {$i < 2500} {incr i} {
execsql { INSERT INTO x8 VALUES( rnddoc(5) ); }
}
} msg] $msg
} {1 {database or disk is full}}
finish_test

133
ext/fts5/test/fts5hash.test Normal file
View File

@ -0,0 +1,133 @@
# 2015 April 21
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# The tests in this file are focused on the code in fts5_hash.c.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5hash
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Return a list of tokens (a vocabulary) that all share the same hash
# key value. This can be used to test hash collisions.
#
proc build_vocab1 {args} {
set O(-nslot) 1024
set O(-nword) 20
set O(-hash) 88
set O(-prefix) ""
if {[llength $args] % 2} { error "bad args" }
array set O2 $args
foreach {k v} $args {
if {[info exists O($k)]==0} { error "bad option: $k" }
set O($k) $v
}
set L [list]
while {[llength $L] < $O(-nword)} {
set t "$O(-prefix)[random_token]"
set h [sqlite3_fts5_token_hash $O(-nslot) $t]
if {$O(-hash)==$h} { lappend L $t }
}
return $L
}
proc random_token {} {
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
set iVal [expr int(rand() * 2000000)]
return [string map $map $iVal]
}
proc random_doc {vocab nWord} {
set doc ""
set nVocab [llength $vocab]
for {set i 0} {$i<$nWord} {incr i} {
set j [expr {int(rand() * $nVocab)}]
lappend doc [lindex $vocab $j]
}
return $doc
}
foreach_detail_mode $testprefix {
set vocab [build_vocab1]
db func r random_doc
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%);
BEGIN;
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii;
INSERT INTO eee(eee) VALUES('integrity-check');
COMMIT;
INSERT INTO eee(eee) VALUES('integrity-check');
}
set hash [sqlite3_fts5_token_hash 1024 xyz]
set vocab [build_vocab1 -prefix xyz -hash $hash]
lappend vocab xyz
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row');
BEGIN;
}
do_test 1.2 {
for {set i 1} {$i <= 100} {incr i} {
execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) }
}
} {}
do_test 1.3 {
db eval { SELECT term, doc FROM vocab } {
set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}]
if {$nRow != $doc} {
error "term=$term fts5vocab=$doc cnt=$nRow"
}
}
set {} {}
} {}
do_execsql_test 1.4 {
COMMIT;
INSERT INTO eee(eee) VALUES('integrity-check');
}
#-----------------------------------------------------------------------
# Add a small and very large token with the same hash value to an
# empty table. At one point this would provoke an asan error.
#
do_test 2.0 {
set big [string repeat 12345 40]
set hash [sqlite3_fts5_token_hash 1024 $big]
while {1} {
set small [random_token]
if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break
}
execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) }
breakpoint
execsql {
INSERT INTO t2 VALUES($small || ' ' || $big);
}
} {}
} ;# foreach_detail_mode
finish_test

213
ext/fts5/test/fts5integrity.test Normal file
View File

@ -0,0 +1,213 @@
# 2015 Jan 13
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests focused on the integrity-check procedure.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5integrity
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE xx USING fts5(x);
INSERT INTO xx VALUES('term');
}
do_execsql_test 1.1 {
INSERT INTO xx(xx) VALUES('integrity-check');
}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE yy USING fts5(x, prefix=1);
INSERT INTO yy VALUES('term');
}
do_execsql_test 2.1 {
INSERT INTO yy(yy) VALUES('integrity-check');
}
#--------------------------------------------------------------------
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE zz USING fts5(z);
INSERT INTO zz(zz, rank) VALUES('pgsz', 32);
INSERT INTO zz VALUES('b b b b b b b b b b b b b b');
INSERT INTO zz SELECT z FROM zz;
INSERT INTO zz SELECT z FROM zz;
INSERT INTO zz SELECT z FROM zz;
INSERT INTO zz SELECT z FROM zz;
INSERT INTO zz SELECT z FROM zz;
INSERT INTO zz SELECT z FROM zz;
INSERT INTO zz(zz) VALUES('optimize');
}
do_execsql_test 3.1 { INSERT INTO zz(zz) VALUES('integrity-check'); }
#--------------------------------------------------------------------
# Mess around with a docsize record. And the averages record. Then
# check that integrity-check picks it up.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE aa USING fts5(zz);
INSERT INTO aa(zz) VALUES('a b c d e');
INSERT INTO aa(zz) VALUES('a b c d');
INSERT INTO aa(zz) VALUES('a b c');
INSERT INTO aa(zz) VALUES('a b');
INSERT INTO aa(zz) VALUES('a');
SELECT length(sz) FROM aa_docsize;
} {1 1 1 1 1}
do_execsql_test 4.1 {
INSERT INTO aa(aa) VALUES('integrity-check');
}
do_catchsql_test 4.2 {
BEGIN;
UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
INSERT INTO aa(aa) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_catchsql_test 4.3 {
ROLLBACK;
BEGIN;
UPDATE aa_data SET block = X'44' WHERE rowid = 1;
INSERT INTO aa(aa) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_catchsql_test 4.4 {
ROLLBACK;
BEGIN;
INSERT INTO aa_docsize VALUES(23, X'04');
INSERT INTO aa(aa) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_catchsql_test 4.5 {
ROLLBACK;
BEGIN;
INSERT INTO aa_docsize VALUES(23, X'00');
INSERT INTO aa_content VALUES(23, '');
INSERT INTO aa(aa) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r}
#exit
execsql { ROLLBACK }
#-------------------------------------------------------------------------
# Test that integrity-check works on a reasonably large db with many
# different terms.
# Document generator command.
proc rnddoc {n} {
set doc [list]
for {set i 0} {$i<$n} {incr i} {
lappend doc [format %.5d [expr int(rand()*10000)]]
}
return $doc
}
db func rnddoc rnddoc
expr srand(0)
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE gg USING fts5(a, prefix="1,2,3");
INSERT INTO gg(gg, rank) VALUES('pgsz', 256);
INSERT INTO gg VALUES(rnddoc(20));
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
INSERT INTO gg SELECT rnddoc(20) FROM gg;
}
do_execsql_test 5.1 {
INSERT INTO gg(gg) VALUES('integrity-check');
}
do_execsql_test 5.2 {
INSERT INTO gg(gg) VALUES('optimize');
}
do_execsql_test 5.3 {
INSERT INTO gg(gg) VALUES('integrity-check');
}
do_test 5.4.1 {
set ok 0
for {set i 0} {$i < 10000} {incr i} {
set T [format %.5d $i]
set res [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC }]
set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
if {$res == [lsort -integer $res2]} { incr ok }
}
set ok
} {10000}
do_test 5.4.2 {
set ok 0
for {set i 0} {$i < 100} {incr i} {
set T "[format %.3d $i]*"
set res [db eval { SELECT rowid FROM gg($T) ORDER BY rowid ASC }]
set res2 [db eval { SELECT rowid FROM gg($T) ORDER BY rowid DESC }]
if {$res == [lsort -integer $res2]} { incr ok }
}
set ok
} {100}
#-------------------------------------------------------------------------
# Similar to 5.*.
#
foreach {tn pgsz} {
1 32
2 36
3 40
4 44
5 48
} {
do_execsql_test 6.$tn.1 {
DROP TABLE IF EXISTS hh;
CREATE VIRTUAL TABLE hh USING fts5(y);
INSERT INTO hh(hh, rank) VALUES('pgsz', $pgsz);
WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
FROM s;
WITH s(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<999)
INSERT INTO hh SELECT printf("%.3d%.3d%.3d %.3d%.3d%.3d",i,i,i,i+1,i+1,i+1)
FROM s;
INSERT INTO hh(hh) VALUES('optimize');
}
do_test 6.$tn.2 {
set ok 0
for {set i 0} {$i < 1000} {incr i} {
set T [format %.3d%.3d%.3d $i $i $i]
set res [db eval { SELECT rowid FROM hh($T) ORDER BY rowid ASC }]
set res2 [db eval { SELECT rowid FROM hh($T) ORDER BY rowid DESC }]
if {$res == [lsort -integer $res2]} { incr ok }
}
set ok
} {1000}
}
finish_test

Some files were not shown because too many files have changed in this diff Show More