diff --git a/Makefile.in b/Makefile.in index 8c90ff82..d533abff 100644 --- a/Makefile.in +++ b/Makefile.in @@ -174,7 +174,7 @@ NAWK = @AWK@ # Object files for the SQLite library (non-amalgamation). # OBJS0 = alter.lo analyze.lo attach.lo auth.lo backup.lo bitvec.lo btmutex.lo \ - btree.lo build.lo callback.lo complete.lo date.lo \ + btree.lo build.lo callback.lo complete.lo ctime.lo date.lo \ delete.lo expr.lo fault.lo fkey.lo func.lo global.lo \ hash.lo journal.lo insert.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ @@ -215,6 +215,7 @@ SRC = \ $(TOP)/src/build.c \ $(TOP)/src/callback.c \ $(TOP)/src/complete.c \ + $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ @@ -350,6 +351,7 @@ TESTSRC2 = \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ $(TOP)/src/date.c \ + $(TOP)/src/ctime.c \ $(TOP)/src/expr.c \ $(TOP)/src/func.c \ $(TOP)/src/insert.c \ @@ -558,6 +560,9 @@ callback.lo: $(TOP)/src/callback.c $(HDR) complete.lo: $(TOP)/src/complete.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/complete.c +ctime.lo: $(TOP)/src/ctime.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/ctime.c + date.lo: $(TOP)/src/date.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/date.c diff --git a/Makefile.vxworks b/Makefile.vxworks index 141b7061..273ce131 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -231,6 +231,7 @@ SRC = \ $(TOP)/src/build.c \ $(TOP)/src/callback.c \ $(TOP)/src/complete.c \ + $(TOP)/src/ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ @@ -392,7 +393,7 @@ TESTSRC = \ TESTSRC2 = \ $(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \ - $(TOP)/src/build.c $(TOP)/src/date.c \ + $(TOP)/src/build.c $(TOP)/src/ctime.c $(TOP)/src/date.c \ $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \ $(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \ $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \ diff --git a/VERSION b/VERSION index 1bff2319..cc690cc4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.21 +3.6.23.1 diff --git a/configure b/configure index b922d17f..6b9b772c 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.6.21. +# Generated by GNU Autoconf 2.62 for sqlite 3.6.23.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.6.21' -PACKAGE_STRING='sqlite 3.6.21' +PACKAGE_VERSION='3.6.23.1' +PACKAGE_STRING='sqlite 3.6.23.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1487,7 +1487,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.6.21 to adapt to many kinds of systems. +\`configure' configures sqlite 3.6.23.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1552,7 +1552,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.6.21:";; + short | recursive ) echo "Configuration of sqlite 3.6.23.1:";; esac cat <<\_ACEOF @@ -1670,7 +1670,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.6.21 +sqlite configure 3.6.23.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1684,7 +1684,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.6.21, which was +It was created by sqlite $as_me 3.6.23.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -13972,7 +13972,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.6.21, which was +This file was extended by sqlite $as_me 3.6.23.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14025,7 +14025,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.6.21 +sqlite config.status 3.6.23.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index cd3f7ab9..3b09bf3d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -23,9 +23,6 @@ ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -/* TODO(shess) Consider exporting this comment to an HTML file or the -** wiki. -*/ /* The full-text index is stored in a series of b+tree (-like) ** structures called segments which map terms to doclists. The ** structures are like b+trees in layout, but are constructed from the @@ -48,13 +45,27 @@ ** 21 bits - BBA ** and so on. ** -** This is identical to how sqlite encodes varints (see util.c). +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be upt to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 ** ** **** Document lists **** ** A doclist (document list) holds a docid-sorted list of hits for a ** given term. Doclists hold docids, and can optionally associate -** token positions and offsets with docids. +** token positions and offsets with docids. A position is the index +** of a word within the document. The first word of the document has +** a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. ** ** A DL_POSITIONS_OFFSETS doclist is stored like this: ** @@ -62,16 +73,12 @@ ** varint docid; ** array { (position list for column 0) ** varint position; (delta from previous position plus POS_BASE) -** varint startOffset; (delta from previous startOffset) -** varint endOffset; (delta from startOffset) ** } ** array { ** varint POS_COLUMN; (marks start of position list for new column) ** varint column; (index of new column) ** array { ** varint position; (delta from previous position plus POS_BASE) -** varint startOffset;(delta from previous startOffset) -** varint endOffset; (delta from startOffset) ** } ** } ** varint POS_END; (marks end of positions for this document. @@ -79,10 +86,23 @@ ** ** Here, array { X } means zero or more occurrences of X, adjacent in ** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer, while an "offset" is a byte offset, -** both based at 0. Note that POS_END and POS_COLUMN occur in the -** same logical place as the position element, and act as sentinals -** ending a position list array. +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the next docid. It has one position 72 (72-2) and then +** terminates with the 0 at K. ** ** A DL_POSITIONS doclist omits the startOffset and endOffset ** information. A DL_DOCIDS doclist omits both the position and @@ -285,6 +305,7 @@ #include #include #include +#include #include "fts3.h" #ifndef SQLITE_CORE @@ -292,70 +313,6 @@ SQLITE_EXTENSION_INIT1 #endif - - -/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it -** would be nice to order the file better, perhaps something along the -** lines of: -** -** - utility functions -** - table setup functions -** - table update functions -** - table query functions -** -** Put the query functions last because they're likely to reference -** typedefs or functions from the table update section. -*/ - -#if 0 -# define FTSTRACE(A) printf A; fflush(stdout) -#else -# define FTSTRACE(A) -#endif - -typedef enum DocListType { - DL_DOCIDS, /* docids only */ - DL_POSITIONS, /* docids + positions */ - DL_POSITIONS_OFFSETS /* docids + positions + offsets */ -} DocListType; - -/* -** By default, only positions and not offsets are stored in the doclists. -** To change this so that offsets are stored too, compile with -** -** -DDL_DEFAULT=DL_POSITIONS_OFFSETS -** -** If DL_DEFAULT is set to DL_DOCIDS, your table can only be inserted -** into (no deletes or updates). -*/ -#ifndef DL_DEFAULT -# define DL_DEFAULT DL_POSITIONS -#endif - -enum { - POS_END = 0, /* end of this position list */ - POS_COLUMN, /* followed by new column number */ - POS_BASE -}; - -/* utility functions */ - -/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single -** record to prevent errors of the form: -** -** my_function(SomeType *b){ -** memset(b, '\0', sizeof(b)); // sizeof(b)!=sizeof(*b) -** } -*/ -/* TODO(shess) Obvious candidates for a header file. */ -#define CLEAR(b) memset(b, '\0', sizeof(*(b))) - -#ifndef NDEBUG -# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b))) -#else -# define SCRAMBLE(b) -#endif - /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -381,13 +338,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; - while( (*q & 0x80) == 0x80 ){ + while( (*q&0x80)==0x80 && q-(unsigned char *)p= FTS3_VARINT_MAX ){ /* bad data */ - assert( 0 ); - return 0; - } } x += y * (*q++); *v = (sqlite_int64) x; @@ -402,7 +355,6 @@ int sqlite3Fts3GetVarint32(const char *p, int *pi){ sqlite_int64 i; int ret = sqlite3Fts3GetVarint(p, &i); *pi = (int) i; - assert( *pi==i ); return ret; } @@ -431,40 +383,49 @@ int sqlite3Fts3VarintLen(sqlite3_uint64 v){ ** 'xyz' becomes xyz ** [pqr] becomes pqr ** `mno` becomes mno +** */ void sqlite3Fts3Dequote(char *z){ - int quote; - int i, j; + char quote; /* Quote character (if any ) */ quote = z[0]; - switch( quote ){ - case '\'': break; - case '"': break; - case '`': break; /* For MySQL compatibility */ - case '[': quote = ']'; break; /* For MS SqlServer compatibility */ - default: return; - } - for(i=1, j=0; z[i]; i++){ - if( z[i]==quote ){ - if( z[i+1]==quote ){ - z[j++] = (char)quote; - i++; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( ALWAYS(z[iIn]) ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; }else{ - z[j++] = 0; - break; + z[iOut++] = z[iIn++]; } - }else{ - z[j++] = z[i]; } + z[iOut] = '\0'; } } +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the next element of the varlist. Add the value of the varint +** to *pVal. +*/ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ sqlite3_int64 iVal; *pp += sqlite3Fts3GetVarint(*pp, &iVal); *pVal += iVal; } +/* +** As long as *pp has not reached its end (pEnd), then do the same +** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. +** But if we have reached the end of the varint, just set *pp=0 and +** leave *pVal unchanged. +*/ static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ if( *pp>=pEnd ){ *pp = 0; @@ -499,30 +460,47 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ return SQLITE_OK; } +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is writting +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + /* ** The xDestroy() virtual table method. */ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ Fts3Table *p = (Fts3Table *)pVtab; + sqlite3 *db = p->db; - /* Create a script to drop the underlying three storage tables. */ - char *zSql = sqlite3_mprintf( - "DROP TABLE IF EXISTS %Q.'%q_content';" - "DROP TABLE IF EXISTS %Q.'%q_segments';" - "DROP TABLE IF EXISTS %Q.'%q_segdir';", - p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName - ); - - /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to - ** execute the SQL script created above. - */ - if( zSql ){ - rc = sqlite3_exec(p->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } + /* Drop the shadow tables */ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. @@ -571,22 +549,33 @@ static int fts3DeclareVtab(Fts3Table *p){ ** as part of the vtab xCreate() method. */ static int fts3CreateTables(Fts3Table *p){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ char *zContentCols; /* Columns of %_content table */ - char *zSql; /* SQL script to create required tables */ + sqlite3 *db = p->db; /* The database connection */ /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } + if( p->bHasContent ){ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; - /* Create the whole SQL script */ - zSql = sqlite3_mprintf( - "CREATE TABLE %Q.'%q_content'(%s);" - "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);" + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segdir'(" "level INTEGER," "idx INTEGER," @@ -596,23 +585,54 @@ static int fts3CreateTables(Fts3Table *p){ "root BLOB," "PRIMARY KEY(level, idx)" ");", - p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName + p->zDb, p->zName ); - - /* Unless a malloc() failure has occurred, execute the SQL script to - ** create the tables used to store data for this FTS3 virtual table. - */ - if( zContentCols==0 || zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); } - - sqlite3_free(zSql); - sqlite3_free(zContentCols); return rc; } +/* +** An sqlite3_exec() callback for fts3TableExists. +*/ +static int fts3TableExistsCallback(void *pArg, int n, char **pp1, char **pp2){ + *(int*)pArg = 1; + return 1; +} + +/* +** Determine if a table currently exists in the database. +*/ +static void fts3TableExists( + int *pRc, /* Success code */ + sqlite3 *db, /* The database connection to test */ + const char *zDb, /* ATTACHed database within the connection */ + const char *zName, /* Name of the FTS3 table */ + const char *zSuffix, /* Shadow table extension */ + u8 *pResult /* Write results here */ +){ + int rc = SQLITE_OK; + int res = 0; + char *zSql; + if( *pRc ) return; + zSql = sqlite3_mprintf( + "SELECT 1 FROM %Q.sqlite_master WHERE name='%q%s'", + zDb, zName, zSuffix + ); + rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0); + sqlite3_free(zSql); + *pResult = res & 0xff; + if( rc!=SQLITE_ABORT ) *pRc = rc; +} + /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. @@ -624,7 +644,7 @@ static int fts3CreateTables(Fts3Table *p){ ** argv[2] -> table name ** argv[...] -> "column name" and other module argument fields. */ -int fts3InitVtab( +static int fts3InitVtab( int isCreate, /* True for xCreate, false for xConnect */ sqlite3 *db, /* The SQLite database connection */ void *pAux, /* Hash table containing tokenizers */ @@ -648,14 +668,6 @@ int fts3InitVtab( const char *zTokenizer = 0; /* Name of tokenizer to use */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ -#ifdef SQLITE_TEST - const char *zTestParam = 0; - if( strncmp(argv[argc-1], "test:", 5)==0 ){ - zTestParam = argv[argc-1]; - argc--; - } -#endif - nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; for(i=3; iazColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nNodeSize = 1000; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; zCsr = (char *)&p->azColumn[nCol]; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); @@ -736,18 +749,19 @@ int fts3InitVtab( ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ + p->bHasContent = 1; + p->bHasDocsize = argv[0][3]=='4'; rc = fts3CreateTables(p); - if( rc!=SQLITE_OK ) goto fts3_init_out; + }else{ + rc = SQLITE_OK; + fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent); + fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize); } + if( rc!=SQLITE_OK ) goto fts3_init_out; rc = fts3DeclareVtab(p); if( rc!=SQLITE_OK ) goto fts3_init_out; -#ifdef SQLITE_TEST - if( zTestParam ){ - p->nNodeSize = atoi(&zTestParam[5]); - } -#endif *ppVTab = &p->base; fts3_init_out: @@ -810,31 +824,31 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; if( pCons->usable==0 ) continue; - /* A direct lookup on the rowid or docid column. This is the best - ** strategy in all cases. Assign a cost of 1.0 and return early. - */ + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) ){ pInfo->idxNum = FTS3_DOCID_SEARCH; pInfo->estimatedCost = 1.0; iCons = i; - break; } /* A MATCH constraint. Use a full-text search. ** ** If there is more than one MATCH constraint available, use the first ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the rowid/docid strategy. + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. */ - if( iCons<0 - && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn ){ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; pInfo->estimatedCost = 2.0; iCons = i; + break; } } @@ -865,12 +879,6 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ return SQLITE_OK; } -/****************************************************************/ -/****************************************************************/ -/****************************************************************/ -/****************************************************************/ - - /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. @@ -880,21 +888,29 @@ static int fulltextClose(sqlite3_vtab_cursor *pCursor){ sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); sqlite3_free(pCsr); return SQLITE_OK; } -static int fts3CursorSeek(Fts3Cursor *pCsr){ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ if( pCsr->isRequireSeek ){ pCsr->isRequireSeek = 0; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; }else{ - int rc; + int rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK ){ + /* If no row was found and no error has occured, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. + */ + rc = SQLITE_CORRUPT; + } pCsr->isEof = 1; - if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){ - rc = SQLITE_ERROR; + if( pContext ){ + sqlite3_result_error_code(pContext, rc); } return rc; } @@ -918,6 +934,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ sqlite3_reset(pCsr->pStmt); fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; } return rc; } @@ -1033,11 +1050,32 @@ static void fts3PutDeltaVarint( *piPrev = iVal; } +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ static void fts3PoslistCopy(char **pp, char **ppPoslist){ char *pEnd = *ppPoslist; char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single 0x00 byte. Except, if the 0x00 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following block moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ while( *pEnd | c ) c = *pEnd++ & 0x80; pEnd++; + if( pp ){ int n = (int)(pEnd - *ppPoslist); char *p = *pp; @@ -1244,9 +1282,10 @@ static int fts3PoslistPhraseMerge( if( (*p1&0xFE)==0 ) break; fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; } - } - if( pSave && pp ){ + + if( pSave ){ + assert( pp && p ); p = pSave; } @@ -1385,11 +1424,8 @@ static int fts3DoclistMerge( ); if( !aBuffer ){ - return SQLITE_NOMEM; - } - if( n1==0 && n2==0 ){ *pnBuffer = 0; - return SQLITE_OK; + return SQLITE_NOMEM; } /* Read the first docid from each doclist */ @@ -1475,7 +1511,7 @@ static int fts3DoclistMerge( char **ppPos = 0; if( mergetype==MERGE_POS_NEAR ){ ppPos = &p; - aTmp = sqlite3_malloc(2*(n1+n2)); + aTmp = sqlite3_malloc(2*(n1+n2+1)); if( !aTmp ){ return SQLITE_NOMEM; } @@ -1595,13 +1631,21 @@ static int fts3TermSelect( int i; TermSelect tsc; Fts3SegFilter filter; /* Segment term filter configuration */ - Fts3SegReader **apSegment = 0; /* Array of segments to read data from */ + Fts3SegReader **apSegment; /* Array of segments to read data from */ int nSegment = 0; /* Size of apSegment array */ - int nAlloc = 0; /* Allocated size of segment array */ + int nAlloc = 16; /* Allocated size of segment array */ int rc; /* Return code */ - sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */ + sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ int iAge = 0; /* Used to assign ages to segments */ + apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc); + if( !apSegment ) return SQLITE_NOMEM; + rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]); + if( rc!=SQLITE_OK ) goto finished; + if( apSegment[0] ){ + nSegment = 1; + } + /* Loop through the entire %_segdir table. For each segment, create a ** Fts3SegReader to iterate through the subset of the segment leaves ** that may contain a term that matches zTerm/nTerm. For non-prefix @@ -1718,8 +1762,6 @@ static int fts3PhraseSelect( int iCol = pPhrase->iColumn; int isTermPos = (pPhrase->nToken>1 || isReqPos); - assert( p->nPendingData==0 ); - for(ii=0; iinToken; ii++){ struct PhraseToken *pTok = &pPhrase->aToken[ii]; char *z = pTok->z; /* Next token of the phrase */ @@ -1751,6 +1793,7 @@ static int fts3PhraseSelect( sqlite3_free(pOut); pOut = pList; } + assert( nOut==0 || pOut!=0 ); } if( rc==SQLITE_OK ){ @@ -1762,6 +1805,74 @@ static int fts3PhraseSelect( return rc; } +static int fts3NearMerge( + int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ + int nNear, /* Parameter to NEAR operator */ + int nTokenLeft, /* Number of tokens in LHS phrase arg */ + char *aLeft, /* Doclist for LHS (incl. positions) */ + int nLeft, /* Size of LHS doclist in bytes */ + int nTokenRight, /* As nTokenLeft */ + char *aRight, /* As aLeft */ + int nRight, /* As nRight */ + char **paOut, /* OUT: Results of merge (malloced) */ + int *pnOut /* OUT: Sized of output buffer */ +){ + char *aOut; + int rc; + + assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); + + aOut = sqlite3_malloc(nLeft+nRight+1); + if( aOut==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, + aOut, pnOut, aLeft, nLeft, aRight, nRight + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + aOut = 0; + } + } + + *paOut = aOut; + return rc; +} + +int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ + int rc; + if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ + sqlite3_free(pLeft->aDoclist); + sqlite3_free(pRight->aDoclist); + pRight->aDoclist = 0; + pLeft->aDoclist = 0; + rc = SQLITE_OK; + }else{ + char *aOut; + int nOut; + + rc = fts3NearMerge(MERGE_POS_NEAR, nNear, + pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, + pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, + &aOut, &nOut + ); + if( rc!=SQLITE_OK ) return rc; + sqlite3_free(pRight->aDoclist); + pRight->aDoclist = aOut; + pRight->nDoclist = nOut; + + rc = fts3NearMerge(MERGE_POS_NEAR, nNear, + pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, + pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, + &aOut, &nOut + ); + sqlite3_free(pLeft->aDoclist); + pLeft->aDoclist = aOut; + pLeft->nDoclist = nOut; + } + return rc; +} + /* ** Evaluate the full-text expression pExpr against fts3 table pTab. Store ** the resulting doclist in *paOut and *pnOut. @@ -1770,7 +1881,8 @@ static int evalFts3Expr( Fts3Table *p, /* Virtual table handle */ Fts3Expr *pExpr, /* Parsed fts3 expression */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut /* OUT: Size of buffer at *paOut */ + int *pnOut, /* OUT: Size of buffer at *paOut */ + int isReqPos /* Require positions in output buffer */ ){ int rc = SQLITE_OK; /* Return code */ @@ -1779,17 +1891,23 @@ static int evalFts3Expr( *pnOut = 0; if( pExpr ){ + assert( pExpr->eType==FTSQUERY_PHRASE + || pExpr->eType==FTSQUERY_NEAR + || isReqPos==0 + ); if( pExpr->eType==FTSQUERY_PHRASE ){ - int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR); - rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut); + rc = fts3PhraseSelect(p, pExpr->pPhrase, + isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), + paOut, pnOut + ); }else{ char *aLeft; char *aRight; int nLeft; int nRight; - if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight)) - && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft)) + if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) + && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) ){ assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT @@ -1798,10 +1916,7 @@ static int evalFts3Expr( case FTSQUERY_NEAR: { Fts3Expr *pLeft; Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - int nParam1; - int nParam2; - char *aBuffer; + int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR; if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ mergetype = MERGE_POS_NEAR; @@ -1814,17 +1929,11 @@ static int evalFts3Expr( assert( pRight->eType==FTSQUERY_PHRASE ); assert( pLeft->eType==FTSQUERY_PHRASE ); - nParam1 = pExpr->nNear+1; - nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2; - aBuffer = sqlite3_malloc(nLeft+nRight+1); - rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer, - pnOut, aLeft, nLeft, aRight, nRight + rc = fts3NearMerge(mergetype, pExpr->nNear, + pLeft->pPhrase->nToken, aLeft, nLeft, + pRight->pPhrase->nToken, aRight, nRight, + paOut, pnOut ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aBuffer); - }else{ - *paOut = aBuffer; - } sqlite3_free(aLeft); break; } @@ -1908,6 +2017,7 @@ static int fts3FilterMethod( /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); + sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); /* Compile a SELECT statement for this cursor. For a full-table-scan, the @@ -1934,15 +2044,19 @@ static int fts3FilterMethod( if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ return SQLITE_NOMEM; } - rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ) return rc; rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_ERROR ){ + p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", + zQuery); + } + return rc; + } - rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist); + rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; } @@ -1991,21 +2105,22 @@ static int fts3ColumnMethod( /* The column value supplied by SQLite must be in range. */ assert( iCol>=0 && iCol<=p->nColumn+1 ); - rc = fts3CursorSeek(pCsr); - if( rc==SQLITE_OK ){ - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. - */ - sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - }else{ + if( iCol==p->nColumn+1 ){ + /* This call is a request for the "docid" column. Since "docid" is an + ** alias for "rowid", use the xRowid() method to obtain the value. + */ + sqlite3_int64 iRowid; + rc = fts3RowidMethod(pCursor, &iRowid); + sqlite3_result_int64(pContext, iRowid); + }else if( iCol==p->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor. + */ + sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); + rc = SQLITE_OK; + }else{ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } @@ -2063,6 +2178,65 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ return SQLITE_OK; } +/* +** Load the doclist associated with expression pExpr to pExpr->aDoclist. +** The loaded doclist contains positions as well as the document ids. +** This is used by the matchinfo(), snippet() and offsets() auxillary +** functions. +*/ +int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){ + return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); +} + +/* +** After ExprLoadDoclist() (see above) has been called, this function is +** used to iterate/search through the position lists that make up the doclist +** stored in pExpr->aDoclist. +*/ +char *sqlite3Fts3FindPositions( + Fts3Expr *pExpr, /* Access this expressions doclist */ + sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ + int iCol /* Column of requested pos-list */ +){ + assert( pExpr->isLoaded ); + if( pExpr->aDoclist ){ + char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + char *pCsr = pExpr->pCurrent; + + assert( pCsr ); + while( pCsriCurrentiCurrent); + } + pExpr->pCurrent = pCsr; + }else{ + if( pExpr->iCurrent==iDocid ){ + int iThis = 0; + if( iCol<0 ){ + /* If iCol is negative, return a pointer to the start of the + ** position-list (instead of a pointer to the start of a list + ** of offsets associated with a specific column). + */ + return pCsr; + } + while( iThis=1 ); - if( nVal>4 ){ + if( nVal>6 ){ sqlite3_result_error(pContext, "wrong number of arguments to function snippet()", -1); return; @@ -2119,12 +2295,17 @@ static void fts3SnippetFunc( if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + case 5: iCol = sqlite3_value_int(apVal[4]); case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); } - - sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis); + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } } /* @@ -2142,7 +2323,9 @@ static void fts3OffsetsFunc( assert( nVal==1 ); if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; assert( pCsr ); - sqlite3Fts3Offsets(pContext, pCsr); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } } /* @@ -2185,6 +2368,27 @@ static void fts3OptimizeFunc( } } +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + if( nVal!=1 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function matchinfo()", -1); + return; + } + + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + sqlite3Fts3Matchinfo(pContext, pCsr); + } +} + /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. @@ -2203,6 +2407,7 @@ static int fts3FindFunctionMethod( { "snippet", fts3SnippetFunc }, { "offsets", fts3OffsetsFunc }, { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, }; int i; /* Iterator variable */ @@ -2228,22 +2433,35 @@ static int fts3RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_NOMEM; /* Return Code */ - char *zSql; /* SQL script to run to rename tables */ + Fts3Table *p = (Fts3Table *)pVtab; + sqlite3 *db; /* Database connection */ + int rc; /* Return Code */ - zSql = sqlite3_mprintf( - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';" - "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';" - "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';" - , p->zDb, p->zName, zName - , p->zDb, p->zName, zName - , p->zDb, p->zName, zName + db = p->db; + rc = SQLITE_OK; + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName ); - if( zSql ){ - rc = sqlite3_exec(p->db, zSql, 0, 0, 0); - sqlite3_free(zSql); + if( rc==SQLITE_ERROR ) rc = SQLITE_OK; + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); return rc; } @@ -2307,13 +2525,14 @@ int sqlite3Fts3Init(sqlite3 *db){ Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; + +#ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); -#ifdef SQLITE_ENABLE_ICU - sqlite3Fts3IcuTokenizerModule(&pIcu); -#endif /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); @@ -2327,14 +2546,18 @@ int sqlite3Fts3Init(sqlite3 *db){ if( rc==SQLITE_OK ){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) +#ifdef SQLITE_ENABLE_ICU || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) +#endif ){ rc = SQLITE_NOMEM; } } #ifdef SQLITE_TEST - sqlite3Fts3ExprInitTestInterface(db); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ExprInitTestInterface(db); + } #endif /* Create the virtual table wrapper around the hash-table and overload @@ -2345,11 +2568,18 @@ int sqlite3Fts3Init(sqlite3 *db){ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ - return sqlite3_create_module_v2( + rc = sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module_v2( + db, "fts4", &fts3Module, (void *)pHash, 0 + ); + } + return rc; } /* An error has occurred. Delete the hash table and return the error code. */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index d5947864..c62cb0d0 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -70,6 +70,8 @@ */ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ /* ** Macro used to suppress compiler warnings for unused parameters. */ @@ -102,7 +104,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[18]; + sqlite3_stmt *aStmt[25]; /* Pointer to string containing the SQL: ** @@ -116,14 +118,17 @@ struct Fts3Table { sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */ int nNodeSize; /* Soft limit for node size */ + u8 bHasContent; /* True if %_content table exists */ + u8 bHasDocsize; /* True if %_docsize table exists */ /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. - ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed + ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. */ + int nMaxPendingData; int nPendingData; sqlite_int64 iPrevDocid; Fts3Hash pendingTerms; @@ -145,6 +150,8 @@ struct Fts3Cursor { char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + u32 *aMatchinfo; /* Information about most recent match */ }; /* @@ -185,6 +192,16 @@ struct Fts3Phrase { /* ** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded +** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, +** containing the results of the NEAR or phrase query in FTS3 doclist +** format. As usual, the initial "Length" field found in doclists stored +** on disk is omitted from this buffer. +** +** Variable pCurrent always points to the start of a docid field within +** aDoclist. Since the doclist is usually scanned in docid order, this can +** be used to accelerate seeking to the required docid within the doclist. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ @@ -193,6 +210,13 @@ struct Fts3Expr { Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + int isLoaded; /* True if aDoclist/nDoclist are initialized. */ + char *aDoclist; /* Buffer containing doclist */ + int nDoclist; /* Size of aDoclist in bytes */ + + sqlite3_int64 iCurrent; + char *pCurrent; }; /* @@ -225,6 +249,7 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *); int sqlite3Fts3SegReaderIterate( Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, @@ -232,6 +257,8 @@ int sqlite3Fts3SegReaderIterate( ); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); +int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); +int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -254,6 +281,10 @@ int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); +char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); +int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *); +int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); + /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); @@ -263,9 +294,10 @@ int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, /* fts3_snippet.c */ void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); -void sqlite3Fts3Snippet(sqlite3_context*, Fts3Cursor*, - const char *, const char *, const char * +void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int ); +void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, @@ -273,7 +305,7 @@ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST -void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 5b955f9c..7542c28a 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -181,7 +181,7 @@ static int getNextToken( ** Enlarge a memory allocation. If an out-of-memory allocation occurs, ** then free the old allocation. */ -void *fts3ReallocOrFree(void *pOrig, int nNew){ +static void *fts3ReallocOrFree(void *pOrig, int nNew){ void *pRet = sqlite3_realloc(pOrig, nNew); if( !pRet ){ sqlite3_free(pOrig); @@ -735,6 +735,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); + sqlite3_free(p->aDoclist); sqlite3_free(p); } } @@ -913,8 +914,8 @@ exprtest_out: ** Register the query expression parser test function fts3_exprtest() ** with database connection db. */ -void sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - sqlite3_create_function( +int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ + return sqlite3_create_function( db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 ); } diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 3cb91f3d..98be5296 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -279,13 +279,12 @@ static void fts3RemoveElementByHash( } } -/* Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is -** found, or NULL if there is no match. -*/ -void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ - int h; /* A hash on key */ - Fts3HashElem *elem; /* The element that matches key */ +Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ int (*xHash)(const void*,int); /* The hash function */ if( pH==0 || pH->ht==0 ) return 0; @@ -293,8 +292,19 @@ void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ assert( xHash!=0 ); h = (*xHash)(pKey,nKey); assert( (pH->htsize & (pH->htsize-1))==0 ); - elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); - return elem ? elem->data : 0; + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; } /* Insert an element into the hash table pH. The key is pKey,nKey diff --git a/ext/fts3/fts3_hash.h b/ext/fts3/fts3_hash.h index b00f365b..399f5154 100644 --- a/ext/fts3/fts3_hash.h +++ b/ext/fts3/fts3_hash.h @@ -75,14 +75,16 @@ void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); void sqlite3Fts3HashClear(Fts3Hash*); +Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); /* ** Shorthand for the functions above */ -#define fts3HashInit sqlite3Fts3HashInit -#define fts3HashInsert sqlite3Fts3HashInsert -#define fts3HashFind sqlite3Fts3HashFind -#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem /* ** Macros for looping over all elements of a hash table. The idiom is diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index 4e680872..4e307781 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -605,9 +605,11 @@ static int porterNext( if( c->iOffset>iStartOffset ){ int n = c->iOffset-iStartOffset; if( n>c->nAllocated ){ + char *pNew; c->nAllocated = n+20; - c->zToken = sqlite3_realloc(c->zToken, c->nAllocated); - if( c->zToken==NULL ) return SQLITE_NOMEM; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; } porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); *pzToken = c->zToken; diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index cdc55aff..7aa86607 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -18,717 +18,1189 @@ #include #include -typedef struct Snippet Snippet; /* -** An instance of the following structure keeps track of generated -** matching-word offset information and snippets. +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. */ -struct Snippet { - int nMatch; /* Total number of matches */ - int nAlloc; /* Space allocated for aMatch[] */ - struct snippetMatch { /* One entry for each matching term */ - char snStatus; /* Status flag for use while constructing snippets */ - short int nByte; /* Number of bytes in the term */ - short int iCol; /* The column that contains the match */ - short int iTerm; /* The index in Query.pTerms[] of the matching term */ - int iToken; /* The index of the matching document token */ - int iStart; /* The offset to the first character of the term */ - } *aMatch; /* Points to space obtained from malloc */ - char *zOffset; /* Text rendering of aMatch[] */ - int nOffset; /* strlen(zOffset) */ - char *zSnippet; /* Snippet text */ - int nSnippet; /* strlen(zSnippet) */ +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Table *pTab; /* FTS3 Table */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; + +/* +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. +*/ +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; + +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; + +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + int iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + int iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; + +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; + +/* +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. +*/ +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + u32 *aMatchinfo; /* Pre-allocated buffer */ }; -/* It is not safe to call isspace(), tolower(), or isalnum() on -** hi-bit-set characters. This is the same solution used in the -** tokenizer. + +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. */ -static int fts3snippetIsspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; -} +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; /* -** A StringBuffer object holds a zero-terminated string that grows -** arbitrarily by appending. Space to hold the string is obtained -** from sqlite3_malloc(). After any memory allocation failure, -** StringBuffer.z is set to NULL and no further allocation is attempted. -*/ -typedef struct StringBuffer { - char *z; /* Text of the string. Space from malloc. */ - int nUsed; /* Number bytes of z[] used, not counting \000 terminator */ - int nAlloc; /* Bytes allocated for z[] */ -} StringBuffer; - - -/* -** Initialize a new StringBuffer. -*/ -static void fts3SnippetSbInit(StringBuffer *p){ - p->nAlloc = 100; - p->nUsed = 0; - p->z = sqlite3_malloc( p->nAlloc ); -} - -/* -** Append text to the string buffer. -*/ -static void fts3SnippetAppend(StringBuffer *p, const char *zNew, int nNew){ - if( p->z==0 ) return; - if( nNew<0 ) nNew = (int)strlen(zNew); - if( p->nUsed + nNew >= p->nAlloc ){ - int nAlloc; - char *zNew; - - nAlloc = p->nUsed + nNew + p->nAlloc; - zNew = sqlite3_realloc(p->z, nAlloc); - if( zNew==0 ){ - sqlite3_free(p->z); - p->z = 0; - return; - } - p->z = zNew; - p->nAlloc = nAlloc; - } - memcpy(&p->z[p->nUsed], zNew, nNew); - p->nUsed += nNew; - p->z[p->nUsed] = 0; -} - -/* If the StringBuffer ends in something other than white space, add a -** single space character to the end. -*/ -static void fts3SnippetAppendWhiteSpace(StringBuffer *p){ - if( p->z && p->nUsed && !fts3snippetIsspace(p->z[p->nUsed-1]) ){ - fts3SnippetAppend(p, " ", 1); - } -} - -/* Remove white space from the end of the StringBuffer */ -static void fts3SnippetTrimWhiteSpace(StringBuffer *p){ - if( p->z ){ - while( p->nUsed && fts3snippetIsspace(p->z[p->nUsed-1]) ){ - p->nUsed--; - } - p->z[p->nUsed] = 0; - } -} - -/* -** Release all memory associated with the Snippet structure passed as -** an argument. -*/ -static void fts3SnippetFree(Snippet *p){ - if( p ){ - sqlite3_free(p->aMatch); - sqlite3_free(p->zOffset); - sqlite3_free(p->zSnippet); - sqlite3_free(p); - } -} - -/* -** Append a single entry to the p->aMatch[] log. -*/ -static int snippetAppendMatch( - Snippet *p, /* Append the entry to this snippet */ - int iCol, int iTerm, /* The column and query term */ - int iToken, /* Matching token in document */ - int iStart, int nByte /* Offset and size of the match */ -){ - int i; - struct snippetMatch *pMatch; - if( p->nMatch+1>=p->nAlloc ){ - struct snippetMatch *pNew; - p->nAlloc = p->nAlloc*2 + 10; - pNew = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); - if( pNew==0 ){ - p->aMatch = 0; - p->nMatch = 0; - p->nAlloc = 0; - return SQLITE_NOMEM; - } - p->aMatch = pNew; - } - i = p->nMatch++; - pMatch = &p->aMatch[i]; - pMatch->iCol = (short)iCol; - pMatch->iTerm = (short)iTerm; - pMatch->iToken = iToken; - pMatch->iStart = iStart; - pMatch->nByte = (short)nByte; - return SQLITE_OK; -} - -/* -** Sizing information for the circular buffer used in snippetOffsetsOfColumn() -*/ -#define FTS3_ROTOR_SZ (32) -#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) - -/* -** Function to iterate through the tokens of a compiled expression. +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: ** -** Except, skip all tokens on the right-hand side of a NOT operator. -** This function is used to find tokens as part of snippet and offset -** generation and we do nt want snippets and offsets to report matches -** for tokens on the RHS of a NOT. +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. */ -static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ - Fts3Expr *p = *ppExpr; - int iToken = *piToken; - if( iToken<0 ){ - /* In this case the expression p is the root of an expression tree. - ** Move to the first token in the expression tree. - */ - while( p->pLeft ){ - p = p->pLeft; +static void fts3GetDeltaPosition(char **pp, int *piPos){ + int iVal; + *pp += sqlite3Fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); } - iToken = 0; }else{ - assert(p && p->eType==FTSQUERY_PHRASE ); - if( iToken<(p->pPhrase->nToken-1) ){ - iToken++; - }else{ - iToken = 0; - while( p->pParent && p->pParent->pLeft!=p ){ - assert( p->pParent->pRight==p ); - p = p->pParent; - } - p = p->pParent; - if( p ){ - assert( p->pRight!=0 ); - p = p->pRight; - while( p->pLeft ){ - p = p->pLeft; - } - } - } + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; } - - *ppExpr = p; - *piToken = iToken; - return p?1:0; + return rc; } /* -** Return TRUE if the expression node pExpr is located beneath the -** RHS of a NOT operator. +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. */ -static int fts3ExprBeneathNot(Fts3Expr *p){ - Fts3Expr *pParent; - while( p ){ - pParent = p->pParent; - if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ - return 1; - } - p = pParent; - } - return 0; -} - -/* -** Add entries to pSnippet->aMatch[] for every match that occurs against -** document zDoc[0..nDoc-1] which is stored in column iColumn. -*/ -static int snippetOffsetsOfColumn( - Fts3Cursor *pCur, /* The fulltest search cursor */ - Snippet *pSnippet, /* The Snippet object to be filled in */ - int iColumn, /* Index of fulltext table column */ - const char *zDoc, /* Text of the fulltext table column */ - int nDoc /* Length of zDoc in bytes */ +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ ){ - const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ - sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ - sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ - Fts3Table *pVtab; /* The full text index */ - int nColumn; /* Number of columns in the index */ - int i, j; /* Loop counters */ - int rc; /* Return code */ - unsigned int match, prevMatch; /* Phrase search bitmasks */ - const char *zToken; /* Next token from the tokenizer */ - int nToken; /* Size of zToken */ - int iBegin, iEnd, iPos; /* Offsets of beginning and end */ - - /* The following variables keep a circular buffer of the last - ** few tokens */ - unsigned int iRotor = 0; /* Index of current token */ - int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ - int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ - - pVtab = (Fts3Table *)pCur->base.pVtab; - nColumn = pVtab->nColumn; - pTokenizer = pVtab->pTokenizer; - pTModule = pTokenizer->pModule; - rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); - if( rc ) return rc; - pTCursor->pTokenizer = pTokenizer; - - prevMatch = 0; - while( (rc = pTModule->xNext(pTCursor, &zToken, &nToken, - &iBegin, &iEnd, &iPos))==SQLITE_OK ){ - Fts3Expr *pIter = pCur->pExpr; - int iIter = -1; - iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; - iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; - match = 0; - for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){ - int nPhrase; /* Number of tokens in current phrase */ - struct PhraseToken *pToken; /* Current token */ - int iCol; /* Column index */ - - if( fts3ExprBeneathNot(pIter) ) continue; - nPhrase = pIter->pPhrase->nToken; - pToken = &pIter->pPhrase->aToken[iIter]; - iCol = pIter->pPhrase->iColumn; - if( iCol>=0 && iColn>nToken ) continue; - if( !pToken->isPrefix && pToken->nn<=nToken ); - if( memcmp(pToken->z, zToken, pToken->n) ) continue; - if( iIter>0 && (prevMatch & (1<=0; j--){ - int k = (iRotor-j) & FTS3_ROTOR_MASK; - rc = snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, - iRotorBegin[k], iRotorLen[k]); - if( rc ) goto end_offsets_of_column; - } - } - } - prevMatch = match<<1; - iRotor++; - } -end_offsets_of_column: - pTModule->xClose(pTCursor); - return rc==SQLITE_DONE ? SQLITE_OK : rc; + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } /* -** Remove entries from the pSnippet structure to account for the NEAR -** operator. When this is called, pSnippet contains the list of token -** offsets produced by treating all NEAR operators as AND operators. -** This function removes any entries that should not be present after -** accounting for the NEAR restriction. For example, if the queried -** document is: +** The argument to this function is always a phrase node. Its doclist +** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes +** to the left of this one in the query tree have already been loaded. ** -** "A B C D E A" +** If this phrase node is part of a series of phrase nodes joined by +** NEAR operators (and is not the left-most of said series), then elements are +** removed from the phrases doclist consistent with the NEAR restriction. If +** required, elements may be removed from the doclists of phrases to the +** left of this one that are part of the same series of NEAR operator +** connected phrases. ** -** and the query is: -** -** A NEAR/0 E -** -** then when this function is called the Snippet contains token offsets -** 0, 4 and 5. This function removes the "0" entry (because the first A -** is not near enough to an E). -** -** When this function is called, the value pointed to by parameter piLeft is -** the integer id of the left-most token in the expression tree headed by -** pExpr. This function increments *piLeft by the total number of tokens -** in the expression tree headed by pExpr. -** -** Return 1 if any trimming occurs. Return 0 if no trimming is required. +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -static int trimSnippetOffsets( - Fts3Expr *pExpr, /* The search expression */ - Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ - int *piLeft /* Index of left-most token in pExpr */ -){ - if( pExpr ){ - if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ - return 1; - } - - switch( pExpr->eType ){ - case FTSQUERY_PHRASE: - *piLeft += pExpr->pPhrase->nToken; - break; - case FTSQUERY_NEAR: { - /* The right-hand-side of a NEAR operator is always a phrase. The - ** left-hand-side is either a phrase or an expression tree that is - ** itself headed by a NEAR operator. The following initializations - ** set local variable iLeft to the token number of the left-most - ** token in the right-hand phrase, and iRight to the right most - ** token in the same phrase. For example, if we had: - ** - ** MATCH '"abc def" NEAR/2 "ghi jkl"' - ** - ** then iLeft will be set to 2 (token number of ghi) and nToken will - ** be set to 4. - */ - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - int iLeft = *piLeft; - int nNear = pExpr->nNear; - int nToken = pRight->pPhrase->nToken; - int jj, ii; - if( pLeft->eType==FTSQUERY_NEAR ){ - pLeft = pLeft->pRight; - } - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - nToken += pLeft->pPhrase->nToken; - - for(ii=0; iinMatch; ii++){ - struct snippetMatch *p = &pSnippet->aMatch[ii]; - if( p->iTerm==iLeft ){ - int isOk = 0; - /* Snippet ii is an occurence of query term iLeft in the document. - ** It occurs at position (p->iToken) of the document. We now - ** search for an instance of token (iLeft-1) somewhere in the - ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within - ** the set of snippetMatch structures. If one is found, proceed. - ** If one cannot be found, then remove snippets ii..(ii+N-1) - ** from the matching snippets, where N is the number of tokens - ** in phrase pRight->pPhrase. - */ - for(jj=0; isOk==0 && jjnMatch; jj++){ - struct snippetMatch *p2 = &pSnippet->aMatch[jj]; - if( p2->iTerm==(iLeft-1) ){ - if( p2->iToken>=(p->iToken-nNear-1) - && p2->iToken<(p->iToken+nNear+nToken) - ){ - isOk = 1; - } - } - } - if( !isOk ){ - int kk; - for(kk=0; kkpPhrase->nToken; kk++){ - pSnippet->aMatch[kk+ii].iTerm = -2; - } - return 1; - } - } - if( p->iTerm==(iLeft-1) ){ - int isOk = 0; - for(jj=0; isOk==0 && jjnMatch; jj++){ - struct snippetMatch *p2 = &pSnippet->aMatch[jj]; - if( p2->iTerm==iLeft ){ - if( p2->iToken<=(p->iToken+nNear+1) - && p2->iToken>(p->iToken-nNear-nToken) - ){ - isOk = 1; - } - } - } - if( !isOk ){ - int kk; - for(kk=0; kkpPhrase->nToken; kk++){ - pSnippet->aMatch[ii-kk].iTerm = -2; - } - return 1; - } - } - } - break; - } - } - - if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ - return 1; - } - } - return 0; -} - -/* -** Compute all offsets for the current row of the query. -** If the offsets have already been computed, this routine is a no-op. -*/ -static int snippetAllOffsets(Fts3Cursor *pCsr, Snippet **ppSnippet){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; /* The FTS3 virtual table */ - int nColumn; /* Number of columns. Docid does count */ - int iColumn; /* Index of of a column */ - int i; /* Loop index */ - int iFirst; /* First column to search */ - int iLast; /* Last coumn to search */ - int iTerm = 0; - Snippet *pSnippet; +static int fts3ExprNearTrim(Fts3Expr *pExpr){ int rc = SQLITE_OK; + Fts3Expr *pParent = pExpr->pParent; - if( pCsr->pExpr==0 ){ - return SQLITE_OK; - } + assert( pExpr->eType==FTSQUERY_PHRASE ); + while( rc==SQLITE_OK + && pParent + && pParent->eType==FTSQUERY_NEAR + && pParent->pRight==pExpr + ){ + /* This expression (pExpr) is the right-hand-side of a NEAR operator. + ** Find the expression to the left of the same operator. + */ + int nNear = pParent->nNear; + Fts3Expr *pLeft = pParent->pLeft; - pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet)); - *ppSnippet = pSnippet; - if( !pSnippet ){ - return SQLITE_NOMEM; - } - memset(pSnippet, 0, sizeof(Snippet)); - - nColumn = p->nColumn; - iColumn = (pCsr->eSearch - 2); - if( iColumn<0 || iColumn>=nColumn ){ - /* Look for matches over all columns of the full-text index */ - iFirst = 0; - iLast = nColumn-1; - }else{ - /* Look for matches in the iColumn-th column of the index only */ - iFirst = iColumn; - iLast = iColumn; - } - for(i=iFirst; rc==SQLITE_OK && i<=iLast; i++){ - const char *zDoc; - int nDoc; - zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1); - nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1); - if( zDoc==0 && sqlite3_column_type(pCsr->pStmt, i+1)!=SQLITE_NULL ){ - rc = SQLITE_NOMEM; - }else{ - rc = snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc); + if( pLeft->eType!=FTSQUERY_PHRASE ){ + assert( pLeft->eType==FTSQUERY_NEAR ); + assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); + pLeft = pLeft->pRight; } - } - while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){ - iTerm = 0; + rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); + + pExpr = pLeft; + pParent = pExpr->pParent; } return rc; } /* -** Convert the information in the aMatch[] array of the snippet -** into the string zOffset[0..nOffset-1]. This string is used as -** the return of the SQL offsets() function. +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). */ -static void snippetOffsetText(Snippet *p){ - int i; - int cnt = 0; - StringBuffer sb; - char zBuf[200]; - if( p->zOffset ) return; - fts3SnippetSbInit(&sb); - for(i=0; inMatch; i++){ - struct snippetMatch *pMatch = &p->aMatch[i]; - if( pMatch->iTerm>=0 ){ - /* If snippetMatch.iTerm is less than 0, then the match was - ** discarded as part of processing the NEAR operator (see the - ** trimSnippetOffsetsForNear() function for details). Ignore - ** it in this case - */ - zBuf[0] = ' '; - sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", - pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); - fts3SnippetAppend(&sb, zBuf, -1); - cnt++; +static int fts3ExprLoadDoclistsCb1(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pExpr->pPhrase->nToken; + + if( pExpr->isLoaded==0 ){ + rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr); + pExpr->isLoaded = 1; + if( rc==SQLITE_OK ){ + rc = fts3ExprNearTrim(pExpr); } } - p->zOffset = sb.z; - p->nOffset = sb.z ? sb.nUsed : 0; + + return rc; } /* -** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set -** of matching words some of which might be in zDoc. zDoc is column -** number iCol. +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb2(Fts3Expr *pExpr, int iPhrase, void *ctx){ + UNUSED_PARAMETER(iPhrase); + UNUSED_PARAMETER(ctx); + if( pExpr->aDoclist ){ + pExpr->pCurrent = pExpr->aDoclist; + pExpr->iCurrent = 0; + pExpr->pCurrent += sqlite3Fts3GetVarint(pExpr->pCurrent, &pExpr->iCurrent); + } + return SQLITE_OK; +} + +/* +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. ** -** iBreak is suggested spot in zDoc where we could begin or end an -** excerpt. Return a value similar to iBreak but possibly adjusted -** to be a little left or right so that the break point is better. +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. */ -static int wordBoundary( - int iBreak, /* The suggested break point */ - const char *zDoc, /* Document text */ - int nDoc, /* Number of bytes in zDoc[] */ - struct snippetMatch *aMatch, /* Matching words */ - int nMatch, /* Number of entries in aMatch[] */ - int iCol /* The column number for zDoc[] */ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ ){ - int i; - if( iBreak<=10 ){ - return 0; + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pTab = (Fts3Table *)pCsr->base.pVtab; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx); + if( rc==SQLITE_OK ){ + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0); } - if( iBreak>=nDoc-10 ){ - return nDoc; - } - for(i=0; ALWAYS(i0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ - return aMatch[i-1].iStart; - } - } - for(i=1; i<=10; i++){ - if( fts3snippetIsspace(zDoc[iBreak-i]) ){ - return iBreak - i + 1; - } - if( fts3snippetIsspace(zDoc[iBreak+i]) ){ - return iBreak + i + 1; - } - } - return iBreak; + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; } - - /* -** Allowed values for Snippet.aMatch[].snStatus +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. */ -#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ -#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ +static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + int iIter = *piIter; -/* -** Generate the text of a snippet. -*/ -static void snippetText( - Fts3Cursor *pCursor, /* The cursor we need the snippet for */ - Snippet *pSnippet, - const char *zStartMark, /* Markup to appear before each match */ - const char *zEndMark, /* Markup to appear after each match */ - const char *zEllipsis /* Ellipsis mark */ -){ - int i, j; - struct snippetMatch *aMatch; - int nMatch; - int nDesired; - StringBuffer sb; - int tailCol; - int tailOffset; - int iCol; - int nDoc; - const char *zDoc; - int iStart, iEnd; - int tailEllipsis = 0; - int iMatch; - - - sqlite3_free(pSnippet->zSnippet); - pSnippet->zSnippet = 0; - aMatch = pSnippet->aMatch; - nMatch = pSnippet->nMatch; - fts3SnippetSbInit(&sb); - - for(i=0; i0; i++){ - if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; - nDesired--; - iCol = aMatch[i].iCol; - zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); - nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); - iStart = aMatch[i].iStart - 40; - iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); - if( iStart<=10 ){ - iStart = 0; - } - if( iCol==tailCol && iStart<=tailOffset+20 ){ - iStart = tailOffset; - } - if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ - fts3SnippetTrimWhiteSpace(&sb); - fts3SnippetAppendWhiteSpace(&sb); - fts3SnippetAppend(&sb, zEllipsis, -1); - fts3SnippetAppendWhiteSpace(&sb); - } - iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; - iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); - if( iEnd>=nDoc-10 ){ - iEnd = nDoc; - tailEllipsis = 0; - }else{ - tailEllipsis = 1; - } - while( iMatchzSnippet = sb.z; - pSnippet->nSnippet = sb.z ? sb.nUsed : 0; } -void sqlite3Fts3Offsets( - sqlite3_context *pCtx, /* SQLite function call context */ - Fts3Cursor *pCsr /* Cursor object */ -){ - Snippet *p; /* Snippet structure */ - int rc = snippetAllOffsets(pCsr, &p); - if( rc==SQLITE_OK ){ - snippetOffsetText(p); - if( p->zOffset ){ - sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_nomem(pCtx); +/* +** Advance the snippet iterator to the next candidate snippet. +*/ +static int fts3SnippetNextCandidate(SnippetIter *pIter){ + int i; /* Loop counter */ + + if( pIter->iCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; + + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); } }else{ - sqlite3_result_error_nomem(pCtx); + int iStart; + int iEnd = 0x7FFFFFFF; + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } + } + if( iEnd==0x7FFFFFFF ){ + return 1; + } + + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + } } - fts3SnippetFree(p); + + return 0; } +/* +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ +){ + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + int iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) ){ + int j; + u64 mPhrase = (u64)1 << i; + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } + } + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; +} + +/* +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. +*/ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + + pPhrase->nToken = pExpr->pPhrase->nToken; + + pCsr = sqlite3Fts3FindPositions(pExpr, p->pCsr->iPrevId, p->iCol); + if( pCsr ){ + int iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + }else{ + assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 ); + } + + return SQLITE_OK; +} + +/* +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. +*/ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ +){ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + int nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ + + memset(&sIter, 0, sizeof(sIter)); + + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; + } + memset(sIter.aPhrase, 0, nByte); + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + (void)fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlight; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover, &mHighlight); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlight; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + sqlite3_free(sIter.aPhrase); + *piScore = iBestScore; + return SQLITE_OK; +} + + +/* +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). +*/ +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } + + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + int nAlloc = pStr->nAlloc+nAppend+100; + char *zNew = sqlite3_realloc(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; + } + pStr->z = zNew; + pStr->nAlloc = nAlloc; + } + + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; + + return SQLITE_OK; +} + +/* +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: +** +** ........X.....X +** +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: +** +** ....X.....X.... +** +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. +*/ +int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ + + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ + + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + nDesired = (nLeft-nRight)/2; + + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + pC->pTokenizer = pTab->pTokenizer; + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; + } + } + } + return SQLITE_OK; +} + +/* +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. +*/ +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1; /* Dummy argument used with tokenizer */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + return SQLITE_OK; + } + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + pC->pTokenizer = pTab->pTokenizer; + + while( rc==SQLITE_OK ){ + int iBegin; /* Offset in zDoc of start of token */ + int iFin; /* Offset in zDoc of end of token */ + int isHighlight; /* True for highlighted terms */ + + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrent0 || iFragment>0) ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; + } + + pMod->xClose(pC); + return rc; +} + + +/* +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). +** +** The number of elements in the column-list is returned. +*/ +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; + + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; + } + + *ppCollist = pEnd; + return nEntry; +} + +static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ + char *pCsr = *pp; + while( *pCsr ){ + int nHit; + sqlite3_int64 iCol = 0; + if( *pCsr==0x01 ){ + pCsr++; + pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); + } + nHit = fts3ColumnlistCount(&pCsr); + assert( nHit>0 ); + if( isGlobal ){ + aOut[iCol*3+1]++; + } + aOut[iCol*3] += nHit; + } + pCsr++; + *pp = pCsr; +} + +/* +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. The "global" stats are those elements of the matchinfo +** array that are constant for all rows returned by the current query. +*/ +static int fts3ExprGlobalMatchinfoCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + MatchInfo *p = (MatchInfo *)pCtx; + char *pCsr; + char *pEnd; + const int iStart = 2 + (iPhrase * p->nCol * 3) + 1; + + assert( pExpr->isLoaded ); + + /* Fill in the global hit count matrix row for this phrase. */ + pCsr = pExpr->aDoclist; + pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + while( pCsraMatchinfo[iStart], 1); + } + + return SQLITE_OK; +} + +/* +** fts3ExprIterate() callback used to collect the "local" matchinfo stats +** for a single query. The "local" stats are those elements of the matchinfo +** array that are different for each row returned by the query. +*/ +static int fts3ExprLocalMatchinfoCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + MatchInfo *p = (MatchInfo *)pCtx; + + if( pExpr->aDoclist ){ + char *pCsr; + int iStart = 2 + (iPhrase * p->nCol * 3); + int i; + + for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; + + pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1); + if( pCsr ){ + fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + } + } + + return SQLITE_OK; +} + +/* +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +*/ +static int fts3GetMatchinfo(Fts3Cursor *pCsr){ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; + + if( pCsr->aMatchinfo==0 ){ + /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + int nPhrase; /* Number of phrases */ + int nMatchinfo; /* Number of u32 elements in match-info */ + + /* Load doclists for each phrase in the query. */ + rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + nMatchinfo = 2 + 3*sInfo.nCol*nPhrase; + if( pTab->bHasDocsize ){ + nMatchinfo += 1 + 2*pTab->nColumn; + } + + sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo); + if( !sInfo.aMatchinfo ){ + return SQLITE_NOMEM; + } + memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo); + + + /* First element of match-info is the number of phrases in the query */ + sInfo.aMatchinfo[0] = nPhrase; + sInfo.aMatchinfo[1] = sInfo.nCol; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo); + if( pTab->bHasDocsize ){ + int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; + rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]); + } + pCsr->aMatchinfo = sInfo.aMatchinfo; + pCsr->isMatchinfoNeeded = 1; + } + + sInfo.aMatchinfo = pCsr->aMatchinfo; + if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){ + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo); + if( pTab->bHasDocsize ){ + int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; + rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]); + } + pCsr->isMatchinfoNeeded = 0; + } + + return SQLITE_OK; +} + +/* +** Implementation of snippet() function. +*/ void sqlite3Fts3Snippet( sqlite3_context *pCtx, /* SQLite function call context */ Fts3Cursor *pCsr, /* Cursor object */ const char *zStart, /* Snippet start text - "" */ const char *zEnd, /* Snippet end text - "" */ - const char *zEllipsis /* Snippet ellipsis text - "..." */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ ){ - Snippet *p; /* Snippet structure */ - int rc = snippetAllOffsets(pCsr, &p); - if( rc==SQLITE_OK ){ - snippetText(pCsr, p, zStart, zEnd, zEllipsis); - if( p->zSnippet ){ - sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_nomem(pCtx); - } - }else{ - sqlite3_result_error_nomem(pCtx); + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; + + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. + */ + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + for(nSnippet=1; 1; nSnippet++){ + + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ + + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } + + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF; + int iS; + if( iCol>=0 && iRead!=iCol ) continue; + + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } + + mCovered |= pFragment->covered; + } + + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + } + + assert( nFToken>0 ); + + for(i=0; iiDocid, p->iCol); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert( iPos>=0 ); + } + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } + + return SQLITE_OK; +} + +/* +** Implementation of offsets() function. +*/ +void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY; /* Dummy argument used with xNext() */ + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); + + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; + + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + int iStart; + int iEnd; + int iCurrent; + const char *zDoc; + int nDoc; + + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx); + + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } + + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ) goto offsets_out; + pC->pTokenizer = pTab->pTokenizer; + + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ + + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } + + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + break; + }else{ + assert( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE ){ + rc = SQLITE_CORRUPT; + } + } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; + } + + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + } + return; +} + +/* +** Implementation of matchinfo() function. +*/ +void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc; + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + } + rc = fts3GetMatchinfo(pCsr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pContext, rc); + }else{ + Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; + int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3); + if( pTab->bHasDocsize ){ + n += sizeof(u32)*(1 + 2*pTab->nColumn); + } + sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); } - fts3SnippetFree(p); } #endif diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index a316fcf5..ab90be8f 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -119,14 +119,14 @@ const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ /* Find the start of the next token. */ z1 = zStr; while( z2==0 ){ - switch( *z1 ){ + char c = *z1; + switch( c ){ case '\0': return 0; /* No more tokens here */ case '\'': case '"': case '`': { - z2 = &z1[1]; - while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++; - if( *z2 ) z2++; + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); break; } case '[': @@ -466,10 +466,10 @@ int sqlite3Fts3InitHashTable( int rc = SQLITE_OK; void *p = (void *)pHash; const int any = SQLITE_ANY; - char *zTest = 0; - char *zTest2 = 0; #ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; void *pdb = (void *)db; zTest = sqlite3_mprintf("%s_test", zName); zTest2 = sqlite3_mprintf("%s_internal_test", zName); @@ -486,10 +486,13 @@ int sqlite3Fts3InitHashTable( || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) #endif - ); + ); +#ifdef SQLITE_TEST sqlite3_free(zTest); sqlite3_free(zTest2); +#endif + return rc; } diff --git a/ext/fts3/fts3_tokenizer.h b/ext/fts3/fts3_tokenizer.h index 906303db..61564450 100644 --- a/ext/fts3/fts3_tokenizer.h +++ b/ext/fts3/fts3_tokenizer.h @@ -145,4 +145,8 @@ struct sqlite3_tokenizer_cursor { /* Tokenizer implementations will typically add additional fields */ }; +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + #endif /* _FTS3_TOKENIZER_H_ */ diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c index 654e55dd..36c5a2f3 100644 --- a/ext/fts3/fts3_tokenizer1.c +++ b/ext/fts3/fts3_tokenizer1.c @@ -182,9 +182,11 @@ static int simpleNext( if( c->iOffset>iStartOffset ){ int i, n = c->iOffset-iStartOffset; if( n>c->nTokenAllocated ){ + char *pNew; c->nTokenAllocated = n+20; - c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated); - if( c->pToken==NULL ) return SQLITE_NOMEM; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; } for(i=0; ippNextElem!=0) + /* ** An instance of this structure is used to create a segment b-tree in the ** database. The internal details of this type are only accessed by the @@ -130,19 +139,26 @@ struct SegmentNode { #define SQL_DELETE_ALL_CONTENT 2 #define SQL_DELETE_ALL_SEGMENTS 3 #define SQL_DELETE_ALL_SEGDIR 4 -#define SQL_SELECT_CONTENT_BY_ROWID 5 -#define SQL_NEXT_SEGMENT_INDEX 6 -#define SQL_INSERT_SEGMENTS 7 -#define SQL_NEXT_SEGMENTS_ID 8 -#define SQL_INSERT_SEGDIR 9 -#define SQL_SELECT_LEVEL 10 -#define SQL_SELECT_ALL_LEVEL 11 -#define SQL_SELECT_LEVEL_COUNT 12 -#define SQL_SELECT_SEGDIR_COUNT_MAX 13 -#define SQL_DELETE_SEGDIR_BY_LEVEL 14 -#define SQL_DELETE_SEGMENTS_RANGE 15 -#define SQL_CONTENT_INSERT 16 -#define SQL_GET_BLOCK 17 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_COUNT_MAX 15 +#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_GET_BLOCK 19 +#define SQL_DELETE_DOCSIZE 20 +#define SQL_REPLACE_DOCSIZE 21 +#define SQL_SELECT_DOCSIZE 22 +#define SQL_SELECT_DOCTOTAL 23 +#define SQL_REPLACE_DOCTOTAL 24 /* ** This function is used to obtain an SQLite prepared statement handle @@ -167,25 +183,32 @@ static int fts3SqlStmt( /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", -/* 5 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", -/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?", -/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", -/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'", -/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ -/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root " +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", -/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root " +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", -/* 12 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 13 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", -/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", -/* 16 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", -/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", +/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", +/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", +/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -242,14 +265,21 @@ static int fts3SqlStmt( ** Returns SQLITE_OK if the statement is successfully executed, or an ** SQLite error code otherwise. */ -static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){ +static void fts3SqlExec( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS3 table */ + int eStmt, /* Index of statement to evaluate */ + sqlite3_value **apVal /* Parameters to bind */ +){ sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); + int rc; + if( *pRC ) return; + rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } - return rc; + *pRC = rc; } @@ -280,13 +310,13 @@ int sqlite3Fts3ReadBlock( sqlite3_bind_int64(pStmt, 1, iBlock); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ){ - return SQLITE_CORRUPT; + return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); } *pnBlock = sqlite3_column_bytes(pStmt, 0); *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); - if( !*pzBlock ){ - return SQLITE_NOMEM; + if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ + return SQLITE_CORRUPT; } } return SQLITE_OK; @@ -429,11 +459,17 @@ static int fts3PendingListAppend( ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ -static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){ +static int fts3PendingTermsAdd( + Fts3Table *p, /* FTS table into which text will be inserted */ + const char *zText, /* Text of document to be inseted */ + int iCol, /* Column number into which text is inserted */ + u32 *pnWord /* OUT: Number of tokens inserted */ +){ int rc; int iStart; int iEnd; int iPos; + int nWord = 0; char const *zToken; int nToken; @@ -457,6 +493,8 @@ static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ PendingList *pList; + + if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. ** Tokens must have a non-zero length. @@ -486,6 +524,7 @@ static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){ } pModule->xClose(pCsr); + *pnWord = nWord; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } @@ -501,7 +540,7 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ + if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } @@ -526,12 +565,12 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ ** Argument apVal is the same as the similarly named argument passed to ** fts3InsertData(). Parameter iDocid is the docid of the new row. */ -static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal){ +static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); if( zText ){ - int rc = fts3PendingTermsAdd(p, zText, i-2); + int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); if( rc!=SQLITE_OK ){ return rc; } @@ -612,18 +651,18 @@ static int fts3InsertData( ** pending terms. */ static int fts3DeleteAll(Fts3Table *p){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ - rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0); - if( rc==SQLITE_OK ){ - rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0); - } - if( rc==SQLITE_OK ){ - rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); } return rc; } @@ -633,20 +672,27 @@ static int fts3DeleteAll(Fts3Table *p){ ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ -static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ + u32 *aSz /* Sizes of deleted document written here */ +){ int rc; sqlite3_stmt *pSelect; + if( *pRC ) return; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; for(i=1; i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, zText, -1); + rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]); if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); - return rc; + *pRC = rc; + return; } } } @@ -654,7 +700,7 @@ static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){ }else{ sqlite3_reset(pSelect); } - return rc; + *pRC = rc; } /* @@ -728,6 +774,21 @@ static int fts3SegReaderNext(Fts3SegReader *pReader){ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ int rc; + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + if( pElem==0 ){ + pReader->aNode = 0; + }else{ + PendingList *pList = (PendingList *)fts3HashData(pElem); + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + pReader->nNode = pReader->nDoclist = pList->nData + 1; + pReader->aNode = pReader->aDoclist = pList->aData; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } if( !pReader->pStmt ){ pReader->aNode = 0; return SQLITE_OK; @@ -837,7 +898,9 @@ void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ sqlite3_reset(pReader->pStmt); p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt; } - sqlite3_free(pReader->zTerm); + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } sqlite3_free(pReader); } } @@ -934,6 +997,103 @@ int sqlite3Fts3SegReaderNew( return rc; } +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1pendingTerms); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + fts3SegReaderNext(pReader); + } + } + + if( isPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + /* ** The second argument to this function is expected to be a statement of @@ -977,7 +1137,7 @@ static int fts3SegReaderNew( ** ** 1) EOF is greater than not EOF. ** -** 2) The current terms (if any) are compared with memcmp(). If one +** 2) The current terms (if any) are compared using memcmp(). If one ** term is a prefix of another, the longer term is considered the ** larger. ** @@ -1105,11 +1265,9 @@ static int fts3WriteSegment( int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iBlock); - rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - } + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); } return rc; } @@ -1135,11 +1293,9 @@ static int fts3WriteSegdir( sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); sqlite3_bind_int64(pStmt, 5, iEndBlock); - rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); } return rc; } @@ -1664,7 +1820,7 @@ static int fts3DeleteSegdir( rc = sqlite3_reset(pDelete); } }else{ - rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); } return rc; @@ -1718,17 +1874,33 @@ static void fts3ColumnFilter( ** segments to create a single, larger segment. */ static int fts3MergeCallback( - Fts3Table *p, - void *pContext, - char *zTerm, - int nTerm, - char *aDoclist, - int nDoclist + Fts3Table *p, /* FTS3 Virtual table handle */ + void *pContext, /* Pointer to SegmentWriter* to write with */ + char *zTerm, /* Term to write to the db */ + int nTerm, /* Number of bytes in zTerm */ + char *aDoclist, /* Doclist associated with zTerm */ + int nDoclist /* Number of bytes in doclist */ ){ SegmentWriter **ppW = (SegmentWriter **)pContext; return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist); } +/* +** sqlite3Fts3SegReaderIterate() callback used when flushing the contents +** of the pending-terms hash table to the database. +*/ +static int fts3FlushCallback( + Fts3Table *p, /* FTS3 Virtual table handle */ + void *pContext, /* Pointer to SegmentWriter* to write with */ + char *zTerm, /* Term to write to the db */ + int nTerm, /* Number of bytes in zTerm */ + char *aDoclist, /* Doclist associated with zTerm */ + int nDoclist /* Number of bytes in doclist */ +){ + SegmentWriter **ppW = (SegmentWriter **)pContext; + return fts3SegWriterAdd(p, ppW, 0, zTerm, nTerm, aDoclist, nDoclist); +} + /* ** This function is used to iterate through a contiguous set of terms ** stored in the full-text index. It merges data contained in one or @@ -1790,8 +1962,7 @@ int sqlite3Fts3SegReaderIterate( Fts3SegReader *pSeg = apSegment[i]; while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){ rc = fts3SegReaderNext(pSeg); - if( rc!=SQLITE_OK ) goto finished; - } + if( rc!=SQLITE_OK ) goto finished; } } } @@ -1825,7 +1996,8 @@ int sqlite3Fts3SegReaderIterate( nMerge++; } - if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){ + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 && !isIgnoreEmpty ){ Fts3SegReader *p0 = apSegment[0]; rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist); if( rc!=SQLITE_OK ) goto finished; @@ -1849,9 +2021,9 @@ int sqlite3Fts3SegReaderIterate( sqlite3_int64 iDocid = apSegment[0]->iDocid; fts3SegReaderNextDocid(apSegment[0], &pList, &nList); j = 1; - while( jpOffsetList - && apSegment[j]->iDocid==iDocid + while( jpOffsetList + && apSegment[j]->iDocid==iDocid ){ fts3SegReaderNextDocid(apSegment[j], 0, 0); j++; @@ -1927,10 +2099,11 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int rc; /* Return code */ int iIdx; /* Index of new segment */ int iNewLevel; /* Level to create new segment at */ - sqlite3_stmt *pStmt; + sqlite3_stmt *pStmt = 0; SegmentWriter *pWriter = 0; int nSegment = 0; /* Number of segments being merged */ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ + Fts3SegReader *pPending = 0; /* Iterator for pending-terms */ Fts3SegFilter filter; /* Segment term filter condition */ if( iLevel<0 ){ @@ -1940,8 +2113,12 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ ** of the new segment is always 0. */ iIdx = 0; + rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending); + if( rc!=SQLITE_OK ) goto finished; rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel); - if( nSegment==1 ){ + if( rc!=SQLITE_OK ) goto finished; + nSegment += (pPending!=0); + if( nSegment<=1 ){ return SQLITE_DONE; } }else{ @@ -1952,17 +2129,18 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ */ iNewLevel = iLevel+1; rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto finished; rc = fts3SegmentCount(p, iLevel, &nSegment); + if( rc!=SQLITE_OK ) goto finished; } - if( rc!=SQLITE_OK ) return rc; assert( nSegment>0 ); assert( iNewLevel>=0 ); /* Allocate space for an array of pointers to segment iterators. */ apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment); if( !apSegment ){ - return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + goto finished; } memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment); @@ -1981,6 +2159,10 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ } } rc = sqlite3_reset(pStmt); + if( pPending ){ + apSegment[i] = pPending; + pPending = 0; + } pStmt = 0; if( rc!=SQLITE_OK ) goto finished; @@ -2005,90 +2187,306 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ } sqlite3_free(apSegment); } + sqlite3Fts3SegReaderFree(p, pPending); sqlite3_reset(pStmt); return rc; } -/* -** This is a comparison function used as a qsort() callback when sorting -** an array of pending terms by term. This occurs as part of flushing -** the contents of the pending-terms hash table to the database. -*/ -static int qsortCompare(const void *lhs, const void *rhs){ - char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); - char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); - int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); - int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); - - int n = (n1pendingTerms); - if( nElem==0 ){ - assert( p->nPendingData==0 ); - return SQLITE_OK; - } - - /* Determine the next index at level 0, merging as necessary. */ - rc = fts3AllocateSegdirIdx(p, 0, &idx); - if( rc!=SQLITE_OK ){ + rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pReader); + if( rc!=SQLITE_OK || pReader==0 ){ return rc; - } - - apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *)); - if( !apElem ){ - return SQLITE_NOMEM; } - i = 0; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - apElem[i++] = pElem; - } - assert( i==nElem ); - - /* TODO(shess) Should we allow user-defined collation sequences, - ** here? I think we only need that once we support prefix searches. - ** Also, should we be using qsort()? + /* Determine the next index at level 0. If level 0 is already full, this + ** call may merge all existing level 0 segments into a single level 1 + ** segment. */ - if( nElem>1 ){ - qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare); - } + rc = fts3AllocateSegdirIdx(p, 0, &idx); + /* If no errors have occured, iterate through the contents of the + ** pending-terms hash table using the Fts3SegReader iterator. The callback + ** writes each term (along with its doclist) to the database via the + ** SegmentWriter handle pWriter. + */ + if( rc==SQLITE_OK ){ + void *c = (void *)&pWriter; /* SegReaderIterate() callback context */ + Fts3SegFilter f; /* SegReaderIterate() parameters */ - /* Write the segment tree into the database. */ - for(i=0; rc==SQLITE_OK && iaData, pList->nData+1); + memset(&f, 0, sizeof(Fts3SegFilter)); + f.flags = FTS3_SEGMENT_REQUIRE_POS; + rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3FlushCallback, c); } + assert( pWriter || rc!=SQLITE_OK ); + + /* If no errors have occured, flush the SegmentWriter object to the + ** database. Then delete the SegmentWriter and Fts3SegReader objects + ** allocated by this function. + */ if( rc==SQLITE_OK ){ rc = fts3SegWriterFlush(p, pWriter, 0, idx); } - - /* Free all allocated resources before returning */ fts3SegWriterFree(pWriter); - sqlite3_free(apElem); - sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3SegReaderFree(p, pReader); + + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } + return rc; +} + +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; ibase.pVtab; + rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0); + if( rc ){ + return rc; + } + sqlite3_bind_int64(pStmt, 1, pCur->iPrevId); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nBlob = sqlite3_column_bytes(pStmt, 0); + pBlob = (const char*)sqlite3_column_blob(pStmt, 0); + for(i=j=0; inColumn && jbase.pVtab; + rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + if( rc ){ + return rc; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nBlob = sqlite3_column_bytes(pStmt, 0); + pBlob = (const char*)sqlite3_column_blob(pStmt, 0); + j = sqlite3Fts3GetVarint(pBlob, &x); + a[0] = nDoc = (u32)(x & 0xffffffff); + for(i=0; inColumn && jiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc( 10*p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); +} + +/* +** Update the 0 record of the %_stat table so that it holds a blob +** which contains the document count followed by the cumulative +** document sizes for all columns. +*/ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[p->nColumn+1]; + rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(p->nColumn+1, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); + }else{ + memset(a, 0, sizeof(u32)*(p->nColumn+1) ); + } + sqlite3_reset(pStmt); + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_free(a); +} + +/* +** Handle a 'special' INSERT of the form: +** +** "INSERT INTO tbl(tbl) VALUES()" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); + + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3SegmentMerge(p, -1); + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + }else{ + sqlite3Fts3PendingTermsClear(p); + } +#ifdef SQLITE_TEST + }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + p->nNodeSize = atoi(&zVal[9]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + p->nMaxPendingData = atoi(&zVal[11]); + rc = SQLITE_OK; +#endif + }else{ + rc = SQLITE_ERROR; + } + return rc; } @@ -2106,6 +2504,16 @@ int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ + u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzDel; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + + + /* Allocate space to hold the change in document sizes */ + aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 ); + if( aSzIns==0 ) return SQLITE_NOMEM; + aSzDel = &aSzIns[p->nColumn]; + memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2); /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ @@ -2122,14 +2530,17 @@ int sqlite3Fts3UpdateMethod( isRemove = 1; iRemove = sqlite3_value_int64(apVal[0]); rc = fts3PendingTermsDocid(p, iRemove); - if( rc==SQLITE_OK ){ - rc = fts3DeleteTerms(p, apVal); - if( rc==SQLITE_OK ){ - rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal); - } + fts3DeleteTerms(&rc, p, apVal, aSzDel); + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); + nChng--; } } } + }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ + sqlite3_free(aSzIns); + return fts3SpecialInsert(p, apVal[p->nColumn+2]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ @@ -2139,10 +2550,19 @@ int sqlite3Fts3UpdateMethod( rc = fts3PendingTermsDocid(p, *pRowid); } if( rc==SQLITE_OK ){ - rc = fts3InsertTerms(p, apVal); + rc = fts3InsertTerms(p, apVal, aSzIns); + } + if( p->bHasDocsize ){ + nChng++; + fts3InsertDocsize(&rc, p, aSzIns); } } + if( p->bHasDocsize ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); + } + + sqlite3_free(aSzIns); return rc; } @@ -2155,14 +2575,15 @@ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, -1); - } + rc = fts3SegmentMerge(p, -1); if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } }else{ - sqlite3_exec(p->db, "ROLLBACK TO fts3 ; RELEASE fts3", 0, 0, 0); + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } return rc; diff --git a/ext/icu/icu.c b/ext/icu/icu.c index c673341a..e99f1424 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -454,7 +454,7 @@ int sqlite3IcuInit(sqlite3 *db){ void *pContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"regexp",-1, SQLITE_ANY, 0, icuRegexpFunc}, + {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 49c00a35..3111cc2e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -423,6 +423,7 @@ nodeAcquire( rc = sqlite3_step(pRtree->pReadNode); if( rc==SQLITE_ROW ){ const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); + assert( sqlite3_column_bytes(pRtree->pReadNode, 0)==pRtree->iNodeSize ); memcpy(pNode->zData, zBlob, pRtree->iNodeSize); nodeReference(pParent); }else{ @@ -2619,31 +2620,69 @@ static int rtreeSqlInit( } /* -** This routine queries database handle db for the page-size used by -** database zDb. If successful, the page-size in bytes is written to -** *piPageSize and SQLITE_OK returned. Otherwise, and an SQLite error -** code is returned. +** The second argument to this function contains the text of an SQL statement +** that returns a single integer value. The statement is compiled and executed +** using database connection db. If successful, the integer value returned +** is written to *piVal and SQLITE_OK returned. Otherwise, an SQLite error +** code is returned and the value of *piVal after returning is not defined. */ -static int getPageSize(sqlite3 *db, const char *zDb, int *piPageSize){ +static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ int rc = SQLITE_NOMEM; + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *piVal = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + } + } + return rc; +} + +/* +** This function is called from within the xConnect() or xCreate() method to +** determine the node-size used by the rtree table being created or connected +** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned. +** +** If this function is being called as part of an xConnect(), then the rtree +** table already exists. In this case the node-size is determined by inspecting +** the root node of the tree. +** +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries +** would fit in a single node, use a smaller node-size. +*/ +static int getNodeSize( + sqlite3 *db, /* Database handle */ + Rtree *pRtree, /* Rtree handle */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc; char *zSql; - sqlite3_stmt *pStmt = 0; - - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", zDb); - if( !zSql ){ - return SQLITE_NOMEM; + if( isCreate ){ + int iPageSize; + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); + rc = getIntFromStmt(db, zSql, &iPageSize); + if( rc==SQLITE_OK ){ + pRtree->iNodeSize = iPageSize-64; + if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ + pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; + } + } + }else{ + zSql = sqlite3_mprintf( + "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", + pRtree->zDb, pRtree->zName + ); + rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); } - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); - if( rc!=SQLITE_OK ){ - return rc; - } - - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *piPageSize = sqlite3_column_int(pStmt, 0); - } - return sqlite3_finalize(pStmt); + return rc; } /* @@ -2664,7 +2703,6 @@ static int rtreeInit( int isCreate /* True for xCreate, false for xConnect */ ){ int rc = SQLITE_OK; - int iPageSize = 0; Rtree *pRtree; int nDb; /* Length of string argv[1] */ int nName; /* Length of string argv[2] */ @@ -2683,11 +2721,6 @@ static int rtreeInit( return SQLITE_ERROR; } - rc = getPageSize(db, argv[1], &iPageSize); - if( rc!=SQLITE_OK ){ - return rc; - } - /* Allocate the sqlite3_vtab structure */ nDb = strlen(argv[1]); nName = strlen(argv[2]); @@ -2706,44 +2739,37 @@ static int rtreeInit( memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); - /* Figure out the node size to use. By default, use 64 bytes less than - ** the database page-size. This ensures that each node is stored on - ** a single database page. - ** - ** If the databasd page-size is so large that more than RTREE_MAXCELLS - ** entries would fit in a single node, use a smaller node-size. - */ - pRtree->iNodeSize = iPageSize-64; - if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ - pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; - } + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate); /* Create/Connect to the underlying relational database schema. If ** that is successful, call sqlite3_declare_vtab() to configure ** the r-tree table schema. */ - if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - }else{ - char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); - char *zTmp; - int ii; - for(ii=4; zSql && iipBt); sqlite3PagerLockingMode(pPager, db->dfltLockMode); sqlite3PagerJournalMode(pPager, db->dfltJournalMode); + sqlite3BtreeSecureDelete(aNew->pBt, + sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); } - aNew->zName = sqlite3DbStrDup(db, zName); aNew->safety_level = 3; + aNew->zName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zName==0 ){ + rc = SQLITE_NOMEM; + } -#if SQLITE_HAS_CODEC + +#ifdef SQLITE_HAS_CODEC if( rc==SQLITE_OK ){ extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); @@ -184,11 +190,9 @@ static void attachFunc( ** we found it. */ if( rc==SQLITE_OK ){ - (void)sqlite3SafetyOn(db); sqlite3BtreeEnterAll(db); rc = sqlite3Init(db, &zErrDyn); sqlite3BtreeLeaveAll(db); - (void)sqlite3SafetyOff(db); } if( rc ){ int iDb = db->nDb - 1; diff --git a/src/backup.c b/src/backup.c index 422198bf..2d01ddbb 100644 --- a/src/backup.c +++ b/src/backup.c @@ -98,10 +98,10 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ }else{ pParse->db = pDb; if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3ErrorClear(pParse); sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); rc = SQLITE_ERROR; } + sqlite3DbFree(pErrorDb, pParse->zErrMsg); sqlite3StackFree(pErrorDb, pParse); } if( rc ){ diff --git a/src/btree.c b/src/btree.c index 385c511a..dbcdf65e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1247,11 +1247,11 @@ static int freeSpace(MemPage *pPage, int start, int size){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( size>=0 ); /* Minimum cell size is 4 */ -#ifdef SQLITE_SECURE_DELETE - /* Overwrite deleted information with zeros when the SECURE_DELETE - ** option is enabled at compile-time */ - memset(&data[start], 0, size); -#endif + if( pPage->pBt->secureDelete ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[start], 0, size); + } /* Add the space back into the linked list of freeblocks. Note that ** even though the freeblock list was checked by btreeInitPage(), @@ -1483,9 +1483,9 @@ static void zeroPage(MemPage *pPage, int flags){ assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pBt->mutex) ); -#ifdef SQLITE_SECURE_DELETE - memset(&data[hdr], 0, pBt->usableSize - hdr); -#endif + if( pBt->secureDelete ){ + memset(&data[hdr], 0, pBt->usableSize - hdr); + } data[hdr] = (char)flags; first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); memset(&data[hdr+1], 0, 4); @@ -1805,6 +1805,9 @@ int sqlite3BtreeOpen( pBt->pCursor = 0; pBt->pPage1 = 0; pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); +#ifdef SQLITE_SECURE_DELETE + pBt->secureDelete = 1; +#endif pBt->pageSize = get2byte(&zDbHeader[16]); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ @@ -2161,6 +2164,23 @@ int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ sqlite3BtreeLeave(p); return n; } + +/* +** Set the secureDelete flag if newFlag is 0 or 1. If newFlag is -1, +** then make no changes. Always return the value of the secureDelete +** setting after the change. +*/ +int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ + int b; + if( p==0 ) return 0; + sqlite3BtreeEnter(p); + if( newFlag>=0 ){ + p->pBt->secureDelete = (newFlag!=0) ? 1 : 0; + } + b = p->pBt->secureDelete; + sqlite3BtreeLeave(p); + return b; +} #endif /* !defined(SQLITE_OMIT_PAGER_PRAGMAS) || !defined(SQLITE_OMIT_VACUUM) */ /* @@ -4904,17 +4924,17 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ nFree = get4byte(&pPage1->aData[36]); put4byte(&pPage1->aData[36], nFree+1); -#ifdef SQLITE_SECURE_DELETE - /* If the SQLITE_SECURE_DELETE compile-time option is enabled, then - ** always fully overwrite deleted information with zeros. - */ - if( (!pPage && (rc = btreeGetPage(pBt, iPage, &pPage, 0))) - || (rc = sqlite3PagerWrite(pPage->pDbPage)) - ){ - goto freepage_out; + if( pBt->secureDelete ){ + /* If the secure_delete option is enabled, then + ** always fully overwrite deleted information with zeros. + */ + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) + ){ + goto freepage_out; + } + memset(pPage->aData, 0, pPage->pBt->pageSize); } - memset(pPage->aData, 0, pPage->pBt->pageSize); -#endif /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. @@ -4965,11 +4985,9 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ if( rc==SQLITE_OK ){ put4byte(&pTrunk->aData[4], nLeaf+1); put4byte(&pTrunk->aData[8+nLeaf*4], iPage); -#ifndef SQLITE_SECURE_DELETE - if( pPage ){ + if( pPage && !pBt->secureDelete ){ sqlite3PagerDontWrite(pPage->pDbPage); } -#endif rc = btreeSetHasContent(pBt, iPage); } TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); @@ -5043,7 +5061,25 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); if( rc ) return rc; } - rc = freePage2(pBt, pOvfl, ovflPgno); + + if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) + && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 + ){ + /* There is no reason any cursor should have an outstanding reference + ** to an overflow page belonging to a cell that is being deleted/updated. + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as + ** freePage2() may zero the page contents if secure-delete mode is + ** enabled. If this 'overflow' page happens to be a page that the + ** caller is iterating through or using in some other way, this + ** can be problematic. + */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = freePage2(pBt, pOvfl, ovflPgno); + } + if( pOvfl ){ sqlite3PagerUnref(pOvfl->pDbPage); } @@ -5287,7 +5323,7 @@ static void insertCell( Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ int *pRC /* Read and write return code from here */ ){ - int idx; /* Where to write new cell content in data[] */ + int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ int end; /* First byte past the last cell pointer in data[] */ int ins; /* Index in data[] where new cell pointer is inserted */ @@ -5778,10 +5814,17 @@ static int balance_nonroot( ** In this case, temporarily copy the cell into the aOvflSpace[] ** buffer. It will be copied out again as soon as the aSpace[] buffer ** is allocated. */ -#ifdef SQLITE_SECURE_DELETE - memcpy(&aOvflSpace[apDiv[i]-pParent->aData], apDiv[i], szNew[i]); - apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; -#endif + if( pBt->secureDelete ){ + int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); + if( (iOff+szNew[i])>pBt->usableSize ){ + rc = SQLITE_CORRUPT_BKPT; + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + }else{ + memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); + apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; + } + } dropCell(pParent, i+nxDiv-pParent->nOverflow, szNew[i], &rc); } } @@ -5901,7 +5944,7 @@ static int balance_nonroot( if( leafData ){ i--; } subtotal = 0; k++; - if( k>NB+1 ){ rc = SQLITE_CORRUPT; goto balance_cleanup; } + if( k>NB+1 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } } } szNew[k] = subtotal; @@ -5955,7 +5998,7 @@ static int balance_nonroot( ** Allocate k new pages. Reuse old pages where possible. */ if( apOld[0]->pgno<=1 ){ - rc = SQLITE_CORRUPT; + rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } pageFlags = apOld[0]->aData[0]; @@ -7393,7 +7436,9 @@ static void checkList( static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - char *zParentContext /* Parent context */ + char *zParentContext, /* Parent context */ + i64 *pnParentMinKey, + i64 *pnParentMaxKey ){ MemPage *pPage; int i, rc, depth, d2, pgno, cnt; @@ -7404,6 +7449,8 @@ static int checkTreePage( int usableSize; char zContext[100]; char *hit = 0; + i64 nMinKey = 0; + i64 nMaxKey = 0; sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); @@ -7446,6 +7493,16 @@ static int checkTreePage( btreeParseCellPtr(pPage, pCell, &info); sz = info.nData; if( !pPage->intKey ) sz += (int)info.nKey; + /* For intKey pages, check that the keys are in order. + */ + else if( i==0 ) nMinKey = nMaxKey = info.nKey; + else{ + if( info.nKey <= nMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + } + nMaxKey = info.nKey; + } assert( sz==info.nPayload ); if( (sz>info.nLocal) && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) @@ -7469,25 +7526,62 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - d2 = checkTreePage(pCheck, pgno, zContext); + d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); if( i>0 && d2!=depth ){ checkAppendMsg(pCheck, zContext, "Child page depth differs"); } depth = d2; } } + if( !pPage->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); sqlite3_snprintf(sizeof(zContext), zContext, "On page %d at right child: ", iPage); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, 0); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - checkTreePage(pCheck, pgno, zContext); + checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); } + /* For intKey leaf pages, check that the min/max keys are in order + ** with any left/parent/right pages. + */ + if( pPage->leaf && pPage->intKey ){ + /* if we are a left child page */ + if( pnParentMinKey ){ + /* if we are the left most child page */ + if( !pnParentMaxKey ){ + if( nMaxKey > *pnParentMinKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (max larger than parent min of %lld)", + nMaxKey, *pnParentMinKey); + } + }else{ + if( nMinKey <= *pnParentMinKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (min less than parent min of %lld)", + nMinKey, *pnParentMinKey); + } + if( nMaxKey > *pnParentMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (max larger than parent max of %lld)", + nMaxKey, *pnParentMaxKey); + } + *pnParentMinKey = nMaxKey; + } + /* else if we're a right child page */ + } else if( pnParentMaxKey ){ + if( nMinKey <= *pnParentMaxKey ){ + checkAppendMsg(pCheck, zContext, + "Rowid %lld out of order (min less than parent max of %lld)", + nMinKey, *pnParentMaxKey); + } + } + } + /* Check for complete coverage of the page */ data = pPage->aData; @@ -7511,7 +7605,7 @@ static int checkTreePage( } if( (pc+size-1)>=usableSize ){ checkAppendMsg(pCheck, 0, - "Corruption detected in cell %d on page %d",i,iPage,0); + "Corruption detected in cell %d on page %d",i,iPage); }else{ for(j=pc+size-1; j>=pc; j--) hit[j]++; } @@ -7617,7 +7711,7 @@ char *sqlite3BtreeIntegrityCheck( checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); } #endif - checkTreePage(&sCheck, aRoot[i], "List of tree roots: "); + checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); } /* Make sure every page in the file is referenced diff --git a/src/btree.h b/src/btree.h index 7852d4ae..4f034f33 100644 --- a/src/btree.h +++ b/src/btree.h @@ -81,6 +81,7 @@ int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); int sqlite3BtreeGetPageSize(Btree*); int sqlite3BtreeMaxPageCount(Btree*,int); +int sqlite3BtreeSecureDelete(Btree*,int); int sqlite3BtreeGetReserve(Btree*); int sqlite3BtreeSetAutoVacuum(Btree *, int); int sqlite3BtreeGetAutoVacuum(Btree *); diff --git a/src/btreeInt.h b/src/btreeInt.h index c9fca97f..f6d95e50 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -407,6 +407,7 @@ struct BtShared { MemPage *pPage1; /* First page of the database */ u8 readOnly; /* True if the underlying file is readonly */ u8 pageSizeFixed; /* True if the page size can no longer be changed */ + u8 secureDelete; /* True if secure_delete is enabled */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ diff --git a/src/build.c b/src/build.c index d8097a76..8e343864 100644 --- a/src/build.c +++ b/src/build.c @@ -202,7 +202,7 @@ void sqlite3FinishCoding(Parse *pParse){ pParse->isMultiWrite && pParse->mayAbort); pParse->rc = SQLITE_DONE; pParse->colNamesSet = 0; - }else if( pParse->rc==SQLITE_OK ){ + }else{ pParse->rc = SQLITE_ERROR; } pParse->nTab = 0; @@ -549,7 +549,8 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ assert( db!=0 ); assert( iDb>=0 && iDbnDb ); - assert( zTabName && zTabName[0] ); + assert( zTabName ); + testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, sqlite3Strlen30(zTabName),0); @@ -1973,13 +1974,12 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + if( noErr ) db->suppressErr++; pTab = sqlite3LocateTable(pParse, isView, pName->a[0].zName, pName->a[0].zDatabase); + if( noErr ) db->suppressErr--; if( pTab==0 ){ - if( noErr ){ - sqlite3ErrorClear(pParse); - } goto exit_drop_table; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -3401,6 +3401,7 @@ int sqlite3OpenTempDatabase(Parse *pParse){ sqlite3 *db = pParse->db; if( db->aDb[1].pBt==0 && !pParse->explain ){ int rc; + Btree *pBt; static const int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | @@ -3408,18 +3409,20 @@ int sqlite3OpenTempDatabase(Parse *pParse){ SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TEMP_DB; - rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, flags, - &db->aDb[1].pBt); + rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, flags, &pBt); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " "file for storing temporary tables"); pParse->rc = rc; return 1; } - assert( (db->flags & SQLITE_InTrans)==0 || db->autoCommit ); + db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); - sqlite3PagerJournalMode(sqlite3BtreePager(db->aDb[1].pBt), - db->dfltJournalMode); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + db->mallocFailed = 1; + return 1; + } + sqlite3PagerJournalMode(sqlite3BtreePager(pBt), db->dfltJournalMode); } return 0; } diff --git a/src/complete.c b/src/complete.c index 0f7b1879..9e914008 100644 --- a/src/complete.c +++ b/src/complete.c @@ -40,11 +40,13 @@ extern const char sqlite3IsEbcdicIdChar[]; #define tkSEMI 0 #define tkWS 1 #define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER #define tkEXPLAIN 3 #define tkCREATE 4 #define tkTEMP 5 #define tkTRIGGER 6 #define tkEND 7 +#endif /* ** Return TRUE if the given SQL string ends in a semicolon. @@ -53,36 +55,38 @@ extern const char sqlite3IsEbcdicIdChar[]; ** Whenever the CREATE TRIGGER keywords are seen, the statement ** must end with ";END;". ** -** This implementation uses a state machine with 7 states: +** This implementation uses a state machine with 8 states: ** -** (0) START At the beginning or end of an SQL statement. This routine +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine ** returns 1 if it ends in the START state and 0 if it ends ** in any other state. ** -** (1) NORMAL We are in the middle of statement which ends with a single +** (2) NORMAL We are in the middle of statement which ends with a single ** semicolon. ** -** (2) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of ** a statement. ** -** (3) CREATE The keyword CREATE has been seen at the beginning of a +** (4) CREATE The keyword CREATE has been seen at the beginning of a ** statement, possibly preceeded by EXPLAIN and/or followed by ** TEMP or TEMPORARY ** -** (4) TRIGGER We are in the middle of a trigger definition that must be +** (5) TRIGGER We are in the middle of a trigger definition that must be ** ended by a semicolon, the keyword END, and another semicolon. ** -** (5) SEMI We've seen the first semicolon in the ";END;" that occurs at +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at ** the end of a trigger definition. ** -** (6) END We've seen the ";END" of the ";END;" that occurs at the end +** (7) END We've seen the ";END" of the ";END;" that occurs at the end ** of a trigger difinition. ** ** Transitions between states above are determined by tokens extracted ** from the input. The following tokens are significant: ** ** (0) tkSEMI A semicolon. -** (1) tkWS Whitespace +** (1) tkWS Whitespace. ** (2) tkOTHER Any other SQL token. ** (3) tkEXPLAIN The "explain" keyword. ** (4) tkCREATE The "create" keyword. @@ -91,6 +95,7 @@ extern const char sqlite3IsEbcdicIdChar[]; ** (7) tkEND The "end" keyword. ** ** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. ** ** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed ** to recognize the end of a trigger can be omitted. All we have to do @@ -104,26 +109,28 @@ int sqlite3_complete(const char *zSql){ /* A complex statement machine used to detect the end of a CREATE TRIGGER ** statement. This is the normal case. */ - static const u8 trans[7][8] = { + static const u8 trans[8][8] = { /* Token: */ - /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 START: */ { 0, 0, 1, 2, 3, 1, 1, 1, }, - /* 1 NORMAL: */ { 0, 1, 1, 1, 1, 1, 1, 1, }, - /* 2 EXPLAIN: */ { 0, 2, 2, 1, 3, 1, 1, 1, }, - /* 3 CREATE: */ { 0, 3, 1, 1, 1, 3, 4, 1, }, - /* 4 TRIGGER: */ { 5, 4, 4, 4, 4, 4, 4, 4, }, - /* 5 SEMI: */ { 5, 5, 4, 4, 4, 4, 4, 6, }, - /* 6 END: */ { 0, 6, 4, 4, 4, 4, 4, 4, }, + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, }; #else - /* If triggers are not suppored by this compile then the statement machine + /* If triggers are not supported by this compile then the statement machine ** used to detect the end of a statement is much simplier */ - static const u8 trans[2][3] = { + static const u8 trans[3][3] = { /* Token: */ /* State: ** SEMI WS OTHER */ - /* 0 START: */ { 0, 0, 1, }, - /* 1 NORMAL: */ { 0, 1, 1, }, + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, }; #endif /* SQLITE_OMIT_TRIGGER */ @@ -159,7 +166,7 @@ int sqlite3_complete(const char *zSql){ break; } while( *zSql && *zSql!='\n' ){ zSql++; } - if( *zSql==0 ) return state==0; + if( *zSql==0 ) return state==1; token = tkWS; break; } @@ -243,7 +250,7 @@ int sqlite3_complete(const char *zSql){ state = trans[state][token]; zSql++; } - return state==0; + return state==1; } #ifndef SQLITE_OMIT_UTF16 diff --git a/src/ctime.c b/src/ctime.c new file mode 100644 index 00000000..dacdc588 --- /dev/null +++ b/src/ctime.c @@ -0,0 +1,385 @@ +/* +** 2010 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements routines used to report what compile-time options +** SQLite was built with. +*/ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + +#include "sqliteInt.h" + +/* +** An array of names of all compile-time options. This array should +** be sorted A-Z. +** +** This array looks large, but in a typical installation actually uses +** only a handful of compile-time options, so most times this array is usually +** rather short and uses little memory space. +*/ +static const char * const azCompileOpt[] = { + +/* These macros are provided to "stringify" the value of the define +** for those options in which the value is meaningful. */ +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) + +#ifdef SQLITE_32BIT_ROWID + "32BIT_ROWID", +#endif +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC + "4_BYTE_ALIGNED_MALLOC", +#endif +#ifdef SQLITE_CASE_SENSITIVE_LIKE + "CASE_SENSITIVE_LIKE", +#endif +#ifdef SQLITE_CHECK_PAGES + "CHECK_PAGES", +#endif +#ifdef SQLITE_COVERAGE_TEST + "COVERAGE_TEST", +#endif +#ifdef SQLITE_DEBUG + "DEBUG", +#endif +#ifdef SQLITE_DEFAULT_LOCKING_MODE + "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), +#endif +#ifdef SQLITE_DISABLE_DIRSYNC + "DISABLE_DIRSYNC", +#endif +#ifdef SQLITE_DISABLE_LFS + "DISABLE_LFS", +#endif +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + "ENABLE_ATOMIC_WRITE", +#endif +#ifdef SQLITE_ENABLE_CEROD + "ENABLE_CEROD", +#endif +#ifdef SQLITE_ENABLE_COLUMN_METADATA + "ENABLE_COLUMN_METADATA", +#endif +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + "ENABLE_EXPENSIVE_ASSERT", +#endif +#ifdef SQLITE_ENABLE_FTS1 + "ENABLE_FTS1", +#endif +#ifdef SQLITE_ENABLE_FTS2 + "ENABLE_FTS2", +#endif +#ifdef SQLITE_ENABLE_FTS3 + "ENABLE_FTS3", +#endif +#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS + "ENABLE_FTS3_PARENTHESIS", +#endif +#ifdef SQLITE_ENABLE_FTS4 + "ENABLE_FTS4", +#endif +#ifdef SQLITE_ENABLE_ICU + "ENABLE_ICU", +#endif +#ifdef SQLITE_ENABLE_IOTRACE + "ENABLE_IOTRACE", +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + "ENABLE_LOAD_EXTENSION", +#endif +#ifdef SQLITE_ENABLE_LOCKING_STYLE + "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), +#endif +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + "ENABLE_MEMORY_MANAGEMENT", +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 + "ENABLE_MEMSYS3", +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + "ENABLE_MEMSYS5", +#endif +#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK + "ENABLE_OVERSIZE_CELL_CHECK", +#endif +#ifdef SQLITE_ENABLE_RTREE + "ENABLE_RTREE", +#endif +#ifdef SQLITE_ENABLE_STAT2 + "ENABLE_STAT2", +#endif +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + "ENABLE_UNLOCK_NOTIFY", +#endif +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + "ENABLE_UPDATE_DELETE_LIMIT", +#endif +#ifdef SQLITE_HAS_CODEC + "HAS_CODEC", +#endif +#ifdef SQLITE_HAVE_ISNAN + "HAVE_ISNAN", +#endif +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + "HOMEGROWN_RECURSIVE_MUTEX", +#endif +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + "IGNORE_AFP_LOCK_ERRORS", +#endif +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + "IGNORE_FLOCK_LOCK_ERRORS", +#endif +#ifdef SQLITE_INT64_TYPE + "INT64_TYPE", +#endif +#ifdef SQLITE_LOCK_TRACE + "LOCK_TRACE", +#endif +#ifdef SQLITE_MEMDEBUG + "MEMDEBUG", +#endif +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT + "MIXED_ENDIAN_64BIT_FLOAT", +#endif +#ifdef SQLITE_NO_SYNC + "NO_SYNC", +#endif +#ifdef SQLITE_OMIT_ALTERTABLE + "OMIT_ALTERTABLE", +#endif +#ifdef SQLITE_OMIT_ANALYZE + "OMIT_ANALYZE", +#endif +#ifdef SQLITE_OMIT_ATTACH + "OMIT_ATTACH", +#endif +#ifdef SQLITE_OMIT_AUTHORIZATION + "OMIT_AUTHORIZATION", +#endif +#ifdef SQLITE_OMIT_AUTOINCREMENT + "OMIT_AUTOINCREMENT", +#endif +#ifdef SQLITE_OMIT_AUTOINIT + "OMIT_AUTOINIT", +#endif +#ifdef SQLITE_OMIT_AUTOVACUUM + "OMIT_AUTOVACUUM", +#endif +#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION + "OMIT_BETWEEN_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_BLOB_LITERAL + "OMIT_BLOB_LITERAL", +#endif +#ifdef SQLITE_OMIT_BTREECOUNT + "OMIT_BTREECOUNT", +#endif +#ifdef SQLITE_OMIT_BUILTIN_TEST + "OMIT_BUILTIN_TEST", +#endif +#ifdef SQLITE_OMIT_CAST + "OMIT_CAST", +#endif +#ifdef SQLITE_OMIT_CHECK + "OMIT_CHECK", +#endif +#ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS + "OMIT_COMPILEOPTION_DIAGS", +#endif +#ifdef SQLITE_OMIT_COMPLETE + "OMIT_COMPLETE", +#endif +#ifdef SQLITE_OMIT_COMPOUND_SELECT + "OMIT_COMPOUND_SELECT", +#endif +#ifdef SQLITE_OMIT_DATETIME_FUNCS + "OMIT_DATETIME_FUNCS", +#endif +#ifdef SQLITE_OMIT_DECLTYPE + "OMIT_DECLTYPE", +#endif +#ifdef SQLITE_OMIT_DEPRECATED + "OMIT_DEPRECATED", +#endif +#ifdef SQLITE_OMIT_DISKIO + "OMIT_DISKIO", +#endif +#ifdef SQLITE_OMIT_EXPLAIN + "OMIT_EXPLAIN", +#endif +#ifdef SQLITE_OMIT_FLAG_PRAGMAS + "OMIT_FLAG_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_FLOATING_POINT + "OMIT_FLOATING_POINT", +#endif +#ifdef SQLITE_OMIT_FOREIGN_KEY + "OMIT_FOREIGN_KEY", +#endif +#ifdef SQLITE_OMIT_GET_TABLE + "OMIT_GET_TABLE", +#endif +#ifdef SQLITE_OMIT_GLOBALRECOVER + "OMIT_GLOBALRECOVER", +#endif +#ifdef SQLITE_OMIT_INCRBLOB + "OMIT_INCRBLOB", +#endif +#ifdef SQLITE_OMIT_INTEGRITY_CHECK + "OMIT_INTEGRITY_CHECK", +#endif +#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION + "OMIT_LIKE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + "OMIT_LOAD_EXTENSION", +#endif +#ifdef SQLITE_OMIT_LOCALTIME + "OMIT_LOCALTIME", +#endif +#ifdef SQLITE_OMIT_LOOKASIDE + "OMIT_LOOKASIDE", +#endif +#ifdef SQLITE_OMIT_MEMORYDB + "OMIT_MEMORYDB", +#endif +#ifdef SQLITE_OMIT_OR_OPTIMIZATION + "OMIT_OR_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_PAGER_PRAGMAS + "OMIT_PAGER_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_PRAGMA + "OMIT_PRAGMA", +#endif +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK + "OMIT_PROGRESS_CALLBACK", +#endif +#ifdef SQLITE_OMIT_QUICKBALANCE + "OMIT_QUICKBALANCE", +#endif +#ifdef SQLITE_OMIT_REINDEX + "OMIT_REINDEX", +#endif +#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS + "OMIT_SCHEMA_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + "OMIT_SCHEMA_VERSION_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SHARED_CACHE + "OMIT_SHARED_CACHE", +#endif +#ifdef SQLITE_OMIT_SUBQUERY + "OMIT_SUBQUERY", +#endif +#ifdef SQLITE_OMIT_TCL_VARIABLE + "OMIT_TCL_VARIABLE", +#endif +#ifdef SQLITE_OMIT_TEMPDB + "OMIT_TEMPDB", +#endif +#ifdef SQLITE_OMIT_TRACE + "OMIT_TRACE", +#endif +#ifdef SQLITE_OMIT_TRIGGER + "OMIT_TRIGGER", +#endif +#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + "OMIT_TRUNCATE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_UTF16 + "OMIT_UTF16", +#endif +#ifdef SQLITE_OMIT_VACUUM + "OMIT_VACUUM", +#endif +#ifdef SQLITE_OMIT_VIEW + "OMIT_VIEW", +#endif +#ifdef SQLITE_OMIT_VIRTUALTABLE + "OMIT_VIRTUALTABLE", +#endif +#ifdef SQLITE_OMIT_WSD + "OMIT_WSD", +#endif +#ifdef SQLITE_OMIT_XFER_OPT + "OMIT_XFER_OPT", +#endif +#ifdef SQLITE_PERFORMANCE_TRACE + "PERFORMANCE_TRACE", +#endif +#ifdef SQLITE_PROXY_DEBUG + "PROXY_DEBUG", +#endif +#ifdef SQLITE_SECURE_DELETE + "SECURE_DELETE", +#endif +#ifdef SQLITE_SMALL_STACK + "SMALL_STACK", +#endif +#ifdef SQLITE_SOUNDEX + "SOUNDEX", +#endif +#ifdef SQLITE_TCL + "TCL", +#endif +#ifdef SQLITE_TEMP_STORE + "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), +#endif +#ifdef SQLITE_TEST + "TEST", +#endif +#ifdef SQLITE_THREADSAFE + "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), +#endif +#ifdef SQLITE_USE_ALLOCA + "USE_ALLOCA", +#endif +#ifdef SQLITE_ZERO_MALLOC + "ZERO_MALLOC" +#endif +}; + +/* +** Given the name of a compile-time option, return true if that option +** was used and false if not. +** +** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix +** is not required for a match. +*/ +int sqlite3_compileoption_used(const char *zOptName){ + int i, n; + if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; + n = sqlite3Strlen30(zOptName); + + /* Since ArraySize(azCompileOpt) is normally in single digits, a + ** linear search is adequate. No need for a binary search. */ + for(i=0; i=0 && Nflags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); @@ -630,7 +630,6 @@ int sqlite3GenerateIndexKey( if( doMakeRec ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol+1, regOut); sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0); - sqlite3ExprCacheAffinityChange(pParse, regBase, nCol+1); } sqlite3ReleaseTempRange(pParse, regBase, nCol+1); return regBase; diff --git a/src/expr.c b/src/expr.c index aee2b740..034088fa 100644 --- a/src/expr.c +++ b/src/expr.c @@ -227,30 +227,6 @@ CollSeq *sqlite3BinaryCompareCollSeq( return pColl; } -/* -** Generate the operands for a comparison operation. Before -** generating the code for each operand, set the EP_AnyAff -** flag on the expression so that it will be able to used a -** cached column value that has previously undergone an -** affinity change. -*/ -static void codeCompareOperands( - Parse *pParse, /* Parsing and code generating context */ - Expr *pLeft, /* The left operand */ - int *pRegLeft, /* Register where left operand is stored */ - int *pFreeLeft, /* Free this register when done */ - Expr *pRight, /* The right operand */ - int *pRegRight, /* Register where right operand is stored */ - int *pFreeRight /* Write temp register for right operand there */ -){ - while( pLeft->op==TK_UPLUS ) pLeft = pLeft->pLeft; - pLeft->flags |= EP_AnyAff; - *pRegLeft = sqlite3ExprCodeTemp(pParse, pLeft, pFreeLeft); - while( pRight->op==TK_UPLUS ) pRight = pRight->pLeft; - pRight->flags |= EP_AnyAff; - *pRegRight = sqlite3ExprCodeTemp(pParse, pRight, pFreeRight); -} - /* ** Generate code for a comparison operator. */ @@ -272,10 +248,6 @@ static int codeCompare( addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, (void*)p4, P4_COLLSEQ); sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); - if( (p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_NONE ){ - sqlite3ExprCacheAffinityChange(pParse, in1, 1); - sqlite3ExprCacheAffinityChange(pParse, in2, 1); - } return addr; } @@ -1906,6 +1878,7 @@ static char *dup8bytes(Vdbe *v, const char *in){ return out; } +#ifndef SQLITE_OMIT_FLOATING_POINT /* ** Generate an instruction that will put the floating point ** value described by z[0..n-1] into register iMem. @@ -1925,6 +1898,7 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); } } +#endif /* @@ -1935,7 +1909,8 @@ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ ** z[n] character is guaranteed to be something that does not look ** like the continuation of the number. */ -static void codeInteger(Vdbe *v, Expr *pExpr, int negFlag, int iMem){ +static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ + Vdbe *v = pParse->pVdbe; if( pExpr->flags & EP_IntValue ){ int i = pExpr->u.iValue; if( negFlag ) i = -i; @@ -1951,7 +1926,11 @@ static void codeInteger(Vdbe *v, Expr *pExpr, int negFlag, int iMem){ zV = dup8bytes(v, (char*)&value); sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); }else{ +#ifdef SQLITE_OMIT_FLOATING_POINT + sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); +#else codeReal(v, z, negFlag, iMem); +#endif } } } @@ -1982,17 +1961,31 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ assert( iReg>0 ); /* Register numbers are always positive */ assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ - /* First replace any existing entry */ + /* The SQLITE_ColumnCache flag disables the column cache. This is used + ** for testing only - to verify that SQLite always gets the same answer + ** with and without the column cache. + */ + if( pParse->db->flags & SQLITE_ColumnCache ) return; + + /* First replace any existing entry. + ** + ** Actually, the way the column cache is currently used, we are guaranteed + ** that the object will never already be in cache. Verify this guarantee. + */ +#ifndef NDEBUG for(i=0, p=pParse->aColCache; iiReg && p->iTable==iTab && p->iColumn==iCol ){ cacheEntryClear(pParse, p); p->iLevel = pParse->iCacheLevel; p->iReg = iReg; - p->affChange = 0; p->lru = pParse->iCacheCnt++; return; } +#endif + assert( p->iReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); } +#endif /* Find an empty slot and replace it */ for(i=0, p=pParse->aColCache; iiTable = iTab; p->iColumn = iCol; p->iReg = iReg; - p->affChange = 0; p->tempReg = 0; p->lru = pParse->iCacheCnt++; return; @@ -2023,7 +2015,6 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ p->iTable = iTab; p->iColumn = iCol; p->iReg = iReg; - p->affChange = 0; p->tempReg = 0; p->lru = pParse->iCacheCnt++; return; @@ -2031,14 +2022,16 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ } /* -** Indicate that a register is being overwritten. Purge the register -** from the column cache. +** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. +** Purge the range of registers from the column cache. */ -void sqlite3ExprCacheRemove(Parse *pParse, int iReg){ +void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ int i; + int iLast = iReg + nReg - 1; struct yColCache *p; for(i=0, p=pParse->aColCache; iiReg==iReg ){ + int r = p->iReg; + if( r>=iReg && r<=iLast ){ cacheEntryClear(pParse, p); p->iReg = 0; } @@ -2097,28 +2090,20 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. -** -** This routine might attempt to reuse the value of the column that -** has already been loaded into a register. The value will always -** be used if it has not undergone any affinity changes. But if -** an affinity change has occurred, then the cached value will only be -** used if allowAffChng is true. */ int sqlite3ExprCodeGetColumn( Parse *pParse, /* Parsing and code generating context */ Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ - int iReg, /* Store results here */ - int allowAffChng /* True if prior affinity changes are OK */ + int iReg /* Store results here */ ){ Vdbe *v = pParse->pVdbe; int i; struct yColCache *p; for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn - && (!p->affChange || allowAffChng) ){ + if( p->iReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ p->lru = pParse->iCacheCnt++; sqlite3ExprCachePinRegister(pParse, p->iReg); return p->iReg; @@ -2156,15 +2141,7 @@ void sqlite3ExprCacheClear(Parse *pParse){ ** registers starting with iStart. */ void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ - int iEnd = iStart + iCount - 1; - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iStart && r<=iEnd ){ - p->affChange = 1; - } - } + sqlite3ExprCacheRemove(pParse, iStart, iCount); } /* @@ -2196,19 +2173,24 @@ void sqlite3ExprCodeCopy(Parse *pParse, int iFrom, int iTo, int nReg){ } } +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* ** Return true if any register in the range iFrom..iTo (inclusive) ** is used as part of the column cache. +** +** This routine is used within assert() and testcase() macros only +** and does not appear in a normal build. */ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ int i; struct yColCache *p; for(i=0, p=pParse->aColCache; iiReg; - if( r>=iFrom && r<=iTo ) return 1; + if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ } return 0; } +#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ /* ** If the last instruction coded is an ephemeral copy of any of @@ -2329,22 +2311,22 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pParse->ckBase>0 ); inReg = pExpr->iColumn + pParse->ckBase; }else{ - testcase( (pExpr->flags & EP_AnyAff)!=0 ); inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, - pExpr->iColumn, pExpr->iTable, target, - pExpr->flags & EP_AnyAff); + pExpr->iColumn, pExpr->iTable, target); } break; } case TK_INTEGER: { - codeInteger(v, pExpr, 0, target); + codeInteger(pParse, pExpr, 0, target); break; } +#ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pExpr->u.zToken, 0, target); break; } +#endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeAddOp4(v, OP_String8, 0, target, 0, pExpr->u.zToken, 0); @@ -2449,8 +2431,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ testcase( op==TK_GE ); testcase( op==TK_EQ ); testcase( op==TK_NE ); - codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, - pExpr->pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, inReg, SQLITE_STOREP2); testcase( regFree1==0 ); @@ -2461,8 +2443,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ case TK_ISNOT: { testcase( op==TK_IS ); testcase( op==TK_ISNOT ); - codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, - pExpr->pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); op = (op==TK_IS) ? TK_EQ : TK_NE; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); @@ -2514,11 +2496,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ case TK_UMINUS: { Expr *pLeft = pExpr->pLeft; assert( pLeft ); - if( pLeft->op==TK_FLOAT ){ + if( pLeft->op==TK_INTEGER ){ + codeInteger(pParse, pLeft, 1, target); +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( pLeft->op==TK_FLOAT ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pLeft->u.zToken, 1, target); - }else if( pLeft->op==TK_INTEGER ){ - codeInteger(v, pLeft, 1, target); +#endif }else{ regFree1 = r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_Integer, 0, r1); @@ -2606,7 +2590,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); for(i=1; ia[i].pExpr, target); sqlite3ExprCachePop(pParse, 1); @@ -2661,7 +2645,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( nFarg ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); } - sqlite3ExprCacheAffinityChange(pParse, r1, nFarg); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -2702,8 +2685,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ struct ExprList_item *pLItem = pExpr->x.pList->a; Expr *pRight = pLItem->pExpr; - codeCompareOperands(pParse, pLeft, &r1, ®Free1, - pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); testcase( regFree1==0 ); testcase( regFree2==0 ); r3 = sqlite3GetTempReg(pParse); @@ -2767,6 +2750,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ target )); +#ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an ** integer. Use OP_RealAffinity to make sure it is really real. */ if( pExpr->iColumn>=0 @@ -2774,6 +2758,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, target); } +#endif break; } @@ -3238,8 +3223,8 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ testcase( op==TK_EQ ); testcase( op==TK_NE ); testcase( jumpIfNull==0 ); - codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, - pExpr->pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull); testcase( regFree1==0 ); @@ -3250,8 +3235,8 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_ISNOT: { testcase( op==TK_IS ); testcase( op==TK_ISNOT ); - codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, - pExpr->pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); op = (op==TK_IS) ? TK_EQ : TK_NE; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, SQLITE_NULLEQ); @@ -3381,8 +3366,8 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ testcase( op==TK_EQ ); testcase( op==TK_NE ); testcase( jumpIfNull==0 ); - codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, - pExpr->pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, jumpIfNull); testcase( regFree1==0 ); @@ -3393,8 +3378,8 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_ISNOT: { testcase( pExpr->op==TK_IS ); testcase( pExpr->op==TK_ISNOT ); - codeCompareOperands(pParse, pExpr->pLeft, &r1, ®Free1, - pExpr->pRight, &r2, ®Free2); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, r1, r2, dest, SQLITE_NULLEQ); @@ -3439,57 +3424,61 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } /* -** Do a deep comparison of two expression trees. Return TRUE (non-zero) -** if they are identical and return FALSE if they differ in any way. +** Do a deep comparison of two expression trees. Return 0 if the two +** expressions are completely identical. Return 1 if they differ only +** by a COLLATE operator at the top level. Return 2 if there are differences +** other than the top-level COLLATE operator. ** -** Sometimes this routine will return FALSE even if the two expressions +** Sometimes this routine will return 2 even if the two expressions ** really are equivalent. If we cannot prove that the expressions are -** identical, we return FALSE just to be safe. So if this routine -** returns false, then you do not really know for certain if the two -** expressions are the same. But if you get a TRUE return, then you +** identical, we return 2 just to be safe. So if this routine +** returns 2, then you do not really know for certain if the two +** expressions are the same. But if you get a 0 or 1 return, then you ** can be sure the expressions are the same. In the places where -** this routine is used, it does not hurt to get an extra FALSE - that +** this routine is used, it does not hurt to get an extra 2 - that ** just might result in some slightly slower code. But returning -** an incorrect TRUE could lead to a malfunction. +** an incorrect 0 or 1 could lead to a malfunction. */ int sqlite3ExprCompare(Expr *pA, Expr *pB){ int i; if( pA==0||pB==0 ){ - return pB==pA; + return pB==pA ? 0 : 2; } assert( !ExprHasAnyProperty(pA, EP_TokenOnly|EP_Reduced) ); assert( !ExprHasAnyProperty(pB, EP_TokenOnly|EP_Reduced) ); if( ExprHasProperty(pA, EP_xIsSelect) || ExprHasProperty(pB, EP_xIsSelect) ){ - return 0; + return 2; } - if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 0; - if( pA->op!=pB->op ) return 0; - if( !sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 0; - if( !sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 0; + if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; + if( pA->op!=pB->op ) return 2; + if( sqlite3ExprCompare(pA->pLeft, pB->pLeft) ) return 2; + if( sqlite3ExprCompare(pA->pRight, pB->pRight) ) return 2; if( pA->x.pList && pB->x.pList ){ - if( pA->x.pList->nExpr!=pB->x.pList->nExpr ) return 0; + if( pA->x.pList->nExpr!=pB->x.pList->nExpr ) return 2; for(i=0; ix.pList->nExpr; i++){ Expr *pExprA = pA->x.pList->a[i].pExpr; Expr *pExprB = pB->x.pList->a[i].pExpr; - if( !sqlite3ExprCompare(pExprA, pExprB) ) return 0; + if( sqlite3ExprCompare(pExprA, pExprB) ) return 2; } }else if( pA->x.pList || pB->x.pList ){ - return 0; + return 2; } - if( pA->iTable!=pB->iTable || pA->iColumn!=pB->iColumn ) return 0; + if( pA->iTable!=pB->iTable || pA->iColumn!=pB->iColumn ) return 2; if( ExprHasProperty(pA, EP_IntValue) ){ if( !ExprHasProperty(pB, EP_IntValue) || pA->u.iValue!=pB->u.iValue ){ - return 0; + return 2; } }else if( pA->op!=TK_COLUMN && pA->u.zToken ){ - if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 0; + if( ExprHasProperty(pB, EP_IntValue) || NEVER(pB->u.zToken==0) ) return 2; if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return 0; + return 2; } } - return 1; + if( (pA->flags & EP_ExpCollate)!=(pB->flags & EP_ExpCollate) ) return 1; + if( (pA->flags & EP_ExpCollate)!=0 && pA->pColl!=pB->pColl ) return 2; + return 0; } @@ -3620,7 +3609,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ struct AggInfo_func *pItem = pAggInfo->aFunc; for(i=0; inFunc; i++, pItem++){ - if( sqlite3ExprCompare(pItem->pExpr, pExpr) ){ + if( sqlite3ExprCompare(pItem->pExpr, pExpr)==0 ){ break; } } @@ -3741,7 +3730,8 @@ int sqlite3GetTempRange(Parse *pParse, int nReg){ int i, n; i = pParse->iRangeReg; n = pParse->nRangeReg; - if( nReg<=n && !usedAsColumnCache(pParse, i, i+n-1) ){ + if( nReg<=n ){ + assert( !usedAsColumnCache(pParse, i, i+n-1) ); pParse->iRangeReg += nReg; pParse->nRangeReg -= nReg; }else{ @@ -3751,6 +3741,7 @@ int sqlite3GetTempRange(Parse *pParse, int nReg){ return i; } void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + sqlite3ExprCacheRemove(pParse, iReg, nReg); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; pParse->iRangeReg = iReg; diff --git a/src/func.c b/src/func.c index 8cb7a037..7ff1fecb 100644 --- a/src/func.c +++ b/src/func.c @@ -117,7 +117,10 @@ static void lengthFunc( } /* -** Implementation of the abs() function +** Implementation of the abs() function. +** +** IMP: R-23979-26855 The abs(X) function returns the absolute value of +** the numeric argument X. */ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); @@ -127,6 +130,9 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ i64 iVal = sqlite3_value_int64(argv[0]); if( iVal<0 ){ if( (iVal<<1)==0 ){ + /* IMP: R-35460-15084 If X is the integer -9223372036854775807 then + ** abs(X) throws an integer overflow error since there is no + ** equivalent positive 64-bit two complement value. */ sqlite3_result_error(context, "integer overflow", -1); return; } @@ -136,10 +142,16 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ break; } case SQLITE_NULL: { + /* IMP: R-37434-19929 Abs(X) returns NULL if X is NULL. */ sqlite3_result_null(context); break; } default: { + /* Because sqlite3_value_double() returns 0.0 if the argument is not + ** something that can be converted into a number, we have: + ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that + ** cannot be converted to a numeric value. + */ double rVal = sqlite3_value_double(argv[0]); if( rVal<0 ) rVal = -rVal; sqlite3_result_double(context, rVal); @@ -259,14 +271,24 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ } if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; r = sqlite3_value_double(argv[0]); - zBuf = sqlite3_mprintf("%.*f",n,r); - if( zBuf==0 ){ - sqlite3_result_error_nomem(context); + /* If Y==0 and X will fit in a 64-bit int, + ** handle the rounding directly, + ** otherwise use printf. + */ + if( n==0 && r>=0 && rdb->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); }else{ - sqlite3_result_zeroblob(context, (int)n); + sqlite3_result_zeroblob(context, (int)n); /* IMP: R-00293-64994 */ } } @@ -1041,9 +1118,16 @@ static void trimFunc( } +/* IMP: R-25361-16150 This function is omitted from SQLite by default. It +** is only available if the SQLITE_SOUNDEX compile-time option is used +** when SQLite is built. +*/ #ifdef SQLITE_SOUNDEX /* ** Compute the soundex encoding of a word. +** +** IMP: R-59782-00072 The soundex(X) function returns a string that is the +** soundex encoding of the string X. */ static void soundexFunc( sqlite3_context *context, @@ -1087,10 +1171,12 @@ static void soundexFunc( zResult[j] = 0; sqlite3_result_text(context, zResult, 4, SQLITE_TRANSIENT); }else{ + /* IMP: R-64894-50321 The string "?000" is returned if the argument + ** is NULL or contains no ASCII alphabetic characters. */ sqlite3_result_text(context, "?000", 4, SQLITE_STATIC); } } -#endif +#endif /* SQLITE_SOUNDEX */ #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -1462,6 +1548,10 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(nullif, 2, 0, 1, nullifFunc ), FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), FUNCTION(changes, 0, 0, 0, changes ), diff --git a/src/global.c b/src/global.c index bdfd1ff2..673a274c 100644 --- a/src/global.c +++ b/src/global.c @@ -164,6 +164,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* isPCacheInit */ 0, /* pInitMutex */ 0, /* nRefInitMutex */ + 0, /* xLog */ + 0, /* pLogArg */ }; diff --git a/src/insert.c b/src/insert.c index 5266af32..53797620 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1261,19 +1261,33 @@ void sqlite3GenerateConstraintChecks( ** the triggers and remove both the table and index b-tree entries. ** ** Otherwise, if there are no triggers or the recursive-triggers - ** flag is not set, call GenerateRowIndexDelete(). This removes - ** the index b-tree entries only. The table b-tree entry will be - ** replaced by the new entry when it is inserted. */ + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. + ** + ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, + ** also invoke MultiWrite() to indicate that this VDBE may require + ** statement rollback (if the statement is aborted after the delete + ** takes place). Earlier versions called sqlite3MultiWrite() regardless, + ** but being more selective here allows statements like: + ** + ** REPLACE INTO t(rowid) VALUES($newrowid) + ** + ** to run without a statement journal if there are no indexes on the + ** table. + */ Trigger *pTrigger = 0; if( pParse->db->flags&SQLITE_RecTriggers ){ pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } - sqlite3MultiWrite(pParse); if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ + sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete( pParse, pTab, baseCur, regRowid, 0, pTrigger, OE_Replace ); - }else{ + }else if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); sqlite3GenerateRowIndexDelete(pParse, pTab, baseCur, 0); } seenReplace = 1; @@ -1715,7 +1729,7 @@ static int xferOptimization( } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && !sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ + if( pDest->pCheck && sqlite3ExprCompare(pSrc->pCheck, pDest->pCheck) ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif diff --git a/src/legacy.c b/src/legacy.c index d432d299..ebab2de3 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -41,6 +41,7 @@ int sqlite3_exec( int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; if( zSql==0 ) zSql = ""; sqlite3_mutex_enter(db->mutex); diff --git a/src/main.c b/src/main.c index a6754278..c8025079 100644 --- a/src/main.c +++ b/src/main.c @@ -257,7 +257,7 @@ int sqlite3_config(int op, ...){ /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE; + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; va_start(ap, op); switch( op ){ @@ -265,7 +265,7 @@ int sqlite3_config(int op, ...){ /* Mutex configuration options are only available in a threadsafe ** compile. */ -#if SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 case SQLITE_CONFIG_SINGLETHREAD: { /* Disable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 0; @@ -378,6 +378,21 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.nLookaside = va_arg(ap, int); break; } + + /* Record a pointer to the logger funcction and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } default: { rc = SQLITE_ERROR; @@ -591,7 +606,7 @@ int sqlite3_close(sqlite3 *db){ return SQLITE_OK; } if( !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -938,7 +953,7 @@ int sqlite3CreateFunc( (!xFunc && (!xFinal && xStep)) || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || (255<(nName = sqlite3Strlen30( zFunctionName))) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } #ifndef SQLITE_OMIT_UTF16 @@ -1035,7 +1050,7 @@ int sqlite3_create_function16( char *zFunc8; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); - zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); @@ -1269,7 +1284,7 @@ const char *sqlite3_errmsg(sqlite3 *db){ return sqlite3ErrStr(SQLITE_NOMEM); } if( !sqlite3SafetyCheckSickOrOk(db) ){ - return sqlite3ErrStr(SQLITE_MISUSE); + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); } sqlite3_mutex_enter(db->mutex); if( db->mallocFailed ){ @@ -1338,7 +1353,7 @@ const void *sqlite3_errmsg16(sqlite3 *db){ */ int sqlite3_errcode(sqlite3 *db){ if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ return SQLITE_NOMEM; @@ -1347,7 +1362,7 @@ int sqlite3_errcode(sqlite3 *db){ } int sqlite3_extended_errcode(sqlite3 *db){ if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } if( !db || db->mallocFailed ){ return SQLITE_NOMEM; @@ -1385,7 +1400,7 @@ static int createCollation( enc2 = SQLITE_UTF16NATIVE; } if( enc2SQLITE_UTF16BE ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } /* Check if this call is removing or replacing an existing collation @@ -1858,7 +1873,7 @@ int sqlite3_create_collation16( char *zName8; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); - zName8 = sqlite3Utf16to8(db, zName, -1); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); if( zName8 ){ rc = createCollation(db, zName8, (u8)enc, SQLITE_COLL_USER, pCtx, xCompare, 0); sqlite3DbFree(db, zName8); @@ -1929,16 +1944,34 @@ int sqlite3_get_autocommit(sqlite3 *db){ return db->autoCommit; } -#ifdef SQLITE_DEBUG /* -** The following routine is subtituted for constant SQLITE_CORRUPT in -** debugging builds. This provides a way to set a breakpoint for when -** corruption is first detected. +** The following routines are subtitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error +** constants. They server two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. */ -int sqlite3Corrupt(void){ +int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CORRUPT, + "database corruption found by source line %d", lineno); return SQLITE_CORRUPT; } -#endif +int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_MISUSE, "misuse detected by source line %d", lineno); + return SQLITE_MISUSE; +} +int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_CANTOPEN, "cannot open file at source line %d", lineno); + return SQLITE_CANTOPEN; +} + #ifndef SQLITE_OMIT_DEPRECATED /* @@ -1982,7 +2015,6 @@ int sqlite3_table_column_metadata( /* Ensure the database schema has been loaded */ sqlite3_mutex_enter(db->mutex); - (void)sqlite3SafetyOn(db); sqlite3BtreeEnterAll(db); rc = sqlite3Init(db, &zErrMsg); if( SQLITE_OK!=rc ){ @@ -2041,7 +2073,6 @@ int sqlite3_table_column_metadata( error_out: sqlite3BtreeLeaveAll(db); - (void)sqlite3SafetyOff(db); /* Whether the function call succeeded or failed, set the output parameters ** to whatever their local counterparts contain. If an error did occur, @@ -2281,6 +2312,40 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + int x = va_arg(ap,int); + db->flags = (x & SQLITE_OptMask) | (db->flags & ~SQLITE_OptMask); + break; + } + +#ifdef SQLITE_N_KEYWORD + /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + ** + ** If zWord is a keyword recognized by the parser, then return the + ** number of keywords. Or if zWord is not a keyword, return 0. + ** + ** This test feature is only available in the amalgamation since + ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite + ** is built using separate source files. + */ + case SQLITE_TESTCTRL_ISKEYWORD: { + const char *zWord = va_arg(ap, const char*); + int n = sqlite3Strlen30(zWord); + rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; + break; + } +#endif + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/mem1.c b/src/mem1.c index fbfa35eb..1a018399 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -42,6 +42,9 @@ static void *sqlite3MemMalloc(int nByte){ if( p ){ p[0] = nByte; p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); } return (void *)p; } @@ -61,6 +64,18 @@ static void sqlite3MemFree(void *pPrior){ free(p); } +/* +** Report the allocated size of a prior return from xMalloc() +** or xRealloc(). +*/ +static int sqlite3MemSize(void *pPrior){ + sqlite3_int64 *p; + if( pPrior==0 ) return 0; + p = (sqlite3_int64*)pPrior; + p--; + return (int)p[0]; +} + /* ** Like realloc(). Resize an allocation previously obtained from ** sqlite3MemMalloc(). @@ -75,28 +90,20 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); nByte = ROUND8(nByte); - p = (sqlite3_int64*)pPrior; p--; p = realloc(p, nByte+8 ); if( p ){ p[0] = nByte; p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + sqlite3MemSize(pPrior), nByte); } return (void*)p; } -/* -** Report the allocated size of a prior return from xMalloc() -** or xRealloc(). -*/ -static int sqlite3MemSize(void *pPrior){ - sqlite3_int64 *p; - if( pPrior==0 ) return 0; - p = (sqlite3_int64*)pPrior; - p--; - return (int)p[0]; -} - /* ** Round up a request size to the next valid allocation size. */ diff --git a/src/mem2.c b/src/mem2.c index 965f304b..5eb937ed 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -210,6 +210,31 @@ static int sqlite3MemRoundup(int n){ return ROUND8(n); } +/* +** Fill a buffer with pseudo-random bytes. This is used to preset +** the content of a new memory allocation to unpredictable values and +** to clear the content of a freed allocation to unpredictable values. +*/ +static void randomFill(char *pBuf, int nByte){ + unsigned int x, y, r; + x = SQLITE_PTR_TO_INT(pBuf); + y = nByte | 1; + while( nByte >= 4 ){ + x = (x>>1) ^ (-(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(int*)pBuf = r; + pBuf += 4; + nByte -= 4; + } + while( nByte-- > 0 ){ + x = (x>>1) ^ (-(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(pBuf++) = r & 0xff; + } +} + /* ** Allocate nByte bytes of memory. */ @@ -260,7 +285,8 @@ static void *sqlite3MemMalloc(int nByte){ adjustStats(nByte, +1); pInt = (int*)&pHdr[1]; pInt[nReserve/sizeof(int)] = REARGUARD; - memset(pInt, 0x65, nReserve); + randomFill((char*)pInt, nByte); + memset(((char*)pInt)+nByte, 0x65, nReserve-nByte); p = (void*)pInt; } sqlite3_mutex_leave(mem.mutex); @@ -274,7 +300,8 @@ static void sqlite3MemFree(void *pPrior){ struct MemBlockHdr *pHdr; void **pBt; char *z; - assert( sqlite3GlobalConfig.bMemstat || mem.mutex!=0 ); + assert( sqlite3GlobalConfig.bMemstat || sqlite3GlobalConfig.bCoreMutex==0 + || mem.mutex!=0 ); pHdr = sqlite3MemsysGetHeader(pPrior); pBt = (void**)pHdr; pBt -= pHdr->nBacktraceSlots; @@ -296,8 +323,8 @@ static void sqlite3MemFree(void *pPrior){ z = (char*)pBt; z -= pHdr->nTitle; adjustStats(pHdr->iSize, -1); - memset(z, 0x2b, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + - pHdr->iSize + sizeof(int) + pHdr->nTitle); + randomFill(z, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + + pHdr->iSize + sizeof(int) + pHdr->nTitle); free(z); sqlite3_mutex_leave(mem.mutex); } @@ -320,7 +347,7 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ if( pNew ){ memcpy(pNew, pPrior, nByteiSize ? nByte : pOldHdr->iSize); if( nByte>pOldHdr->iSize ){ - memset(&((char*)pNew)[pOldHdr->iSize], 0x2b, nByte - pOldHdr->iSize); + randomFill(&((char*)pNew)[pOldHdr->iSize], nByte - pOldHdr->iSize); } sqlite3MemFree(pPrior); } diff --git a/src/mem5.c b/src/mem5.c index 3fe04e24..a828cf81 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -268,7 +268,11 @@ static void *memsys5MallocUnsafe(int nByte){ ** two in order to create a new free block of size iLogsize. */ for(iBin=iLogsize; mem5.aiFreelist[iBin]<0 && iBin<=LOGMAX; iBin++){} - if( iBin>LOGMAX ) return 0; + if( iBin>LOGMAX ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte); + return 0; + } i = memsys5UnlinkFirst(iBin); while( iBin>iLogsize ){ int newSize; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 181f8212..237d24ef 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -27,7 +27,16 @@ struct sqlite3_mutex { int id; /* Mutex type */ int nRef; /* Number of enterances */ DWORD owner; /* Thread holding this mutex */ +#ifdef SQLITE_DEBUG + int trace; /* True to trace changes */ +#endif }; +#define SQLITE_W32_MUTEX_INITIALIZER { 0 } +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, 0L, (DWORD)0, 0 } +#else +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, 0L, (DWORD)0 } +#endif /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, @@ -71,8 +80,12 @@ struct sqlite3_mutex { static int winMutexHeld(sqlite3_mutex *p){ return p->nRef!=0 && p->owner==GetCurrentThreadId(); } +static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ + return p->nRef==0 || p->owner!=tid; +} static int winMutexNotheld(sqlite3_mutex *p){ - return p->nRef==0 || p->owner!=GetCurrentThreadId(); + DWORD tid = GetCurrentThreadId(); + return winMutexNotheld2(p, tid); } #endif @@ -80,7 +93,14 @@ static int winMutexNotheld(sqlite3_mutex *p){ /* ** Initialize and deinitialize the mutex subsystem. */ -static sqlite3_mutex winMutex_staticMutexes[6]; +static sqlite3_mutex winMutex_staticMutexes[6] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER +}; static int winMutex_isInit = 0; /* As winMutexInit() and winMutexEnd() are called as part ** of the sqlite3_initialize and sqlite3_shutdown() @@ -214,14 +234,23 @@ static void winMutexFree(sqlite3_mutex *p){ ** more than once, the behavior is undefined. */ static void winMutexEnter(sqlite3_mutex *p){ - assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld(p) ); + DWORD tid = GetCurrentThreadId(); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); EnterCriticalSection(&p->mutex); - p->owner = GetCurrentThreadId(); + p->owner = tid; p->nRef++; +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif } static int winMutexTry(sqlite3_mutex *p){ +#ifndef NDEBUG + DWORD tid = GetCurrentThreadId(); +#endif int rc = SQLITE_BUSY; - assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld(p) ); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); /* ** The sqlite3_mutex_try() routine is very rarely used, and when it ** is used it is merely an optimization. So it is OK for it to always @@ -235,12 +264,17 @@ static int winMutexTry(sqlite3_mutex *p){ */ #if 0 if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ - p->owner = GetCurrentThreadId(); + p->owner = tid; p->nRef++; rc = SQLITE_OK; } #else UNUSED_PARAMETER(p); +#endif +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK && p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } #endif return rc; } @@ -252,11 +286,19 @@ static int winMutexTry(sqlite3_mutex *p){ ** is not currently allocated. SQLite will never do either. */ static void winMutexLeave(sqlite3_mutex *p){ +#ifndef NDEBUG + DWORD tid = GetCurrentThreadId(); +#endif assert( p->nRef>0 ); - assert( p->owner==GetCurrentThreadId() ); + assert( p->owner==tid ); p->nRef--; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); LeaveCriticalSection(&p->mutex); +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif } sqlite3_mutex_methods *sqlite3DefaultMutex(void){ diff --git a/src/os.c b/src/os.c index 598383d5..f3600cb0 100644 --- a/src/os.c +++ b/src/os.c @@ -112,11 +112,11 @@ int sqlite3OsOpen( ){ int rc; DO_OS_MALLOC_TEST(0); - /* 0x7f1f is a mask of SQLITE_OPEN_ flags that are valid to be passed + /* 0x7f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x7f1f, pFlagsOut); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x7f3f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } diff --git a/src/os_unix.c b/src/os_unix.c index d7a9d8ec..769e75df 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -128,10 +128,18 @@ # else # include # include -# include # endif #endif /* SQLITE_ENABLE_LOCKING_STYLE */ +#if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) +# include +#endif + +/* +** Allowed values of unixFile.fsFlags +*/ +#define SQLITE_FSFLAGS_IS_MSDOS 0x1 + /* ** If we are to be thread-safe, include the pthreads header and define ** the SQLITE_UNIX_THREADS macro. @@ -199,6 +207,9 @@ struct unixFile { #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ #endif +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) + unsigned fsFlags; /* cached details from statfs() */ +#endif #if SQLITE_THREADSAFE && defined(__linux__) pthread_t tid; /* The thread that "owns" this unixFile */ #endif @@ -747,6 +758,9 @@ struct unixLockInfo { int cnt; /* Number of SHARED locks held */ int locktype; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ int nRef; /* Number of pointers to this structure */ +#if defined(SQLITE_ENABLE_LOCKING_STYLE) + unsigned long long sharedByte; /* for AFP simulated shared lock */ +#endif struct unixLockInfo *pNext; /* List of all unixLockInfo objects */ struct unixLockInfo *pPrev; /* .... doubly linked */ }; @@ -990,9 +1004,10 @@ static int findLockInfo( ** is a race condition such that another thread has already populated ** the first page of the database, no damage is done. */ - if( statbuf.st_size==0 ){ + if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ rc = write(fd, "S", 1); if( rc!=1 ){ + pFile->lastErrno = errno; return SQLITE_IOERR; } rc = fstat(fd, &statbuf); @@ -1032,6 +1047,9 @@ static int findLockInfo( pLock->nRef = 1; pLock->cnt = 0; pLock->locktype = 0; +#if defined(SQLITE_ENABLE_LOCKING_STYLE) + pLock->sharedByte = 0; +#endif pLock->pNext = lockList; pLock->pPrev = 0; if( lockList ) lockList->pPrev = pLock; @@ -1096,7 +1114,7 @@ static int transferOwnership(unixFile *pFile){ } if( pFile->locktype!=NO_LOCK ){ /* We cannot change ownership while we are holding a lock! */ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } OSTRACE4("Transfer ownership of %d from %d to %d\n", pFile->h, pFile->tid, hSelf); @@ -1165,62 +1183,6 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ return rc; } -/* -** Perform a file locking operation on a range of bytes in a file. -** The "op" parameter should be one of F_RDLCK, F_WRLCK, or F_UNLCK. -** Return 0 on success or -1 for failure. On failure, write the error -** code into *pErrcode. -** -** If the SQLITE_WHOLE_FILE_LOCKING bit is clear, then only lock -** the range of bytes on the locking page between SHARED_FIRST and -** SHARED_SIZE. If SQLITE_WHOLE_FILE_LOCKING is set, then lock all -** bytes from 0 up to but not including PENDING_BYTE, and all bytes -** that follow SHARED_FIRST. -** -** In other words, of SQLITE_WHOLE_FILE_LOCKING if false (the historical -** default case) then only lock a small range of bytes from SHARED_FIRST -** through SHARED_FIRST+SHARED_SIZE-1. But if SQLITE_WHOLE_FILE_LOCKING is -** true then lock every byte in the file except for PENDING_BYTE and -** RESERVED_BYTE. -** -** SQLITE_WHOLE_FILE_LOCKING=true overlaps SQLITE_WHOLE_FILE_LOCKING=false -** and so the locking schemes are compatible. One type of lock will -** effectively exclude the other type. The reason for using the -** SQLITE_WHOLE_FILE_LOCKING=true is that by indicating the full range -** of bytes to be read or written, we give hints to NFS to help it -** maintain cache coherency. On the other hand, whole file locking -** is slower, so we don't want to use it except for NFS. -*/ -static int rangeLock(unixFile *pFile, int op, int *pErrcode){ - struct flock lock; - int rc; - lock.l_type = op; - lock.l_start = SHARED_FIRST; - lock.l_whence = SEEK_SET; - if( (pFile->fileFlags & SQLITE_WHOLE_FILE_LOCKING)==0 ){ - lock.l_len = SHARED_SIZE; - rc = fcntl(pFile->h, F_SETLK, &lock); - *pErrcode = errno; - }else{ - lock.l_len = 0; - rc = fcntl(pFile->h, F_SETLK, &lock); - *pErrcode = errno; - if( NEVER(op==F_UNLCK) || rc!=(-1) ){ - lock.l_start = 0; - lock.l_len = PENDING_BYTE; - rc = fcntl(pFile->h, F_SETLK, &lock); - if( ALWAYS(op!=F_UNLCK) && rc==(-1) ){ - *pErrcode = errno; - lock.l_type = F_UNLCK; - lock.l_start = SHARED_FIRST; - lock.l_len = 0; - fcntl(pFile->h, F_SETLK, &lock); - } - } - } - return rc; -} - /* ** Lock the file with the lock specified by parameter locktype - one ** of the following: @@ -1289,7 +1251,7 @@ static int unixLock(sqlite3_file *id, int locktype){ struct unixLockInfo *pLock = pFile->pLock; struct flock lock; int s = 0; - int tErrno; + int tErrno = 0; assert( pFile ); OSTRACE7("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, @@ -1385,8 +1347,11 @@ static int unixLock(sqlite3_file *id, int locktype){ assert( pLock->locktype==0 ); /* Now get the read-lock */ - s = rangeLock(pFile, F_RDLCK, &tErrno); - + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( (s = fcntl(pFile->h, F_SETLK, &lock))==(-1) ){ + tErrno = errno; + } /* Drop the temporary PENDING lock */ lock.l_start = PENDING_BYTE; lock.l_len = 1L; @@ -1426,16 +1391,17 @@ static int unixLock(sqlite3_file *id, int locktype){ switch( locktype ){ case RESERVED_LOCK: lock.l_start = RESERVED_BYTE; - s = fcntl(pFile->h, F_SETLK, &lock); - tErrno = errno; break; case EXCLUSIVE_LOCK: - s = rangeLock(pFile, F_WRLCK, &tErrno); + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; break; default: assert(0); } + s = fcntl(pFile->h, F_SETLK, &lock); if( s==(-1) ){ + tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(rc) ){ pFile->lastErrno = tErrno; @@ -1525,13 +1491,19 @@ static void setPendingFd(unixFile *pFile){ ** ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. +** +** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED +** the byte range is divided into 2 parts and the first part is unlocked then +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** remove the write lock on a region when a read lock is set. */ -static int unixUnlock(sqlite3_file *id, int locktype){ - unixFile *pFile = (unixFile*)id; /* The open file */ - struct unixLockInfo *pLock; /* Structure describing current lock state */ - struct flock lock; /* Information passed into fcntl() */ - int rc = SQLITE_OK; /* Return code from this interface */ - int h; /* The underlying file descriptor */ +static int _posixUnlock(sqlite3_file *id, int locktype, int handleNFSUnlock){ + unixFile *pFile = (unixFile*)id; + struct unixLockInfo *pLock; + struct flock lock; + int rc = SQLITE_OK; + int h; int tErrno; /* Error code from system call errors */ assert( pFile ); @@ -1543,7 +1515,7 @@ static int unixUnlock(sqlite3_file *id, int locktype){ return SQLITE_OK; } if( CHECK_THREADID(pFile) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } unixEnterMutex(); h = pFile->h; @@ -1570,14 +1542,68 @@ static int unixUnlock(sqlite3_file *id, int locktype){ pFile->inNormalWrite = 0; #endif - + /* downgrading to a shared lock on NFS involves clearing the write lock + ** before establishing the readlock - to avoid a race condition we downgrade + ** the lock in 2 blocks, so that part of the range will be covered by a + ** write lock until the rest is covered by a read lock: + ** 1: [WWWWW] + ** 2: [....W] + ** 3: [RRRRW] + ** 4: [RRRR.] + */ if( locktype==SHARED_LOCK ){ - if( rangeLock(pFile, F_RDLCK, &tErrno)==(-1) ){ - rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; + if( handleNFSUnlock ){ + off_t divSize = SHARED_SIZE - 1; + + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( fcntl(h, F_SETLK, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( fcntl(h, F_SETLK, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST+divSize; + lock.l_len = SHARED_SIZE-divSize; + if( fcntl(h, F_SETLK, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; + } + }else{ + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( fcntl(h, F_SETLK, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } + goto end_unlock; } - goto end_unlock; } } lock.l_type = F_UNLCK; @@ -1644,6 +1670,17 @@ end_unlock: return rc; } +/* +** Lower the locking level on file descriptor pFile to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int unixUnlock(sqlite3_file *id, int locktype){ + return _posixUnlock(id, locktype, 0); +} + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -2355,7 +2392,7 @@ static int semClose(sqlite3_file *id) { */ typedef struct afpLockingContext afpLockingContext; struct afpLockingContext { - unsigned long long sharedByte; + int reserved; const char *dbPath; /* Name of the open file */ }; @@ -2432,9 +2469,14 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ assert( pFile ); afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + if( context->reserved ){ + *pResOut = 1; + return SQLITE_OK; + } + unixEnterMutex(); /* Because pFile->pLock is shared across threads */ /* Check if a thread in this process holds such a lock */ - if( pFile->locktype>SHARED_LOCK ){ + if( pFile->pLock->locktype>SHARED_LOCK ){ reserved = 1; } @@ -2456,6 +2498,7 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ } } + unixLeaveMutex(); OSTRACE4("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved); *pResOut = reserved; @@ -2489,11 +2532,13 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ static int afpLock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; + struct unixLockInfo *pLock = pFile->pLock; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; assert( pFile ); - OSTRACE5("LOCK %d %s was %s pid=%d (afp)\n", pFile->h, - locktypeName(locktype), locktypeName(pFile->locktype), getpid()); + OSTRACE7("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, + locktypeName(locktype), locktypeName(pFile->locktype), + locktypeName(pLock->locktype), pLock->cnt , getpid()); /* If there is already a lock of this type or more restrictive on the ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as @@ -2506,6 +2551,9 @@ static int afpLock(sqlite3_file *id, int locktype){ } /* Make sure the locking sequence is correct + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); assert( locktype!=PENDING_LOCK ); @@ -2522,6 +2570,32 @@ static int afpLock(sqlite3_file *id, int locktype){ unixLeaveMutex(); return rc; } + pLock = pFile->pLock; + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->locktype!=pLock->locktype && + (pLock->locktype>=PENDING_LOCK || locktype>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto afp_end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( locktype==SHARED_LOCK && + (pLock->locktype==SHARED_LOCK || pLock->locktype==RESERVED_LOCK) ){ + assert( locktype==SHARED_LOCK ); + assert( pFile->locktype==0 ); + assert( pLock->cnt>0 ); + pFile->locktype = SHARED_LOCK; + pLock->cnt++; + pFile->pOpen->nLock++; + goto afp_end_lock; + } /* A PENDING lock is needed before acquiring a SHARED lock and before ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will @@ -2542,15 +2616,19 @@ static int afpLock(sqlite3_file *id, int locktype){ ** operating system calls for the specified lock. */ if( locktype==SHARED_LOCK ){ - int lk, lrc1, lrc2; - int lrc1Errno = 0; + int lrc1, lrc2, lrc1Errno; + long lk, mask; + assert( pLock->cnt==0 ); + assert( pLock->locktype==0 ); + + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; /* Now get the read-lock SHARED_LOCK */ /* note that the quality of the randomness doesn't matter that much */ lk = random(); - context->sharedByte = (lk & 0x7fffffff)%(SHARED_SIZE - 1); + pLock->sharedByte = (lk & mask)%(SHARED_SIZE - 1); lrc1 = afpSetLock(context->dbPath, pFile, - SHARED_FIRST+context->sharedByte, 1, 1); + SHARED_FIRST+pLock->sharedByte, 1, 1); if( IS_LOCK_ERROR(lrc1) ){ lrc1Errno = pFile->lastErrno; } @@ -2569,7 +2647,12 @@ static int afpLock(sqlite3_file *id, int locktype){ } else { pFile->locktype = SHARED_LOCK; pFile->pOpen->nLock++; + pLock->cnt = 1; } + }else if( locktype==EXCLUSIVE_LOCK && pLock->cnt>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file @@ -2580,6 +2663,9 @@ static int afpLock(sqlite3_file *id, int locktype){ if (locktype >= RESERVED_LOCK && pFile->locktype < RESERVED_LOCK) { /* Acquire a RESERVED lock */ failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( !failed ){ + context->reserved = 1; + } } if (!failed && locktype == EXCLUSIVE_LOCK) { /* Acquire an EXCLUSIVE lock */ @@ -2588,13 +2674,13 @@ static int afpLock(sqlite3_file *id, int locktype){ ** reestablish the shared lock if we can't get the afpUnlock */ if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + - context->sharedByte, 1, 0)) ){ + pLock->sharedByte, 1, 0)) ){ int failed2 = SQLITE_OK; /* now attemmpt to get the exclusive lock range */ failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 1); if( failed && (failed2 = afpSetLock(context->dbPath, pFile, - SHARED_FIRST + context->sharedByte, 1, 1)) ){ + SHARED_FIRST + pLock->sharedByte, 1, 1)) ){ /* Can't reestablish the shared lock. Sqlite can't deal, this is ** a critical I/O error */ @@ -2613,8 +2699,10 @@ static int afpLock(sqlite3_file *id, int locktype){ if( rc==SQLITE_OK ){ pFile->locktype = locktype; + pLock->locktype = locktype; }else if( locktype==EXCLUSIVE_LOCK ){ pFile->locktype = PENDING_LOCK; + pLock->locktype = PENDING_LOCK; } afp_end_lock: @@ -2634,45 +2722,94 @@ afp_end_lock: static int afpUnlock(sqlite3_file *id, int locktype) { int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; - afpLockingContext *pCtx = (afpLockingContext *) pFile->lockingContext; + struct unixLockInfo *pLock; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + int skipShared = 0; +#ifdef SQLITE_TEST + int h = pFile->h; +#endif assert( pFile ); - OSTRACE5("UNLOCK %d %d was %d pid=%d (afp)\n", pFile->h, locktype, - pFile->locktype, getpid()); + OSTRACE7("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, locktype, + pFile->locktype, pFile->pLock->locktype, pFile->pLock->cnt, getpid()); assert( locktype<=SHARED_LOCK ); if( pFile->locktype<=locktype ){ return SQLITE_OK; } if( CHECK_THREADID(pFile) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } unixEnterMutex(); + pLock = pFile->pLock; + assert( pLock->cnt!=0 ); if( pFile->locktype>SHARED_LOCK ){ + assert( pLock->locktype==pFile->locktype ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifndef NDEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); + pFile->inNormalWrite = 0; +#endif if( pFile->locktype==EXCLUSIVE_LOCK ){ - rc = afpSetLock(pCtx->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); - if( rc==SQLITE_OK && locktype==SHARED_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); + if( rc==SQLITE_OK && (locktype==SHARED_LOCK || pLock->cnt>1) ){ /* only re-establish the shared lock if necessary */ - int sharedLockByte = SHARED_FIRST+pCtx->sharedByte; - rc = afpSetLock(pCtx->dbPath, pFile, sharedLockByte, 1, 1); + int sharedLockByte = SHARED_FIRST+pLock->sharedByte; + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); + } else { + skipShared = 1; } } if( rc==SQLITE_OK && pFile->locktype>=PENDING_LOCK ){ - rc = afpSetLock(pCtx->dbPath, pFile, PENDING_BYTE, 1, 0); + rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); } - if( rc==SQLITE_OK && pFile->locktype>=RESERVED_LOCK ){ - rc = afpSetLock(pCtx->dbPath, pFile, RESERVED_BYTE, 1, 0); + if( rc==SQLITE_OK && pFile->locktype>=RESERVED_LOCK && context->reserved ){ + rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + if( !rc ){ + context->reserved = 0; + } + } + if( rc==SQLITE_OK && (locktype==SHARED_LOCK || pLock->cnt>1)){ + pLock->locktype = SHARED_LOCK; } - }else if( locktype==NO_LOCK ){ - /* clear the shared lock */ - int sharedLockByte = SHARED_FIRST+pCtx->sharedByte; - rc = afpSetLock(pCtx->dbPath, pFile, sharedLockByte, 1, 0); } + if( rc==SQLITE_OK && locktype==NO_LOCK ){ - if( rc==SQLITE_OK ){ - if( locktype==NO_LOCK ){ + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + unsigned long long sharedLockByte = SHARED_FIRST+pLock->sharedByte; + pLock->cnt--; + if( pLock->cnt==0 ){ + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( !skipShared ){ + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); + } + if( !rc ){ + pLock->locktype = NO_LOCK; + pFile->locktype = NO_LOCK; + } + } + if( rc==SQLITE_OK ){ struct unixOpenCnt *pOpen = pFile->pOpen; + pOpen->nLock--; assert( pOpen->nLock>=0 ); if( pOpen->nLock==0 ){ @@ -2680,10 +2817,9 @@ static int afpUnlock(sqlite3_file *id, int locktype) { } } } + unixLeaveMutex(); - if( rc==SQLITE_OK ){ - pFile->locktype = locktype; - } + if( rc==SQLITE_OK ) pFile->locktype = locktype; return rc; } @@ -2691,6 +2827,7 @@ static int afpUnlock(sqlite3_file *id, int locktype) { ** Close a file & cleanup AFP specific locking context */ static int afpClose(sqlite3_file *id) { + int rc = SQLITE_OK; if( id ){ unixFile *pFile = (unixFile*)id; afpUnlock(id, NO_LOCK); @@ -2703,12 +2840,13 @@ static int afpClose(sqlite3_file *id) { */ setPendingFd(pFile); } + releaseLockInfo(pFile->pLock); releaseOpenCnt(pFile->pOpen); sqlite3_free(pFile->lockingContext); - closeUnixFile(id); + rc = closeUnixFile(id); unixLeaveMutex(); } - return SQLITE_OK; + return rc; } #endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ @@ -2721,6 +2859,29 @@ static int afpClose(sqlite3_file *id) { ********************* End of the AFP lock implementation ********************** ******************************************************************************/ +/****************************************************************************** +*************************** Begin NFS Locking ********************************/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* + ** Lower the locking level on file descriptor pFile to locktype. locktype + ** must be either NO_LOCK or SHARED_LOCK. + ** + ** If the locking level of the file descriptor is already at or below + ** the requested locking level, this routine is a no-op. + */ +static int nfsUnlock(sqlite3_file *id, int locktype){ + return _posixUnlock(id, locktype, 1); +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the NFS lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. +** +********************* End of the NFS lock implementation ********************** +******************************************************************************/ /****************************************************************************** **************** Non-locking sqlite3_file methods ***************************** @@ -2747,7 +2908,9 @@ static int afpClose(sqlite3_file *id) { */ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ int got; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) i64 newOffset; +#endif TIMER_START; #if defined(USE_PREAD) got = pread(id->h, pBuf, cnt, offset); @@ -2821,7 +2984,9 @@ static int unixRead( */ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ int got; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) i64 newOffset; +#endif TIMER_START; #if defined(USE_PREAD) got = pwrite(id->h, pBuf, cnt, offset); @@ -3015,6 +3180,11 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ */ if( rc ) rc = fsync(fd); +#elif defined(__APPLE__) + /* fdatasync() on HFS+ doesn't yet flush the file size if it changed correctly + ** so currently we default to the macro that redefines fdatasync to fsync + */ + rc = fsync(fd); #else rc = fdatasync(fd); #if OS_VXWORKS @@ -3349,23 +3519,6 @@ IOMETHODS( ) #endif -/* -** The "Whole File Locking" finder returns the same set of methods as -** the posix locking finder. But it also sets the SQLITE_WHOLE_FILE_LOCKING -** flag to force the posix advisory locks to cover the whole file instead -** of just a small span of bytes near the 1GiB boundary. Whole File Locking -** is useful on NFS-mounted files since it helps NFS to maintain cache -** coherency. But it is a detriment to other filesystems since it runs -** slower. -*/ -static const sqlite3_io_methods *posixWflIoFinderImpl(const char*z, unixFile*p){ - UNUSED_PARAMETER(z); - p->fileFlags = SQLITE_WHOLE_FILE_LOCKING; - return &posixIoMethods; -} -static const sqlite3_io_methods - *(*const posixWflIoFinder)(const char*,unixFile *p) = posixWflIoFinderImpl; - /* ** The proxy locking method is a "super-method" in the sense that it ** opens secondary file descriptors for the conch and lock files and @@ -3390,6 +3543,17 @@ IOMETHODS( ) #endif +/* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + nfsIoFinder, /* Finder function name */ + nfsIoMethods, /* sqlite3_io_methods object name */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + nfsUnlock, /* xUnlock method */ + unixCheckReservedLock /* xCheckReservedLock method */ +) +#endif #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE /* @@ -3410,11 +3574,7 @@ static const sqlite3_io_methods *autolockIoFinderImpl( { "hfs", &posixIoMethods }, { "ufs", &posixIoMethods }, { "afpfs", &afpIoMethods }, -#ifdef SQLITE_ENABLE_AFP_LOCKING_SMB { "smbfs", &afpIoMethods }, -#else - { "smbfs", &flockIoMethods }, -#endif { "webdav", &nolockIoMethods }, { 0, 0 } }; @@ -3447,8 +3607,11 @@ static const sqlite3_io_methods *autolockIoFinderImpl( lockInfo.l_whence = SEEK_SET; lockInfo.l_type = F_RDLCK; if( fcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { - pNew->fileFlags = SQLITE_WHOLE_FILE_LOCKING; - return &posixIoMethods; + if( strcmp(fsInfo.f_fstypename, "nfs")==0 ){ + return &nfsIoMethods; + } else { + return &posixIoMethods; + } }else{ return &dotlockIoMethods; } @@ -3559,7 +3722,11 @@ static int fillInUnixFile( #endif } - if( pLockingStyle == &posixIoMethods ){ + if( pLockingStyle == &posixIoMethods +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + || pLockingStyle == &nfsIoMethods +#endif + ){ unixEnterMutex(); rc = findLockInfo(pNew, &pNew->pLock, &pNew->pOpen); if( rc!=SQLITE_OK ){ @@ -3601,9 +3768,15 @@ static int fillInUnixFile( ** according to requirement F11141. So we do not need to make a ** copy of the filename. */ pCtx->dbPath = zFilename; + pCtx->reserved = 0; srandomdev(); unixEnterMutex(); - rc = findLockInfo(pNew, NULL, &pNew->pOpen); + rc = findLockInfo(pNew, &pNew->pLock, &pNew->pOpen); + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew->lockingContext); + close(h); + h = -1; + } unixLeaveMutex(); } } @@ -3652,6 +3825,8 @@ static int fillInUnixFile( pNew->lastErrno = 0; #if OS_VXWORKS if( rc!=SQLITE_OK ){ + if( h>=0 ) close(h); + h = -1; unlink(zFilename); isDelete = 0; } @@ -3695,7 +3870,7 @@ static int openDirectory(const char *zFilename, int *pFd){ } } *pFd = fd; - return (fd>=0?SQLITE_OK:SQLITE_CANTOPEN); + return (fd>=0?SQLITE_OK:SQLITE_CANTOPEN_BKPT); } /* @@ -3804,16 +3979,17 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** Even if a subsequent open() call does succeed, the consequences of ** not searching for a resusable file descriptor are not dire. */ if( 0==stat(zPath, &sStat) ){ - struct unixOpenCnt *pO; - struct unixFileId id; - id.dev = sStat.st_dev; - id.ino = sStat.st_ino; + struct unixOpenCnt *pOpen; unixEnterMutex(); - for(pO=openList; pO && memcmp(&id, &pO->fileId, sizeof(id)); pO=pO->pNext); - if( pO ){ + pOpen = openList; + while( pOpen && (pOpen->fileId.dev!=sStat.st_dev + || pOpen->fileId.ino!=sStat.st_ino) ){ + pOpen = pOpen->pNext; + } + if( pOpen ){ UnixUnusedFd **pp; - for(pp=&pO->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); + for(pp=&pOpen->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); pUnused = *pp; if( pUnused ){ *pp = pUnused->pNext; @@ -3867,6 +4043,9 @@ static int unixOpen( int isCreate = (flags & SQLITE_OPEN_CREATE); int isReadonly = (flags & SQLITE_OPEN_READONLY); int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +#if SQLITE_ENABLE_LOCKING_STYLE + int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); +#endif /* If creating a master or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() @@ -3954,7 +4133,7 @@ static int unixOpen( fd = open(zName, openFlags, openMode); } if( fd<0 ){ - rc = SQLITE_CANTOPEN; + rc = SQLITE_CANTOPEN_BKPT; goto open_finished; } } @@ -4000,8 +4179,25 @@ static int unixOpen( noLock = eType!=SQLITE_OPEN_MAIN_DB; + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; + if( fstatfs(fd, &fsInfo) == -1 ){ + ((unixFile*)pFile)->lastErrno = errno; + if( dirfd>=0 ) close(dirfd); /* silently leak if fail, in error */ + close(fd); /* silently leak if fail, in error */ + return SQLITE_IOERR_ACCESS; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE #if SQLITE_PREFER_PROXY_LOCKING - if( zPath!=NULL && !noLock && pVfs->xOpen ){ + isAutoProxy = 1; +#endif + if( isAutoProxy && (zPath!=NULL) && (!noLock) && pVfs->xOpen ){ char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); int useProxy = 0; @@ -4033,6 +4229,14 @@ static int unixOpen( rc = fillInUnixFile(pVfs, fd, dirfd, pFile, zPath, noLock, isDelete); if( rc==SQLITE_OK ){ rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); + if( rc!=SQLITE_OK ){ + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + */ + unixClose(pFile); + return rc; + } } goto open_finished; } @@ -4153,7 +4357,7 @@ static int unixFullPathname( }else{ int nCwd; if( getcwd(zOut, nOut-1)==0 ){ - return SQLITE_CANTOPEN; + return SQLITE_CANTOPEN_BKPT; } nCwd = (int)strlen(zOut); sqlite3_snprintf(nOut-nCwd, &zOut[nCwd], "/%s", zPath); @@ -4460,11 +4664,6 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** of the database file for multiple readers and writers on the same ** host (the conch ensures that they all use the same local lock file). ** -** There is a third file - the host ID file - used as a persistent record -** of a unique identifier for the host, a 128-byte unique host id file -** in the path defined by the HOSTIDPATH macro (default value is -** /Library/Caches/.com.apple.sqliteConchHostId). -** ** Requesting the lock proxy does not immediately take the conch, it is ** only taken when the first request to lock database file is made. ** This matches the semantics of the traditional locking behavior, where @@ -4490,10 +4689,6 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** Enables the logging of error messages during host id file ** retrieval and creation ** -** HOSTIDPATH -** -** Overrides the default host ID file path location -** ** LOCKPROXYDIR ** ** Overrides the default directory used for lock proxy files that @@ -4518,11 +4713,6 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ */ #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -#ifdef SQLITE_TEST -/* simulate multiple hosts by creating unique hostid file paths */ -int sqlite3_hostid_num = 0; -#endif - /* ** The proxyLockingContext has the path and file structures for the remote ** and local proxy files in it @@ -4534,134 +4724,16 @@ struct proxyLockingContext { unixFile *lockProxy; /* Open proxy lock file */ char *lockProxyPath; /* Name of the proxy lock file */ char *dbPath; /* Name of the open file */ - int conchHeld; /* True if the conch is currently held */ + int conchHeld; /* 1 if the conch is held, -1 if lockless */ void *oldLockingContext; /* Original lockingcontext to restore on close */ sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ }; -/* HOSTIDLEN and CONCHLEN both include space for the string -** terminating nul +/* +** The proxy lock file path for the database at dbPath is written into lPath, +** which must point to valid, writable memory large enough for a maxLen length +** file path. */ -#define HOSTIDLEN 128 -#define CONCHLEN (MAXPATHLEN+HOSTIDLEN+1) -#ifndef HOSTIDPATH -# define HOSTIDPATH "/Library/Caches/.com.apple.sqliteConchHostId" -#endif - -/* basically a copy of unixRandomness with different -** test behavior built in */ -static int proxyGenerateHostID(char *pHostID){ - int pid, fd, len; - unsigned char *key = (unsigned char *)pHostID; - - memset(key, 0, HOSTIDLEN); - len = 0; - fd = open("/dev/urandom", O_RDONLY); - if( fd>=0 ){ - len = read(fd, key, HOSTIDLEN); - close(fd); /* silently leak the fd if it fails */ - } - if( len < HOSTIDLEN ){ - time_t t; - time(&t); - memcpy(key, &t, sizeof(t)); - pid = getpid(); - memcpy(&key[sizeof(t)], &pid, sizeof(pid)); - } - -#ifdef MAKE_PRETTY_HOSTID - { - int i; - /* filter the bytes into printable ascii characters and NUL terminate */ - key[(HOSTIDLEN-1)] = 0x00; - for( i=0; i<(HOSTIDLEN-1); i++ ){ - unsigned char pa = key[i]&0x7F; - if( pa<0x20 ){ - key[i] = (key[i]&0x80 == 0x80) ? pa+0x40 : pa+0x20; - }else if( pa==0x7F ){ - key[i] = (key[i]&0x80 == 0x80) ? pa=0x20 : pa+0x7E; - } - } - } -#endif - return SQLITE_OK; -} - -/* writes the host id path to path, path should be an pre-allocated buffer -** with enough space for a path -*/ -static void proxyGetHostIDPath(char *path, size_t len){ - strlcpy(path, HOSTIDPATH, len); -#ifdef SQLITE_TEST - if( sqlite3_hostid_num>0 ){ - char suffix[2] = "1"; - suffix[0] = suffix[0] + sqlite3_hostid_num; - strlcat(path, suffix, len); - } -#endif - OSTRACE3("GETHOSTIDPATH %s pid=%d\n", path, getpid()); -} - -/* get the host ID from a sqlite hostid file stored in the -** user-specific tmp directory, create the ID if it's not there already -*/ -static int proxyGetHostID(char *pHostID, int *pError){ - int fd; - char path[MAXPATHLEN]; - size_t len; - int rc=SQLITE_OK; - - proxyGetHostIDPath(path, MAXPATHLEN); - /* try to create the host ID file, if it already exists read the contents */ - fd = open(path, O_CREAT|O_WRONLY|O_EXCL, 0644); - if( fd<0 ){ - int err=errno; - - if( err!=EEXIST ){ -#ifdef SQLITE_PROXY_DEBUG /* set the sqlite error message instead */ - fprintf(stderr, "sqlite error creating host ID file %s: %s\n", - path, strerror(err)); -#endif - return SQLITE_PERM; - } - /* couldn't create the file, read it instead */ - fd = open(path, O_RDONLY|O_EXCL); - if( fd<0 ){ -#ifdef SQLITE_PROXY_DEBUG /* set the sqlite error message instead */ - int err = errno; - fprintf(stderr, "sqlite error opening host ID file %s: %s\n", - path, strerror(err)); -#endif - return SQLITE_PERM; - } - len = pread(fd, pHostID, HOSTIDLEN, 0); - if( len<0 ){ - *pError = errno; - rc = SQLITE_IOERR_READ; - }else if( len 0) ){ + /* only mkdir if leaf dir != "." or "/" or ".." */ + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ + buf[i]='\0'; + if( mkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ + int err=errno; + if( err!=EEXIST ) { + OSTRACE5("CREATELOCKPATH FAILED creating %s, " + "'%s' proxy lock path=%s pid=%d\n", + buf, strerror(err), lockPath, getpid()); + return err; + } + } + } + start=i+1; + } + buf[i] = lockPath[i]; + } + OSTRACE3("CREATELOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid()); + return 0; +} + /* ** Create a new VFS file descriptor (stored in memory obtained from ** sqlite3_malloc) and open the file named "path" in the file descriptor. @@ -4715,48 +4815,263 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ ** The caller is responsible not only for closing the file descriptor ** but also for freeing the memory associated with the file descriptor. */ -static int proxyCreateUnixFile(const char *path, unixFile **ppFile) { +static int proxyCreateUnixFile( + const char *path, /* path for the new unixFile */ + unixFile **ppFile, /* unixFile created and returned by ref */ + int islockfile /* if non zero missing dirs will be created */ +) { + int fd = -1; + int dirfd = -1; unixFile *pNew; - int flags = SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; int rc = SQLITE_OK; + int openFlags = O_RDWR | O_CREAT; sqlite3_vfs dummyVfs; + int terrno = 0; + UnixUnusedFd *pUnused = NULL; - pNew = (unixFile *)sqlite3_malloc(sizeof(unixFile)); - if( !pNew ){ - return SQLITE_NOMEM; + /* 1. first try to open/create the file + ** 2. if that fails, and this is a lock file (not-conch), try creating + ** the parent directories and then try again. + ** 3. if that fails, try to open the file read-only + ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file + */ + pUnused = findReusableFd(path, openFlags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM; + } + } + if( fd<0 ){ + fd = open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + terrno = errno; + if( fd<0 && errno==ENOENT && islockfile ){ + if( proxyCreateLockPath(path) == SQLITE_OK ){ + fd = open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + } + } + } + if( fd<0 ){ + openFlags = O_RDONLY; + fd = open(path, openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); + terrno = errno; + } + if( fd<0 ){ + if( islockfile ){ + return SQLITE_BUSY; + } + switch (terrno) { + case EACCES: + return SQLITE_PERM; + case EIO: + return SQLITE_IOERR_LOCK; /* even though it is the conch */ + default: + return SQLITE_CANTOPEN_BKPT; + } + } + + pNew = (unixFile *)sqlite3_malloc(sizeof(*pNew)); + if( pNew==NULL ){ + rc = SQLITE_NOMEM; + goto end_create_proxy; } memset(pNew, 0, sizeof(unixFile)); - - /* Call unixOpen() to open the proxy file. The flags passed to unixOpen() - ** suggest that the file being opened is a "main database". This is - ** necessary as other file types do not necessarily support locking. It - ** is better to use unixOpen() instead of opening the file directly with - ** open(), as unixOpen() sets up the various mechanisms required to - ** make sure a call to close() does not cause the system to discard - ** POSIX locks prematurely. - ** - ** It is important that the xOpen member of the VFS object passed to - ** unixOpen() is NULL. This tells unixOpen() may try to open a proxy-file - ** for the proxy-file (creating a potential infinite loop). - */ + pNew->openFlags = openFlags; dummyVfs.pAppData = (void*)&autolockIoFinder; - dummyVfs.xOpen = 0; - rc = unixOpen(&dummyVfs, path, (sqlite3_file *)pNew, flags, &flags); - if( rc==SQLITE_OK && (flags&SQLITE_OPEN_READONLY) ){ - pNew->pMethod->xClose((sqlite3_file *)pNew); - rc = SQLITE_CANTOPEN; + pUnused->fd = fd; + pUnused->flags = openFlags; + pNew->pUnused = pUnused; + + rc = fillInUnixFile(&dummyVfs, fd, dirfd, (sqlite3_file*)pNew, path, 0, 0); + if( rc==SQLITE_OK ){ + *ppFile = pNew; + return SQLITE_OK; } - - if( rc!=SQLITE_OK ){ - sqlite3_free(pNew); - pNew = 0; - } - - *ppFile = pNew; +end_create_proxy: + close(fd); /* silently leak fd if error, we're already in error */ + sqlite3_free(pNew); + sqlite3_free(pUnused); return rc; } -/* takes the conch by taking a shared lock and read the contents conch, if +#ifdef SQLITE_TEST +/* simulate multiple hosts by creating unique hostid file paths */ +int sqlite3_hostid_num = 0; +#endif + +#define PROXY_HOSTIDLEN 16 /* conch file host id length */ + +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +** bytes of writable memory. +*/ +static int proxyGetHostID(unsigned char *pHostID, int *pError){ + struct timespec timeout = {1, 0}; /* 1 sec timeout */ + + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memset(pHostID, 0, PROXY_HOSTIDLEN); + if( gethostuuid(pHostID, &timeout) ){ + int err = errno; + if( pError ){ + *pError = err; + } + return SQLITE_IOERR; + } +#ifdef SQLITE_TEST + /* simulate multiple hosts by creating unique hostid file paths */ + if( sqlite3_hostid_num != 0){ + pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); + } +#endif + + return SQLITE_OK; +} + +/* The conch file contains the header, host id and lock file path + */ +#define PROXY_CONCHVERSION 2 /* 1-byte header, 16-byte host id, path */ +#define PROXY_HEADERLEN 1 /* conch file header length */ +#define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) +#define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) + +/* +** Takes an open conch file, copies the contents to a new path and then moves +** it back. The newly created file's file descriptor is assigned to the +** conch file structure and finally the original conch file descriptor is +** closed. Returns zero if successful. +*/ +static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + char tPath[MAXPATHLEN]; + char buf[PROXY_MAXCONCHLEN]; + char *cPath = pCtx->conchFilePath; + size_t readLen = 0; + size_t pathLen = 0; + char errmsg[64] = ""; + int fd = -1; + int rc = -1; + + /* create a new path by replace the trailing '-conch' with '-break' */ + pathLen = strlcpy(tPath, cPath, MAXPATHLEN); + if( pathLen>MAXPATHLEN || pathLen<6 || + (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ + sprintf(errmsg, "path error (len %d)", (int)pathLen); + goto end_breaklock; + } + /* read the conch content */ + readLen = pread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); + if( readLenh); + conchFile->h = fd; + conchFile->openFlags = O_RDWR | O_CREAT; + +end_breaklock: + if( rc ){ + if( fd>=0 ){ + unlink(tPath); + close(fd); + } + fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); + } + return rc; +} + +/* Take the requested lock on the conch file and break a stale lock if the +** host id matches. +*/ +static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + int nTries = 0; + struct timespec conchModTime; + + do { + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + nTries ++; + if( rc==SQLITE_BUSY ){ + /* If the lock failed (busy): + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait + * 10 sec and try again + * 3rd try: break the lock unless the mod time has changed. + */ + struct stat buf; + if( fstat(conchFile->h, &buf) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_LOCK; + } + + if( nTries==1 ){ + conchModTime = buf.st_mtimespec; + usleep(500000); /* wait 0.5 sec and try the lock again*/ + continue; + } + + assert( nTries>1 ); + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ + return SQLITE_BUSY; + } + + if( nTries==2 ){ + char tBuf[PROXY_MAXCONCHLEN]; + int len = pread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + if( len<0 ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_LOCK; + } + if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ + /* don't break the lock if the host id doesn't match */ + if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ + return SQLITE_BUSY; + } + }else{ + /* don't break the lock on short read or a version mismatch */ + return SQLITE_BUSY; + } + usleep(10000000); /* wait 10 sec and try the lock again */ + continue; + } + + assert( nTries==3 ); + if( 0==proxyBreakConchLock(pFile, myHostID) ){ + rc = SQLITE_OK; + if( lockType==EXCLUSIVE_LOCK ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + } + if( !rc ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + } + } + } + } while( rc==SQLITE_BUSY && nTries<3 ); + + return rc; +} + +/* Takes the conch by taking a shared lock and read the contents conch, if ** lockPath is non-NULL, the host ID and lock file path must match. A NULL ** lockPath means that the lockPath in the conch file will be used if the ** host IDs match, or a new lock path will be generated automatically @@ -4765,149 +5080,219 @@ static int proxyCreateUnixFile(const char *path, unixFile **ppFile) { static int proxyTakeConch(unixFile *pFile){ proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - if( pCtx->conchHeld>0 ){ + if( pCtx->conchHeld!=0 ){ return SQLITE_OK; }else{ unixFile *conchFile = pCtx->conchFile; - char testValue[CONCHLEN]; - char conchValue[CONCHLEN]; + uuid_t myHostID; + int pError = 0; + char readBuf[PROXY_MAXCONCHLEN]; char lockPath[MAXPATHLEN]; - char *tLockPath = NULL; + char *tempLockPath = NULL; int rc = SQLITE_OK; - int readRc = SQLITE_OK; - int syncPerms = 0; - + int createConch = 0; + int hostIdMatch = 0; + int readLen = 0; + int tryOldLockPath = 0; + int forceNewLockPath = 0; + OSTRACE4("TAKECONCH %d for %s pid=%d\n", conchFile->h, (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid()); - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); - if( rc==SQLITE_OK ){ - int pError = 0; - memset(testValue, 0, CONCHLEN); /* conch is fixed size */ - rc = proxyGetHostID(testValue, &pError); - if( (rc&0xff)==SQLITE_IOERR ){ - pFile->lastErrno = pError; - } - if( pCtx->lockProxyPath ){ - strlcpy(&testValue[HOSTIDLEN], pCtx->lockProxyPath, MAXPATHLEN); - } + rc = proxyGetHostID(myHostID, &pError); + if( (rc&0xff)==SQLITE_IOERR ){ + pFile->lastErrno = pError; + goto end_takeconch; } + rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); if( rc!=SQLITE_OK ){ goto end_takeconch; } - - readRc = unixRead((sqlite3_file *)conchFile, conchValue, CONCHLEN, 0); - if( readRc!=SQLITE_IOERR_SHORT_READ ){ - if( readRc!=SQLITE_OK ){ - if( (rc&0xff)==SQLITE_IOERR ){ - pFile->lastErrno = conchFile->lastErrno; + /* read the existing conch file */ + readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); + if( readLen<0 ){ + /* I/O error: lastErrno set by seekAndRead */ + pFile->lastErrno = conchFile->lastErrno; + rc = SQLITE_IOERR_READ; + goto end_takeconch; + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + readBuf[0]!=(char)PROXY_CONCHVERSION ){ + /* a short read or version format mismatch means we need to create a new + ** conch file. + */ + createConch = 1; + } + /* if the host id matches and the lock path already exists in the conch + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path + */ + do { /* in case we need to try again for an :auto: named lock file */ + + if( !createConch && !forceNewLockPath ){ + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + PROXY_HOSTIDLEN); + /* if the conch has data compare the contents */ + if( !pCtx->lockProxyPath ){ + /* for auto-named local lock file, just check the host ID and we'll + ** use the local lock file path that's already in there + */ + if( hostIdMatch ){ + size_t pathLen = (readLen - PROXY_PATHINDEX); + + if( pathLen>=MAXPATHLEN ){ + pathLen=MAXPATHLEN-1; + } + memcpy(lockPath, &readBuf[PROXY_PATHINDEX], pathLen); + lockPath[pathLen] = 0; + tempLockPath = lockPath; + tryOldLockPath = 1; + /* create a copy of the lock path if the conch is taken */ + goto end_takeconch; + } + }else if( hostIdMatch + && !strncmp(pCtx->lockProxyPath, &readBuf[PROXY_PATHINDEX], + readLen-PROXY_PATHINDEX) + ){ + /* conch host and lock path match */ + goto end_takeconch; } - rc = readRc; + } + + /* if the conch isn't writable and doesn't match, we can't take it */ + if( (conchFile->openFlags&O_RDWR) == 0 ){ + rc = SQLITE_BUSY; goto end_takeconch; } - /* if the conch has data compare the contents */ + + /* either the conch didn't match or we need to create a new one */ if( !pCtx->lockProxyPath ){ - /* for auto-named local lock file, just check the host ID and we'll - ** use the local lock file path that's already in there */ - if( !memcmp(testValue, conchValue, HOSTIDLEN) ){ - tLockPath = (char *)&conchValue[HOSTIDLEN]; - goto end_takeconch; - } - }else{ - /* we've got the conch if conchValue matches our path and host ID */ - if( !memcmp(testValue, conchValue, CONCHLEN) ){ - goto end_takeconch; - } + proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); + tempLockPath = lockPath; + /* create a copy of the lock path _only_ if the conch is taken */ } - }else{ - /* a short read means we're "creating" the conch (even though it could - ** have been user-intervention), if we acquire the exclusive lock, - ** we'll try to match the current on-disk permissions of the database + + /* update conch with host and path (this will fail if other process + ** has a shared lock already), if the host id matches, use the big + ** stick. */ - syncPerms = 1; - } - - /* either conch was emtpy or didn't match */ - if( !pCtx->lockProxyPath ){ - proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); - tLockPath = lockPath; - strlcpy(&testValue[HOSTIDLEN], lockPath, MAXPATHLEN); - } - - /* update conch with host and path (this will fail if other process - ** has a shared lock already) */ - rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); - if( rc==SQLITE_OK ){ - rc = unixWrite((sqlite3_file *)conchFile, testValue, CONCHLEN, 0); - if( rc==SQLITE_OK && syncPerms ){ - struct stat buf; - int err = fstat(pFile->h, &buf); - if( err==0 ){ - /* try to match the database file permissions, ignore failure */ -#ifndef SQLITE_PROXY_DEBUG - fchmod(conchFile->h, buf.st_mode); -#else - if( fchmod(conchFile->h, buf.st_mode)!=0 ){ - int code = errno; - fprintf(stderr, "fchmod %o FAILED with %d %s\n", - buf.st_mode, code, strerror(code)); - } else { - fprintf(stderr, "fchmod %o SUCCEDED\n",buf.st_mode); - } - }else{ - int code = errno; - fprintf(stderr, "STAT FAILED[%d] with %d %s\n", - err, code, strerror(code)); -#endif + futimes(conchFile->h, NULL); + if( hostIdMatch && !createConch ){ + if( conchFile->pLock && conchFile->pLock->cnt>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + } else { + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } - } - } - conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); - -end_takeconch: - OSTRACE2("TRANSPROXY: CLOSE %d\n", pFile->h); - if( rc==SQLITE_OK && pFile->openFlags ){ - if( pFile->h>=0 ){ -#ifdef STRICT_CLOSE_ERROR - if( close(pFile->h) ){ - pFile->lastErrno = errno; - return SQLITE_IOERR_CLOSE; - } -#else - close(pFile->h); /* silently leak fd if fail */ -#endif - } - pFile->h = -1; - int fd = open(pCtx->dbPath, pFile->openFlags, - SQLITE_DEFAULT_FILE_PERMISSIONS); - OSTRACE2("TRANSPROXY: OPEN %d\n", fd); - if( fd>=0 ){ - pFile->h = fd; }else{ - rc=SQLITE_CANTOPEN; /* SQLITE_BUSY? proxyTakeConch called - during locking */ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, EXCLUSIVE_LOCK); } - } - if( rc==SQLITE_OK && !pCtx->lockProxy ){ - char *path = tLockPath ? tLockPath : pCtx->lockProxyPath; - /* ACS: Need to make a copy of path sometimes */ - rc = proxyCreateUnixFile(path, &pCtx->lockProxy); - } - if( rc==SQLITE_OK ){ - pCtx->conchHeld = 1; - - if( tLockPath ){ - pCtx->lockProxyPath = sqlite3DbStrDup(0, tLockPath); - if( pCtx->lockProxy->pMethod == &afpIoMethods ){ - ((afpLockingContext *)pCtx->lockProxy->lockingContext)->dbPath = - pCtx->lockProxyPath; + if( rc==SQLITE_OK ){ + char writeBuffer[PROXY_MAXCONCHLEN]; + int writeSize = 0; + + writeBuffer[0] = (char)PROXY_CONCHVERSION; + memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); + if( pCtx->lockProxyPath!=NULL ){ + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); + }else{ + strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); + } + writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); + ftruncate(conchFile->h, writeSize); + rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); + fsync(conchFile->h); + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database + */ + if( rc==SQLITE_OK && createConch ){ + struct stat buf; + int err = fstat(pFile->h, &buf); + if( err==0 ){ + mode_t cmode = buf.st_mode&(S_IRUSR|S_IWUSR | S_IRGRP|S_IWGRP | + S_IROTH|S_IWOTH); + /* try to match the database file R/W permissions, ignore failure */ +#ifndef SQLITE_PROXY_DEBUG + fchmod(conchFile->h, cmode); +#else + if( fchmod(conchFile->h, cmode)!=0 ){ + int code = errno; + fprintf(stderr, "fchmod %o FAILED with %d %s\n", + cmode, code, strerror(code)); + } else { + fprintf(stderr, "fchmod %o SUCCEDED\n",cmode); + } + }else{ + int code = errno; + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + err, code, strerror(code)); +#endif + } } } - } else { - conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); - } - OSTRACE3("TAKECONCH %d %s\n", conchFile->h, rc==SQLITE_OK?"ok":"failed"); - return rc; + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); + + end_takeconch: + OSTRACE2("TRANSPROXY: CLOSE %d\n", pFile->h); + if( rc==SQLITE_OK && pFile->openFlags ){ + if( pFile->h>=0 ){ +#ifdef STRICT_CLOSE_ERROR + if( close(pFile->h) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_CLOSE; + } +#else + close(pFile->h); /* silently leak fd if fail */ +#endif + } + pFile->h = -1; + int fd = open(pCtx->dbPath, pFile->openFlags, + SQLITE_DEFAULT_FILE_PERMISSIONS); + OSTRACE2("TRANSPROXY: OPEN %d\n", fd); + if( fd>=0 ){ + pFile->h = fd; + }else{ + rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called + during locking */ + } + } + if( rc==SQLITE_OK && !pCtx->lockProxy ){ + char *path = tempLockPath ? tempLockPath : pCtx->lockProxyPath; + rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ + /* we couldn't create the proxy lock file with the old lock file path + ** so try again via auto-naming + */ + forceNewLockPath = 1; + tryOldLockPath = 0; + continue; /* go back to the do {} while start point, try again */ + } + } + if( rc==SQLITE_OK ){ + /* Need to make a copy of path if we extracted the value + ** from the conch file or the path was allocated on the stack + */ + if( tempLockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); + if( !pCtx->lockProxyPath ){ + rc = SQLITE_NOMEM; + } + } + } + if( rc==SQLITE_OK ){ + pCtx->conchHeld = 1; + + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ + afpLockingContext *afpCtx; + afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; + afpCtx->dbPath = pCtx->lockProxyPath; + } + } else { + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + OSTRACE3("TAKECONCH %d %s\n", conchFile->h, rc==SQLITE_OK?"ok":"failed"); + return rc; + } while (1); /* in case we need to retry the :auto: lock file - we should never get here except via the 'continue' call. */ } } @@ -4924,8 +5309,10 @@ static int proxyReleaseConch(unixFile *pFile){ OSTRACE4("RELEASECONCH %d for %s pid=%d\n", conchFile->h, (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), getpid()); + if( pCtx->conchHeld>0 ){ + rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } pCtx->conchHeld = 0; - rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); OSTRACE3("RELEASECONCH %d %s\n", conchFile->h, (rc==SQLITE_OK ? "ok" : "failed")); return rc; @@ -5021,8 +5408,8 @@ static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ /* afp style keeps a reference to the db path in the filePath field ** of the struct */ assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); - strcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath); - }else + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, MAXPATHLEN); + } else #endif if( pFile->pMethod == &dotlockIoMethods ){ /* dot lock style uses the locking context to store the dot lock @@ -5032,7 +5419,7 @@ static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ }else{ /* all other styles use the locking context to store the db file path */ assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); - strcpy(dbPath, (char *)pFile->lockingContext); + strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); } return SQLITE_OK; } @@ -5072,27 +5459,53 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); if( rc==SQLITE_OK ){ - rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile); + rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile, 0); + if( rc==SQLITE_CANTOPEN && ((pFile->openFlags&O_RDWR) == 0) ){ + /* if (a) the open flags are not O_RDWR, (b) the conch isn't there, and + ** (c) the file system is read-only, then enable no-locking access. + ** Ugh, since O_RDONLY==0x0000 we test for !O_RDWR since unixOpen asserts + ** that openFlags will have only one of O_RDONLY or O_RDWR. + */ + struct statfs fsInfo; + struct stat conchInfo; + int goLockless = 0; + + if( stat(pCtx->conchFilePath, &conchInfo) == -1 ) { + int err = errno; + if( (err==ENOENT) && (statfs(dbPath, &fsInfo) != -1) ){ + goLockless = (fsInfo.f_flags&MNT_RDONLY) == MNT_RDONLY; + } + } + if( goLockless ){ + pCtx->conchHeld = -1; /* read only FS/ lockless */ + rc = SQLITE_OK; + } + } } if( rc==SQLITE_OK && lockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); } + if( rc==SQLITE_OK ){ + pCtx->dbPath = sqlite3DbStrDup(0, dbPath); + if( pCtx->dbPath==NULL ){ + rc = SQLITE_NOMEM; + } + } if( rc==SQLITE_OK ){ /* all memory is allocated, proxys are created and assigned, ** switch the locking context and pMethod then return. */ - pCtx->dbPath = sqlite3DbStrDup(0, dbPath); pCtx->oldLockingContext = pFile->lockingContext; pFile->lockingContext = pCtx; pCtx->pOldMethod = pFile->pMethod; pFile->pMethod = &proxyIoMethods; }else{ if( pCtx->conchFile ){ - rc = pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); - if( rc ) return rc; + pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); sqlite3_free(pCtx->conchFile); } + sqlite3_free(pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); sqlite3_free(pCtx); } @@ -5181,8 +5594,12 @@ static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { int rc = proxyTakeConch(pFile); if( rc==SQLITE_OK ){ proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - unixFile *proxy = pCtx->lockProxy; - return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + }else{ /* conchHeld < 0 is lockless */ + pResOut=0; + } } return rc; } @@ -5216,9 +5633,13 @@ static int proxyLock(sqlite3_file *id, int locktype) { int rc = proxyTakeConch(pFile); if( rc==SQLITE_OK ){ proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - unixFile *proxy = pCtx->lockProxy; - rc = proxy->pMethod->xLock((sqlite3_file*)proxy, locktype); - pFile->locktype = proxy->locktype; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xLock((sqlite3_file*)proxy, locktype); + pFile->locktype = proxy->locktype; + }else{ + /* conchHeld < 0 is lockless */ + } } return rc; } @@ -5236,9 +5657,13 @@ static int proxyUnlock(sqlite3_file *id, int locktype) { int rc = proxyTakeConch(pFile); if( rc==SQLITE_OK ){ proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - unixFile *proxy = pCtx->lockProxy; - rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, locktype); - pFile->locktype = proxy->locktype; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, locktype); + pFile->locktype = proxy->locktype; + }else{ + /* conchHeld < 0 is lockless */ + } } return rc; } @@ -5365,7 +5790,6 @@ int sqlite3_os_init(void){ #endif UNIXVFS("unix-none", nolockIoFinder ), UNIXVFS("unix-dotfile", dotlockIoFinder ), - UNIXVFS("unix-wfl", posixWflIoFinder ), #if OS_VXWORKS UNIXVFS("unix-namedsem", semIoFinder ), #endif @@ -5377,6 +5801,7 @@ int sqlite3_os_init(void){ #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) UNIXVFS("unix-afp", afpIoFinder ), + UNIXVFS("unix-nfs", nfsIoFinder ), UNIXVFS("unix-proxy", proxyIoFinder ), #endif }; diff --git a/src/os_win.c b/src/os_win.c index bd23a848..4721eadf 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1417,7 +1417,7 @@ static int winOpen( return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY)&~SQLITE_OPEN_READWRITE), pOutFlags); }else{ - return SQLITE_CANTOPEN; + return SQLITE_CANTOPEN_BKPT; } } if( pOutFlags ){ @@ -1439,7 +1439,7 @@ static int winOpen( ){ CloseHandle(h); free(zConverted); - return SQLITE_CANTOPEN; + return SQLITE_CANTOPEN_BKPT; } if( isTemp ){ pFile->zDeleteOnClose = zConverted; diff --git a/src/pager.c b/src/pager.c index 1c41758d..c33ceeac 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2019,6 +2019,9 @@ end_playback: rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } + if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ + rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + } if( rc==SQLITE_OK ){ rc = pager_end_transaction(pPager, zMaster[0]!='\0'); testcase( rc!=SQLITE_OK ); @@ -2875,9 +2878,7 @@ static int pager_write_pagelist(PgHdr *pList){ ** any such pages to the file. ** ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag - ** set (set by sqlite3PagerDontWrite()). Note that if compiled with - ** SQLITE_SECURE_DELETE the PGHDR_DONT_WRITE bit is never set and so - ** the second test is always true. + ** set (set by sqlite3PagerDontWrite()). */ if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ @@ -3164,7 +3165,7 @@ int sqlite3PagerOpen( ** as it will not be possible to open the journal file or even ** check for a hot-journal before reading. */ - rc = SQLITE_CANTOPEN; + rc = SQLITE_CANTOPEN_BKPT; } if( rc!=SQLITE_OK ){ sqlite3_free(zPathname); @@ -3341,6 +3342,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + *ppPager = pPager; return SQLITE_OK; } @@ -3490,8 +3492,24 @@ static int readDbPage(PgHdr *pPg){ rc = SQLITE_OK; } if( pgno==1 ){ - u8 *dbFileVers = &((u8*)pPg->pData)[24]; - memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + if( rc ){ + /* If the read is unsuccessful, set the dbFileVers[] to something + ** that will never be a valid file version. dbFileVers[] is a copy + ** of bytes 24..39 of the database. Bytes 28..31 should always be + ** zero. Bytes 32..35 and 35..39 should be page numbers which are + ** never 0xffffffff. So filling pPager->dbFileVers[] with all 0xff + ** bytes should suffice. + ** + ** For an encrypted database, the situation is more complex: bytes + ** 24..39 of the database are white noise. But the probability of + ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** we should still be ok. + */ + memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); + }else{ + u8 *dbFileVers = &((u8*)pPg->pData)[24]; + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } } CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); @@ -3623,7 +3641,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ - rc = SQLITE_CANTOPEN; + rc = SQLITE_CANTOPEN_BKPT; sqlite3OsClose(pPager->jfd); } }else{ @@ -3842,7 +3860,7 @@ int sqlite3PagerAcquire( goto pager_acquire_err; } - if( MEMDB || nMax<(int)pgno || noContent ){ + if( MEMDB || nMax<(int)pgno || noContent || !isOpen(pPager->fd) ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -3862,9 +3880,8 @@ int sqlite3PagerAcquire( TESTONLY( rc = ) addToSavepointBitvecs(pPager, pgno); testcase( rc==SQLITE_NOMEM ); sqlite3EndBenignMalloc(); - }else{ - memset(pPg->pData, 0, pPager->pageSize); } + memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ assert( pPg->pPager==pPager ); @@ -4386,7 +4403,6 @@ int sqlite3PagerIswriteable(DbPage *pPg){ } #endif -#ifndef SQLITE_SECURE_DELETE /* ** A call to this routine tells the pager that it is not necessary to ** write the information on page pPg back to the disk, even though @@ -4412,7 +4428,6 @@ void sqlite3PagerDontWrite(PgHdr *pPg){ #endif } } -#endif /* !defined(SQLITE_SECURE_DELETE) */ /* ** This routine is called to increment the value of the database file @@ -4982,30 +4997,35 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ ** operation. Store this value in nNew. Then free resources associated ** with any savepoints that are destroyed by this operation. */ - nNew = iSavepoint + (op==SAVEPOINT_ROLLBACK); + nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); for(ii=nNew; iinSavepoint; ii++){ sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); } pPager->nSavepoint = nNew; - /* If this is a rollback operation, playback the specified savepoint. + /* If this is a release of the outermost savepoint, truncate + ** the sub-journal to zero bytes in size. */ + if( op==SAVEPOINT_RELEASE ){ + if( nNew==0 && isOpen(pPager->sjfd) ){ + /* Only truncate if it is an in-memory sub-journal. */ + if( sqlite3IsMemJournal(pPager->sjfd) ){ + rc = sqlite3OsTruncate(pPager->sjfd, 0); + assert( rc==SQLITE_OK ); + } + pPager->nSubRec = 0; + } + } + /* Else this is a rollback operation, playback the specified savepoint. ** If this is a temp-file, it is possible that the journal file has ** not yet been opened. In this case there have been no changes to ** the database file, so the playback operation can be skipped. */ - if( op==SAVEPOINT_ROLLBACK && isOpen(pPager->jfd) ){ + else if( isOpen(pPager->jfd) ){ PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - /* If this is a release of the outermost savepoint, truncate - ** the sub-journal to zero bytes in size. */ - if( nNew==0 && op==SAVEPOINT_RELEASE && isOpen(pPager->sjfd) ){ - assert( rc==SQLITE_OK ); - rc = sqlite3OsTruncate(pPager->sjfd, 0); - pPager->nSubRec = 0; - } } return rc; } diff --git a/src/parse.y b/src/parse.y index f1c4216d..98714ee3 100644 --- a/src/parse.y +++ b/src/parse.y @@ -316,6 +316,7 @@ refargs(A) ::= . { A = OE_None*0x0101; /* EV: R-19803-45884 */} refargs(A) ::= refargs(X) refarg(Y). { A = (X & ~Y.mask) | Y.value; } %type refarg {struct {int value; int mask;}} refarg(A) ::= MATCH nm. { A.value = 0; A.mask = 0x000000; } +refarg(A) ::= ON INSERT refact. { A.value = 0; A.mask = 0x000000; } refarg(A) ::= ON DELETE refact(X). { A.value = X; A.mask = 0x0000ff; } refarg(A) ::= ON UPDATE refact(X). { A.value = X<<8; A.mask = 0x00ff00; } %type refact {int} diff --git a/src/pcache.c b/src/pcache.c index bd46e54f..41536e8b 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -189,6 +189,7 @@ void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ if( pCache->pCache ){ sqlite3GlobalConfig.pcache.xDestroy(pCache->pCache); pCache->pCache = 0; + pCache->pPage1 = 0; } pCache->szPage = szPage; } @@ -242,6 +243,7 @@ int sqlite3PcacheFetch( pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); pPg=pPg->pDirtyPrev ); + pCache->pSynced = pPg; if( !pPg ){ for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); } diff --git a/src/pragma.c b/src/pragma.c index 2ed46d9d..d5413734 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -285,6 +285,7 @@ void sqlite3Pragma( Db *pDb; Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); if( v==0 ) return; + sqlite3VdbeRunOnlyOnce(v); pParse->nMem = 2; /* Interpret the [database.] part of the pragma statement. iDb is the @@ -416,6 +417,31 @@ void sqlite3Pragma( returnSingleInt(pParse, "max_page_count", newMax); }else + /* + ** PRAGMA [database.]secure_delete + ** PRAGMA [database.]secure_delete=ON/OFF + ** + ** The first form reports the current setting for the + ** secure_delete flag. The second form changes the secure_delete + ** flag setting and reports thenew value. + */ + if( sqlite3StrICmp(zLeft,"secure_delete")==0 ){ + Btree *pBt = pDb->pBt; + int b = -1; + assert( pBt!=0 ); + if( zRight ){ + b = getBoolean(zRight); + } + if( pId2->n==0 && b>=0 ){ + int ii; + for(ii=0; iinDb; ii++){ + sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); + } + } + b = sqlite3BtreeSecureDelete(pBt, b); + returnSingleInt(pParse, "secure_delete", b); + }else + /* ** PRAGMA [database.]page_count ** @@ -1334,6 +1360,26 @@ void sqlite3Pragma( }else #endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + /* + ** PRAGMA compile_options + ** + ** Return the names of all compile-time options used in this build, + ** one option per row. + */ + if( sqlite3StrICmp(zLeft, "compile_options")==0 ){ + int i = 0; + const char *zOpt; + sqlite3VdbeSetNumCols(v, 1); + pParse->nMem = 1; + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "compile_option", SQLITE_STATIC); + while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ + sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, zOpt, 0); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + }else +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases @@ -1368,7 +1414,7 @@ void sqlite3Pragma( }else #endif -#if SQLITE_HAS_CODEC +#ifdef SQLITE_HAS_CODEC if( sqlite3StrICmp(zLeft, "key")==0 && zRight ){ sqlite3_key(db, zRight, sqlite3Strlen30(zRight)); }else @@ -1409,17 +1455,15 @@ void sqlite3Pragma( }else /** END CRYPTO **/ #endif -#if SQLITE_HAS_CODEC || defined(SQLITE_ENABLE_CEROD) +#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) if( sqlite3StrICmp(zLeft, "activate_extensions")==0 ){ -#if SQLITE_HAS_CODEC +#ifdef SQLITE_HAS_CODEC if( sqlite3StrNICmp(zRight, "see-", 4)==0 ){ - extern void sqlite3_activate_see(const char*); sqlite3_activate_see(&zRight[4]); } #endif #ifdef SQLITE_ENABLE_CEROD if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){ - extern void sqlite3_activate_cerod(const char*); sqlite3_activate_cerod(&zRight[6]); } #endif @@ -1429,12 +1473,6 @@ void sqlite3Pragma( {/* Empty ELSE clause */} - /* Code an OP_Expire at the end of each PRAGMA program to cause - ** the VDBE implementing the pragma to expire. Most (all?) pragmas - ** are only valid for a single execution. - */ - sqlite3VdbeAddOp2(v, OP_Expire, 1, 0); - /* ** Reset the safety level, in case the fullfsync flag or synchronous ** setting changed. diff --git a/src/prepare.c b/src/prepare.c index 0789ca50..f1b1e005 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -192,9 +192,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; - (void)sqlite3SafetyOff(db); sqlite3InitCallback(&initData, 3, (char **)azArg, 0); - (void)sqlite3SafetyOn(db); if( initData.rc ){ rc = initData.rc; goto error_out; @@ -315,9 +313,8 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s", + "SELECT name, rootpage, sql FROM '%q'.%s ORDER BY rowid", db->aDb[iDb].zName, zMasterName); - (void)sqlite3SafetyOff(db); #ifndef SQLITE_OMIT_AUTHORIZATION { int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); @@ -330,7 +327,6 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ } #endif if( rc==SQLITE_OK ) rc = initData.rc; - (void)sqlite3SafetyOn(db); sqlite3DbFree(db, zSql); #ifndef SQLITE_OMIT_ANALYZE if( rc==SQLITE_OK ){ @@ -469,7 +465,7 @@ static void schemaIsValid(Parse *pParse){ } /* Read the schema cookie from the database. If it does not match the - ** value stored as part of the in the in-memory schema representation, + ** value stored as part of the in-memory schema representation, ** set Parse.rc to SQLITE_SCHEMA. */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ @@ -539,11 +535,6 @@ static int sqlite3Prepare( goto end_prepare; } pParse->pReprepare = pReprepare; - - if( sqlite3SafetyOn(db) ){ - rc = SQLITE_MISUSE; - goto end_prepare; - } assert( ppStmt && *ppStmt==0 ); assert( !db->mallocFailed ); assert( sqlite3_mutex_held(db->mutex) ); @@ -579,7 +570,6 @@ static int sqlite3Prepare( if( rc ){ const char *zDb = db->aDb[i].zName; sqlite3Error(db, rc, "database schema is locked: %s", zDb); - (void)sqlite3SafetyOff(db); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; } @@ -596,7 +586,6 @@ static int sqlite3Prepare( testcase( nBytes==mxLen+1 ); if( nBytes>mxLen ){ sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); - (void)sqlite3SafetyOff(db); rc = sqlite3ApiExit(db, SQLITE_TOOBIG); goto end_prepare; } @@ -653,10 +642,6 @@ static int sqlite3Prepare( } #endif - if( sqlite3SafetyOff(db) ){ - rc = SQLITE_MISUSE; - } - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; @@ -704,7 +689,7 @@ static int sqlite3LockAndPrepare( assert( ppStmt!=0 ); *ppStmt = 0; if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); @@ -743,7 +728,7 @@ int sqlite3Reprepare(Vdbe *p){ db->mallocFailed = 1; } assert( pNew==0 ); - return (rc==SQLITE_LOCKED) ? SQLITE_LOCKED : SQLITE_SCHEMA; + return rc; }else{ assert( pNew!=0 ); } @@ -812,10 +797,10 @@ static int sqlite3Prepare16( assert( ppStmt ); *ppStmt = 0; if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); - zSql8 = sqlite3Utf16to8(db, zSql, nBytes); + zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, 0, ppStmt, &zTail8); } diff --git a/src/printf.c b/src/printf.c index 17edb090..6ab230b8 100644 --- a/src/printf.c +++ b/src/printf.c @@ -460,7 +460,9 @@ void sqlite3VXPrintf( case etEXP: case etGENERIC: realvalue = va_arg(ap,double); -#ifndef SQLITE_OMIT_FLOATING_POINT +#ifdef SQLITE_OMIT_FLOATING_POINT + length = 0; +#else if( precision<0 ) precision = 6; /* Set default precision */ if( precision>etBUFSIZE/2-10 ) precision = etBUFSIZE/2-10; if( realvalue<0.0 ){ @@ -606,7 +608,7 @@ void sqlite3VXPrintf( while( nPad-- ) bufpt[i++] = '0'; length = width; } -#endif +#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ break; case etSIZE: *(va_arg(ap,int*)) = pAccum->nChar; @@ -653,7 +655,7 @@ void sqlite3VXPrintf( isnull = escarg==0; if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); k = precision; - for(i=n=0; (ch=escarg[i])!=0 && k!=0; i++, k--){ + for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){ if( ch==q ) n++; } needQuote = !isnull && xtype==etSQLESCAPE2; @@ -937,6 +939,38 @@ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ return z; } +/* +** This is the routine that actually formats the sqlite3_log() message. +** We house it in a separate routine from sqlite3_log() to avoid using +** stack space on small-stack systems when logging is disabled. +** +** sqlite3_log() must render into a static buffer. It cannot dynamically +** allocate memory because it might be called while the memory allocator +** mutex is held. +*/ +static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ + StrAccum acc; /* String accumulator */ + char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + + sqlite3StrAccumInit(&acc, zMsg, sizeof(zMsg), 0); + acc.useMalloc = 0; + sqlite3VXPrintf(&acc, 0, zFormat, ap); + sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, + sqlite3StrAccumFinish(&acc)); +} + +/* +** Format and write a message to the log if logging is enabled. +*/ +void sqlite3_log(int iErrCode, const char *zFormat, ...){ + va_list ap; /* Vararg list */ + if( sqlite3GlobalConfig.xLog ){ + va_start(ap, zFormat); + renderLogMsg(iErrCode, zFormat, ap); + va_end(ap); + } +} + #if defined(SQLITE_DEBUG) /* ** A version of printf() that understands %lld. Used for debugging. diff --git a/src/resolve.c b/src/resolve.c index 0a59be7c..3a44aef6 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -243,19 +243,18 @@ static int lookupName( int iCol; pSchema = pTab->pSchema; cntTab++; - if( sqlite3IsRowid(zCol) ){ - iCol = -1; - }else{ - for(iCol=0; iColnCol; iCol++){ - Column *pCol = &pTab->aCol[iCol]; - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - if( iCol==pTab->iPKey ){ - iCol = -1; - } - break; + for(iCol=0; iColnCol; iCol++){ + Column *pCol = &pTab->aCol[iCol]; + if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( iCol==pTab->iPKey ){ + iCol = -1; } + break; } } + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) ){ + iCol = -1; /* IMP: R-44911-55124 */ + } if( iColnCol ){ cnt++; if( iCol<0 ){ @@ -282,7 +281,7 @@ static int lookupName( */ if( cnt==0 && cntTab==1 && sqlite3IsRowid(zCol) ){ cnt = 1; - pExpr->iColumn = -1; + pExpr->iColumn = -1; /* IMP: R-44911-55124 */ pExpr->affinity = SQLITE_AFF_INTEGER; } @@ -665,6 +664,9 @@ static int resolveOrderByTermToExprList( int i; /* Loop counter */ ExprList *pEList; /* The columns of the result set */ NameContext nc; /* Name context for resolving pE */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code from subprocedures */ + u8 savedSuppErr; /* Saved value of db->suppressErr */ assert( sqlite3ExprIsInteger(pE, &i)==0 ); pEList = pSelect->pEList; @@ -677,17 +679,19 @@ static int resolveOrderByTermToExprList( nc.pEList = pEList; nc.allowAgg = 1; nc.nErr = 0; - if( sqlite3ResolveExprNames(&nc, pE) ){ - sqlite3ErrorClear(pParse); - return 0; - } + db = pParse->db; + savedSuppErr = db->suppressErr; + db->suppressErr = 1; + rc = sqlite3ResolveExprNames(&nc, pE); + db->suppressErr = savedSuppErr; + if( rc ) return 0; /* Try to match the ORDER BY expression against an expression ** in the result set. Return an 1-based index of the matching ** result-set entry. */ for(i=0; inExpr; i++){ - if( sqlite3ExprCompare(pEList->a[i].pExpr, pE) ){ + if( sqlite3ExprCompare(pEList->a[i].pExpr, pE)<2 ){ return i+1; } } diff --git a/src/select.c b/src/select.c index 7555a391..be4b95c0 100644 --- a/src/select.c +++ b/src/select.c @@ -189,6 +189,39 @@ static int columnIndex(Table *pTab, const char *zCol){ return -1; } +/* +** Search the first N tables in pSrc, from left to right, looking for a +** table that has a column named zCol. +** +** When found, set *piTab and *piCol to the table index and column index +** of the matching column and return TRUE. +** +** If not found, return FALSE. +*/ +static int tableAndColumnIndex( + SrcList *pSrc, /* Array of tables to search */ + int N, /* Number of tables in pSrc->a[] to search */ + const char *zCol, /* Name of the column we are looking for */ + int *piTab, /* Write index of pSrc->a[] here */ + int *piCol /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ +){ + int i; /* For looping over tables in pSrc */ + int iCol; /* Index of column matching zCol */ + + assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ + for(i=0; ia[i].pTab, zCol); + if( iCol>=0 ){ + if( piTab ){ + *piTab = i; + *piCol = iCol; + } + return 1; + } + } + return 0; +} + /* ** This function is used to add terms implied by JOIN syntax to the ** WHERE clause expression of a SELECT statement. The new term, which @@ -203,8 +236,9 @@ static int columnIndex(Table *pTab, const char *zCol){ static void addWhereTerm( Parse *pParse, /* Parsing context */ SrcList *pSrc, /* List of tables in FROM clause */ - int iSrc, /* Index of first table to join in pSrc */ + int iLeft, /* Index of first table to join in pSrc */ int iColLeft, /* Index of column in first table */ + int iRight, /* Index of second table in pSrc */ int iColRight, /* Index of column in second table */ int isOuterJoin, /* True if this is an OUTER join */ Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ @@ -214,12 +248,13 @@ static void addWhereTerm( Expr *pE2; Expr *pEq; - assert( pSrc->nSrc>(iSrc+1) ); - assert( pSrc->a[iSrc].pTab ); - assert( pSrc->a[iSrc+1].pTab ); + assert( iLeftnSrc>iRight ); + assert( pSrc->a[iLeft].pTab ); + assert( pSrc->a[iRight].pTab ); - pE1 = sqlite3CreateColumnExpr(db, pSrc, iSrc, iColLeft); - pE2 = sqlite3CreateColumnExpr(db, pSrc, iSrc+1, iColRight); + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); + pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); if( pEq && isOuterJoin ){ @@ -308,11 +343,15 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ "an ON or USING clause", 0); return 1; } - for(j=0; jnCol; j++){ - char *zName = pLeftTab->aCol[j].zName; - int iRightCol = columnIndex(pRightTab, zName); - if( iRightCol>=0 ){ - addWhereTerm(pParse, pSrc, i, j, iRightCol, isOuter, &p->pWhere); + for(j=0; jnCol; j++){ + char *zName; /* Name of column in the right table */ + int iLeft; /* Matching left table */ + int iLeftCol; /* Matching column in the left table */ + + zName = pRightTab->aCol[j].zName; + if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) ){ + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, + isOuter, &p->pWhere); } } } @@ -344,15 +383,22 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ if( pRight->pUsing ){ IdList *pList = pRight->pUsing; for(j=0; jnId; j++){ - char *zName = pList->a[j].zName; - int iLeftCol = columnIndex(pLeftTab, zName); - int iRightCol = columnIndex(pRightTab, zName); - if( iLeftCol<0 || iRightCol<0 ){ + char *zName; /* Name of the term in the USING clause */ + int iLeft; /* Table on the left with matching column name */ + int iLeftCol; /* Column number of matching column on the left */ + int iRightCol; /* Column number of matching column on the right */ + + zName = pList->a[j].zName; + iRightCol = columnIndex(pRightTab, zName); + if( iRightCol<0 + || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) + ){ sqlite3ErrorMsg(pParse, "cannot join using column %s - column " "not present in both tables", zName); return 1; } - addWhereTerm(pParse, pSrc, i, iLeftCol, iRightCol, isOuter, &p->pWhere); + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, + isOuter, &p->pWhere); } } } @@ -911,7 +957,7 @@ static const char *columnType( ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ - if( ALWAYS(iCol>=0 && iColpEList->nExpr) ){ + if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ /* If iCol is less than zero, then the expression requests the ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. @@ -2472,7 +2518,7 @@ static void substSelect( ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** -** (12) Not implemented. Subsumed into restriction (3). Was previously +** (**) Not implemented. Subsumed into restriction (3). Was previously ** a separate restriction deriving from ticket #350. ** ** (13) The subquery and outer query do not both use LIMIT @@ -2546,6 +2592,7 @@ static int flattenSubquery( */ assert( p!=0 ); assert( p->pPrior==0 ); /* Unable to flatten compound queries */ + if( db->flags & SQLITE_QueryFlattener ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; @@ -3169,14 +3216,14 @@ static int selectExpander(Walker *pWalker, Select *p){ } if( i>0 && zTName==0 ){ - struct SrcList_item *pLeft = &pTabList->a[i-1]; - if( (pLeft[1].jointype & JT_NATURAL)!=0 && - columnIndex(pLeft->pTab, zName)>=0 ){ + if( (pFrom->jointype & JT_NATURAL)!=0 + && tableAndColumnIndex(pTabList, i, zName, 0, 0) + ){ /* In a NATURAL join, omit the join columns from the - ** table on the right */ + ** table to the right of the join */ continue; } - if( sqlite3IdListIndex(pLeft[1].pUsing, zName)>=0 ){ + if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ /* In a join with a USING clause, omit columns in the ** using clause from the table on the right. */ continue; @@ -3280,18 +3327,19 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - assert( (p->selFlags & SF_HasTypeInfo)==0 ); - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - assert( pSel ); - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); + if( (p->selFlags & SF_HasTypeInfo)==0 ){ + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + assert( pSel ); + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSel); + } } } return WRC_Continue; @@ -3443,8 +3491,8 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ sqlite3VdbeAddOp4(v, OP_AggStep, 0, regAgg, pF->iMem, (void*)pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ReleaseTempRange(pParse, regAgg, nArg); sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); sqlite3ExprCacheClear(pParse); @@ -3870,7 +3918,7 @@ int sqlite3Select( int r2; r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); + pCol->pTab, pCol->iColumn, pCol->iTable, r1); if( r1!=r2 ){ sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); } diff --git a/src/shell.c b/src/shell.c index 973fc5a4..a1c0b5e2 100644 --- a/src/shell.c +++ b/src/shell.c @@ -195,868 +195,6 @@ static void endTimer(void){ */ #define UNUSED_PARAMETER(x) (void)(x) - -/************************************************************************** -*************************************************************************** -** Begin genfkey logic. -*/ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined SQLITE_OMIT_SUBQUERY - -#define GENFKEY_ERROR 1 -#define GENFKEY_DROPTRIGGER 2 -#define GENFKEY_CREATETRIGGER 3 -static int genfkey_create_triggers(sqlite3 *, const char *, void *, - int (*)(void *, int, const char *) -); - -struct GenfkeyCb { - void *pCtx; - int eType; - int (*xData)(void *, int, const char *); -}; -typedef struct GenfkeyCb GenfkeyCb; - -/* The code in this file defines a sqlite3 virtual-table module that -** provides a read-only view of the current database schema. There is one -** row in the schema table for each column in the database schema. -*/ -#define SCHEMA \ -"CREATE TABLE x(" \ - "database," /* Name of database (i.e. main, temp etc.) */ \ - "tablename," /* Name of table */ \ - "cid," /* Column number (from left-to-right, 0 upward) */ \ - "name," /* Column name */ \ - "type," /* Specified type (i.e. VARCHAR(32)) */ \ - "not_null," /* Boolean. True if NOT NULL was specified */ \ - "dflt_value," /* Default value for this column */ \ - "pk" /* True if this column is part of the primary key */ \ -")" - -#define SCHEMA2 \ -"CREATE TABLE x(" \ - "database," /* Name of database (i.e. main, temp etc.) */ \ - "from_tbl," /* Name of table */ \ - "fkid," \ - "seq," \ - "to_tbl," \ - "from_col," \ - "to_col," \ - "on_update," \ - "on_delete," \ - "match" \ -")" - -#define SCHEMA3 \ -"CREATE TABLE x(" \ - "database," /* Name of database (i.e. main, temp etc.) */ \ - "tablename," /* Name of table */ \ - "seq," \ - "name," \ - "isunique" \ -")" - -#define SCHEMA4 \ -"CREATE TABLE x(" \ - "database," /* Name of database (i.e. main, temp etc.) */ \ - "indexname," /* Name of table */ \ - "seqno," \ - "cid," \ - "name" \ -")" - -#define SCHEMA5 \ -"CREATE TABLE x(" \ - "database," /* Name of database (i.e. main, temp etc.) */ \ - "triggername," /* Name of trigger */ \ - "dummy" /* Unused */ \ -")" - -typedef struct SchemaTable SchemaTable; -static struct SchemaTable { - const char *zName; - const char *zObject; - const char *zPragma; - const char *zSchema; -} aSchemaTable[] = { - { "table_info", "table", "PRAGMA %Q.table_info(%Q)", SCHEMA }, - { "foreign_key_list", "table", "PRAGMA %Q.foreign_key_list(%Q)", SCHEMA2 }, - { "index_list", "table", "PRAGMA %Q.index_list(%Q)", SCHEMA3 }, - { "index_info", "index", "PRAGMA %Q.index_info(%Q)", SCHEMA4 }, - { "trigger_list", "trigger", "SELECT 1", SCHEMA5 }, - { 0, 0, 0, 0 } -}; - -typedef struct schema_vtab schema_vtab; -typedef struct schema_cursor schema_cursor; - -/* A schema table object */ -struct schema_vtab { - sqlite3_vtab base; - sqlite3 *db; - SchemaTable *pType; -}; - -/* A schema table cursor object */ -struct schema_cursor { - sqlite3_vtab_cursor base; - sqlite3_stmt *pDbList; - sqlite3_stmt *pTableList; - sqlite3_stmt *pColumnList; - int rowid; -}; - -/* -** Table destructor for the schema module. -*/ -static int schemaDestroy(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); - return 0; -} - -/* -** Table constructor for the schema module. -*/ -static int schemaCreate( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - int rc = SQLITE_NOMEM; - schema_vtab *pVtab; - SchemaTable *pType = &aSchemaTable[0]; - - UNUSED_PARAMETER(pzErr); - if( argc>3 ){ - int i; - pType = 0; - for(i=0; aSchemaTable[i].zName; i++){ - if( 0==strcmp(argv[3], aSchemaTable[i].zName) ){ - pType = &aSchemaTable[i]; - } - } - if( !pType ){ - return SQLITE_ERROR; - } - } - - pVtab = sqlite3_malloc(sizeof(schema_vtab)); - if( pVtab ){ - memset(pVtab, 0, sizeof(schema_vtab)); - pVtab->db = (sqlite3 *)pAux; - pVtab->pType = pType; - rc = sqlite3_declare_vtab(db, pType->zSchema); - } - *ppVtab = (sqlite3_vtab *)pVtab; - return rc; -} - -/* -** Open a new cursor on the schema table. -*/ -static int schemaOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - int rc = SQLITE_NOMEM; - schema_cursor *pCur; - UNUSED_PARAMETER(pVTab); - pCur = sqlite3_malloc(sizeof(schema_cursor)); - if( pCur ){ - memset(pCur, 0, sizeof(schema_cursor)); - *ppCursor = (sqlite3_vtab_cursor *)pCur; - rc = SQLITE_OK; - } - return rc; -} - -/* -** Close a schema table cursor. -*/ -static int schemaClose(sqlite3_vtab_cursor *cur){ - schema_cursor *pCur = (schema_cursor *)cur; - sqlite3_finalize(pCur->pDbList); - sqlite3_finalize(pCur->pTableList); - sqlite3_finalize(pCur->pColumnList); - sqlite3_free(pCur); - return SQLITE_OK; -} - -static void columnToResult(sqlite3_context *ctx, sqlite3_stmt *pStmt, int iCol){ - switch( sqlite3_column_type(pStmt, iCol) ){ - case SQLITE_NULL: - sqlite3_result_null(ctx); - break; - case SQLITE_INTEGER: - sqlite3_result_int64(ctx, sqlite3_column_int64(pStmt, iCol)); - break; - case SQLITE_FLOAT: - sqlite3_result_double(ctx, sqlite3_column_double(pStmt, iCol)); - break; - case SQLITE_TEXT: { - const char *z = (const char *)sqlite3_column_text(pStmt, iCol); - sqlite3_result_text(ctx, z, -1, SQLITE_TRANSIENT); - break; - } - } -} - -/* -** Retrieve a column of data. -*/ -static int schemaColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ - schema_cursor *pCur = (schema_cursor *)cur; - switch( i ){ - case 0: - columnToResult(ctx, pCur->pDbList, 1); - break; - case 1: - columnToResult(ctx, pCur->pTableList, 0); - break; - default: - columnToResult(ctx, pCur->pColumnList, i-2); - break; - } - return SQLITE_OK; -} - -/* -** Retrieve the current rowid. -*/ -static int schemaRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - schema_cursor *pCur = (schema_cursor *)cur; - *pRowid = pCur->rowid; - return SQLITE_OK; -} - -static int finalize(sqlite3_stmt **ppStmt){ - int rc = sqlite3_finalize(*ppStmt); - *ppStmt = 0; - return rc; -} - -static int schemaEof(sqlite3_vtab_cursor *cur){ - schema_cursor *pCur = (schema_cursor *)cur; - return (pCur->pDbList ? 0 : 1); -} - -/* -** Advance the cursor to the next row. -*/ -static int schemaNext(sqlite3_vtab_cursor *cur){ - int rc = SQLITE_OK; - schema_cursor *pCur = (schema_cursor *)cur; - schema_vtab *pVtab = (schema_vtab *)(cur->pVtab); - char *zSql = 0; - - while( !pCur->pColumnList || SQLITE_ROW!=sqlite3_step(pCur->pColumnList) ){ - if( SQLITE_OK!=(rc = finalize(&pCur->pColumnList)) ) goto next_exit; - - while( !pCur->pTableList || SQLITE_ROW!=sqlite3_step(pCur->pTableList) ){ - if( SQLITE_OK!=(rc = finalize(&pCur->pTableList)) ) goto next_exit; - - assert(pCur->pDbList); - while( SQLITE_ROW!=sqlite3_step(pCur->pDbList) ){ - rc = finalize(&pCur->pDbList); - goto next_exit; - } - - /* Set zSql to the SQL to pull the list of tables from the - ** sqlite_master (or sqlite_temp_master) table of the database - ** identfied by the row pointed to by the SQL statement pCur->pDbList - ** (iterating through a "PRAGMA database_list;" statement). - */ - if( sqlite3_column_int(pCur->pDbList, 0)==1 ){ - zSql = sqlite3_mprintf( - "SELECT name FROM sqlite_temp_master WHERE type=%Q", - pVtab->pType->zObject - ); - }else{ - sqlite3_stmt *pDbList = pCur->pDbList; - zSql = sqlite3_mprintf( - "SELECT name FROM %Q.sqlite_master WHERE type=%Q", - sqlite3_column_text(pDbList, 1), pVtab->pType->zObject - ); - } - if( !zSql ){ - rc = SQLITE_NOMEM; - goto next_exit; - } - - rc = sqlite3_prepare(pVtab->db, zSql, -1, &pCur->pTableList, 0); - sqlite3_free(zSql); - if( rc!=SQLITE_OK ) goto next_exit; - } - - /* Set zSql to the SQL to the table_info pragma for the table currently - ** identified by the rows pointed to by statements pCur->pDbList and - ** pCur->pTableList. - */ - zSql = sqlite3_mprintf(pVtab->pType->zPragma, - sqlite3_column_text(pCur->pDbList, 1), - sqlite3_column_text(pCur->pTableList, 0) - ); - - if( !zSql ){ - rc = SQLITE_NOMEM; - goto next_exit; - } - rc = sqlite3_prepare(pVtab->db, zSql, -1, &pCur->pColumnList, 0); - sqlite3_free(zSql); - if( rc!=SQLITE_OK ) goto next_exit; - } - pCur->rowid++; - -next_exit: - /* TODO: Handle rc */ - return rc; -} - -/* -** Reset a schema table cursor. -*/ -static int schemaFilter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - int rc; - schema_vtab *pVtab = (schema_vtab *)(pVtabCursor->pVtab); - schema_cursor *pCur = (schema_cursor *)pVtabCursor; - UNUSED_PARAMETER(idxNum); - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - pCur->rowid = 0; - finalize(&pCur->pTableList); - finalize(&pCur->pColumnList); - finalize(&pCur->pDbList); - rc = sqlite3_prepare(pVtab->db,"SELECT 0, 'main'", -1, &pCur->pDbList, 0); - return (rc==SQLITE_OK ? schemaNext(pVtabCursor) : rc); -} - -/* -** Analyse the WHERE condition. -*/ -static int schemaBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ - UNUSED_PARAMETER(tab); - UNUSED_PARAMETER(pIdxInfo); - return SQLITE_OK; -} - -/* -** A virtual table module that merely echos method calls into TCL -** variables. -*/ -static sqlite3_module schemaModule = { - 0, /* iVersion */ - schemaCreate, - schemaCreate, - schemaBestIndex, - schemaDestroy, - schemaDestroy, - schemaOpen, /* xOpen - open a cursor */ - schemaClose, /* xClose - close a cursor */ - schemaFilter, /* xFilter - configure scan constraints */ - schemaNext, /* xNext - advance a cursor */ - schemaEof, /* xEof */ - schemaColumn, /* xColumn - read data */ - schemaRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ -}; - -/* -** Extension load function. -*/ -static int installSchemaModule(sqlite3 *db, sqlite3 *sdb){ - sqlite3_create_module(db, "schema", &schemaModule, (void *)sdb); - return 0; -} - -/* -** sj(zValue, zJoin) -** -** The following block contains the implementation of an aggregate -** function that returns a string. Each time the function is stepped, -** it appends data to an internal buffer. When the aggregate is finalized, -** the contents of the buffer are returned. -** -** The first time the aggregate is stepped the buffer is set to a copy -** of the first argument. The second time and subsequent times it is -** stepped a copy of the second argument is appended to the buffer, then -** a copy of the first. -** -** Example: -** -** INSERT INTO t1(a) VALUES('1'); -** INSERT INTO t1(a) VALUES('2'); -** INSERT INTO t1(a) VALUES('3'); -** SELECT sj(a, ', ') FROM t1; -** -** => "1, 2, 3" -** -*/ -struct StrBuffer { - char *zBuf; -}; -typedef struct StrBuffer StrBuffer; -static void joinFinalize(sqlite3_context *context){ - StrBuffer *p; - p = (StrBuffer *)sqlite3_aggregate_context(context, sizeof(StrBuffer)); - sqlite3_result_text(context, p->zBuf, -1, SQLITE_TRANSIENT); - sqlite3_free(p->zBuf); -} -static void joinStep( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - StrBuffer *p; - UNUSED_PARAMETER(argc); - p = (StrBuffer *)sqlite3_aggregate_context(context, sizeof(StrBuffer)); - if( p->zBuf==0 ){ - p->zBuf = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); - }else{ - char *zTmp = p->zBuf; - p->zBuf = sqlite3_mprintf("%s%s%s", - zTmp, sqlite3_value_text(argv[1]), sqlite3_value_text(argv[0]) - ); - sqlite3_free(zTmp); - } -} - -/* -** dq(zString) -** -** This scalar function accepts a single argument and interprets it as -** a text value. The return value is the argument enclosed in double -** quotes. If any double quote characters are present in the argument, -** these are escaped. -** -** dq('the raven "Nevermore."') == '"the raven ""Nevermore."""' -*/ -static void doublequote( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int ii; - char *zOut; - char *zCsr; - const char *zIn = (const char *)sqlite3_value_text(argv[0]); - int nIn = sqlite3_value_bytes(argv[0]); - - UNUSED_PARAMETER(argc); - zOut = sqlite3_malloc(nIn*2+3); - zCsr = zOut; - *zCsr++ = '"'; - for(ii=0; iinMalloc ){ - char *zNew; - nMalloc = 16 + (nOut+nCopy)*2; - zNew = (char*)sqlite3_realloc(zOut, nMalloc); - if( zNew==0 ){ - sqlite3_result_error_nomem(context); - return; - }else{ - zOut = zNew; - } - } - assert( nMalloc>=(nOut+nCopy) ); - memcpy(&zOut[nOut], zCopy, nCopy); - i += nReplace; - nOut += nCopy; - } - - sqlite3_result_text(context, zOut, nOut, SQLITE_TRANSIENT); - sqlite3_free(zOut); -} - -/* -** A callback for sqlite3_exec() invokes the callback specified by the -** GenfkeyCb structure pointed to by the void* passed as the first argument. -*/ -static int invokeCallback(void *p, int nArg, char **azArg, char **azCol){ - GenfkeyCb *pCb = (GenfkeyCb *)p; - UNUSED_PARAMETER(nArg); - UNUSED_PARAMETER(azCol); - return pCb->xData(pCb->pCtx, pCb->eType, azArg[0]); -} - -static int detectSchemaProblem( - sqlite3 *db, /* Database connection */ - const char *zMessage, /* English language error message */ - const char *zSql, /* SQL statement to run */ - GenfkeyCb *pCb -){ - sqlite3_stmt *pStmt; - int rc; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - char *zDel; - int iFk = sqlite3_column_int(pStmt, 0); - const char *zTab = (const char *)sqlite3_column_text(pStmt, 1); - zDel = sqlite3_mprintf("Error in table %s: %s", zTab, zMessage); - rc = pCb->xData(pCb->pCtx, pCb->eType, zDel); - sqlite3_free(zDel); - if( rc!=SQLITE_OK ) return rc; - zDel = sqlite3_mprintf( - "DELETE FROM temp.fkey WHERE from_tbl = %Q AND fkid = %d" - , zTab, iFk - ); - sqlite3_exec(db, zDel, 0, 0, 0); - sqlite3_free(zDel); - } - sqlite3_finalize(pStmt); - return SQLITE_OK; -} - -/* -** Create and populate temporary table "fkey". -*/ -static int populateTempTable(sqlite3 *db, GenfkeyCb *pCallback){ - int rc; - - rc = sqlite3_exec(db, - "CREATE VIRTUAL TABLE temp.v_fkey USING schema(foreign_key_list);" - "CREATE VIRTUAL TABLE temp.v_col USING schema(table_info);" - "CREATE VIRTUAL TABLE temp.v_idxlist USING schema(index_list);" - "CREATE VIRTUAL TABLE temp.v_idxinfo USING schema(index_info);" - "CREATE VIRTUAL TABLE temp.v_triggers USING schema(trigger_list);" - "CREATE TABLE temp.fkey AS " - "SELECT from_tbl, to_tbl, fkid, from_col, to_col, on_update, on_delete " - "FROM temp.v_fkey WHERE database = 'main';" - , 0, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - - rc = detectSchemaProblem(db, "foreign key columns do not exist", - "SELECT fkid, from_tbl " - "FROM temp.fkey " - "WHERE to_col IS NOT NULL AND NOT EXISTS (SELECT 1 " - "FROM temp.v_col WHERE tablename=to_tbl AND name==to_col" - ")", pCallback - ); - if( rc!=SQLITE_OK ) return rc; - - /* At this point the temp.fkey table is mostly populated. If any foreign - ** keys were specified so that they implicitly refer to they primary - ** key of the parent table, the "to_col" values of the temp.fkey rows - ** are still set to NULL. - ** - ** This is easily fixed for single column primary keys, but not for - ** composites. With a composite primary key, there is no way to reliably - ** query sqlite for the order in which the columns that make up the - ** composite key were declared i.e. there is no way to tell if the - ** schema actually contains "PRIMARY KEY(a, b)" or "PRIMARY KEY(b, a)". - ** Therefore, this case is not handled. The following function call - ** detects instances of this case. - */ - rc = detectSchemaProblem(db, "implicit mapping to composite primary key", - "SELECT fkid, from_tbl " - "FROM temp.fkey " - "WHERE to_col IS NULL " - "GROUP BY fkid, from_tbl HAVING count(*) > 1", pCallback - ); - if( rc!=SQLITE_OK ) return rc; - - /* Detect attempts to implicitly map to the primary key of a table - ** that has no primary key column. - */ - rc = detectSchemaProblem(db, "implicit mapping to non-existant primary key", - "SELECT fkid, from_tbl " - "FROM temp.fkey " - "WHERE to_col IS NULL AND NOT EXISTS " - "(SELECT 1 FROM temp.v_col WHERE pk AND tablename = temp.fkey.to_tbl)" - , pCallback - ); - if( rc!=SQLITE_OK ) return rc; - - /* Fix all the implicit primary key mappings in the temp.fkey table. */ - rc = sqlite3_exec(db, - "UPDATE temp.fkey SET to_col = " - "(SELECT name FROM temp.v_col WHERE pk AND tablename=temp.fkey.to_tbl)" - " WHERE to_col IS NULL;" - , 0, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - - /* Now check that all all parent keys are either primary keys or - ** subject to a unique constraint. - */ - rc = sqlite3_exec(db, - "CREATE TABLE temp.idx2 AS SELECT " - "il.tablename AS tablename," - "ii.indexname AS indexname," - "ii.name AS col " - "FROM temp.v_idxlist AS il, temp.v_idxinfo AS ii " - "WHERE il.isunique AND il.database='main' AND ii.indexname = il.name;" - "INSERT INTO temp.idx2 " - "SELECT tablename, 'pk', name FROM temp.v_col WHERE pk;" - - "CREATE TABLE temp.idx AS SELECT " - "tablename, indexname, sj(dq(col),',') AS cols " - "FROM (SELECT * FROM temp.idx2 ORDER BY col) " - "GROUP BY tablename, indexname;" - - "CREATE TABLE temp.fkey2 AS SELECT " - "fkid, from_tbl, to_tbl, sj(dq(to_col),',') AS cols " - "FROM (SELECT * FROM temp.fkey ORDER BY to_col) " - "GROUP BY fkid, from_tbl;" - - "CREATE TABLE temp.triggers AS SELECT " - "triggername FROM temp.v_triggers WHERE database='main' AND " - "triggername LIKE 'genfkey%';" - , 0, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - rc = detectSchemaProblem(db, "foreign key is not unique", - "SELECT fkid, from_tbl " - "FROM temp.fkey2 " - "WHERE NOT EXISTS (SELECT 1 " - "FROM temp.idx WHERE tablename=to_tbl AND fkey2.cols==idx.cols" - ")", pCallback - ); - if( rc!=SQLITE_OK ) return rc; - - return rc; -} - -#define GENFKEY_ERROR 1 -#define GENFKEY_DROPTRIGGER 2 -#define GENFKEY_CREATETRIGGER 3 -static int genfkey_create_triggers( - sqlite3 *sdb, /* Connection to read schema from */ - const char *zDb, /* Name of db to read ("main", "temp") */ - void *pCtx, /* Context pointer to pass to xData */ - int (*xData)(void *, int, const char *) -){ - const char *zSql = - "SELECT multireplace('" - - "-- Triggers for foreign key mapping:\n" - "--\n" - "-- /from_readable/ REFERENCES /to_readable/\n" - "-- on delete /on_delete/\n" - "-- on update /on_update/\n" - "--\n" - - /* The "BEFORE INSERT ON " trigger. This trigger's job is to - ** throw an exception if the user tries to insert a row into the - ** referencing table for which there is no corresponding row in - ** the referenced table. - */ - "CREATE TRIGGER /name/_insert_referencing BEFORE INSERT ON /tbl/ WHEN \n" - " /key_notnull/ AND NOT EXISTS (SELECT 1 FROM /ref/ WHERE /cond1/)\n" - "BEGIN\n" - " SELECT RAISE(ABORT, ''constraint failed'');\n" - "END;\n" - - /* The "BEFORE UPDATE ON " trigger. This trigger's job - ** is to throw an exception if the user tries to update a row in the - ** referencing table causing it to correspond to no row in the - ** referenced table. - */ - "CREATE TRIGGER /name/_update_referencing BEFORE\n" - " UPDATE OF /rkey_list/ ON /tbl/ WHEN \n" - " /key_notnull/ AND \n" - " NOT EXISTS (SELECT 1 FROM /ref/ WHERE /cond1/)\n" - "BEGIN\n" - " SELECT RAISE(ABORT, ''constraint failed'');\n" - "END;\n" - - - /* The "BEFORE DELETE ON " trigger. This trigger's job - ** is to detect when a row is deleted from the referenced table to - ** which rows in the referencing table correspond. The action taken - ** depends on the value of the 'ON DELETE' clause. - */ - "CREATE TRIGGER /name/_delete_referenced BEFORE DELETE ON /ref/ WHEN\n" - " EXISTS (SELECT 1 FROM /tbl/ WHERE /cond2/)\n" - "BEGIN\n" - " /delete_action/\n" - "END;\n" - - /* The "AFTER UPDATE ON " trigger. This trigger's job - ** is to detect when the key columns of a row in the referenced table - ** to which one or more rows in the referencing table correspond are - ** updated. The action taken depends on the value of the 'ON UPDATE' - ** clause. - */ - "CREATE TRIGGER /name/_update_referenced AFTER\n" - " UPDATE OF /fkey_list/ ON /ref/ WHEN \n" - " EXISTS (SELECT 1 FROM /tbl/ WHERE /cond2/)\n" - "BEGIN\n" - " /update_action/\n" - "END;\n" - "'" - - /* These are used in the SQL comment written above each set of triggers */ - ", '/from_readable/', from_tbl || '(' || sj(from_col, ', ') || ')'" - ", '/to_readable/', to_tbl || '(' || sj(to_col, ', ') || ')'" - ", '/on_delete/', on_delete" - ", '/on_update/', on_update" - - ", '/name/', 'genfkey' || min(rowid)" - ", '/tbl/', dq(from_tbl)" - ", '/ref/', dq(to_tbl)" - ", '/key_notnull/', sj('new.' || dq(from_col) || ' IS NOT NULL', ' AND ')" - - ", '/fkey_list/', sj(dq(to_col), ', ')" - ", '/rkey_list/', sj(dq(from_col), ', ')" - - ", '/cond1/', sj(multireplace('new./from/ == /to/'" - ", '/from/', dq(from_col)" - ", '/to/', dq(to_col)" - "), ' AND ')" - ", '/cond2/', sj(multireplace('old./to/ == /from/'" - ", '/from/', dq(from_col)" - ", '/to/', dq(to_col)" - "), ' AND ')" - - ", '/update_action/', CASE on_update " - "WHEN 'SET NULL' THEN " - "multireplace('UPDATE /tbl/ SET /setlist/ WHERE /where/;' " - ", '/setlist/', sj(dq(from_col)||' = NULL',', ')" - ", '/tbl/', dq(from_tbl)" - ", '/where/', sj(dq(from_col)||' = old.'||dq(to_col),' AND ')" - ")" - "WHEN 'CASCADE' THEN " - "multireplace('UPDATE /tbl/ SET /setlist/ WHERE /where/;' " - ", '/setlist/', sj(dq(from_col)||' = new.'||dq(to_col),', ')" - ", '/tbl/', dq(from_tbl)" - ", '/where/', sj(dq(from_col)||' = old.'||dq(to_col),' AND ')" - ")" - "ELSE " - " 'SELECT RAISE(ABORT, ''constraint failed'');'" - "END " - - ", '/delete_action/', CASE on_delete " - "WHEN 'SET NULL' THEN " - "multireplace('UPDATE /tbl/ SET /setlist/ WHERE /where/;' " - ", '/setlist/', sj(dq(from_col)||' = NULL',', ')" - ", '/tbl/', dq(from_tbl)" - ", '/where/', sj(dq(from_col)||' = old.'||dq(to_col),' AND ')" - ")" - "WHEN 'CASCADE' THEN " - "multireplace('DELETE FROM /tbl/ WHERE /where/;' " - ", '/tbl/', dq(from_tbl)" - ", '/where/', sj(dq(from_col)||' = old.'||dq(to_col),' AND ')" - ")" - "ELSE " - " 'SELECT RAISE(ABORT, ''constraint failed'');'" - "END " - - ") FROM temp.fkey " - "GROUP BY from_tbl, fkid" - ; - - int rc; - const int enc = SQLITE_UTF8; - sqlite3 *db = 0; - - GenfkeyCb cb; - cb.xData = xData; - cb.pCtx = pCtx; - - UNUSED_PARAMETER(zDb); - - /* Open the working database handle. */ - rc = sqlite3_open(":memory:", &db); - if( rc!=SQLITE_OK ) goto genfkey_exit; - - /* Create the special scalar and aggregate functions used by this program. */ - sqlite3_create_function(db, "dq", 1, enc, 0, doublequote, 0, 0); - sqlite3_create_function(db, "multireplace", -1, enc, db, multireplace, 0, 0); - sqlite3_create_function(db, "sj", 2, enc, 0, 0, joinStep, joinFinalize); - - /* Install the "schema" virtual table module */ - installSchemaModule(db, sdb); - - /* Create and populate a temp table with the information required to - ** build the foreign key triggers. See function populateTempTable() - ** for details. - */ - cb.eType = GENFKEY_ERROR; - rc = populateTempTable(db, &cb); - if( rc!=SQLITE_OK ) goto genfkey_exit; - - /* Unless the --no-drop option was specified, generate DROP TRIGGER - ** statements to drop any triggers in the database generated by a - ** previous run of this program. - */ - cb.eType = GENFKEY_DROPTRIGGER; - rc = sqlite3_exec(db, - "SELECT 'DROP TRIGGER main.' || dq(triggername) || ';' FROM triggers" - ,invokeCallback, (void *)&cb, 0 - ); - if( rc!=SQLITE_OK ) goto genfkey_exit; - - /* Run the main query to create the trigger definitions. */ - cb.eType = GENFKEY_CREATETRIGGER; - rc = sqlite3_exec(db, zSql, invokeCallback, (void *)&cb, 0); - if( rc!=SQLITE_OK ) goto genfkey_exit; - -genfkey_exit: - sqlite3_close(db); - return rc; -} - - -#endif -/* End genfkey logic. */ -/*************************************************************************/ -/*************************************************************************/ - /* ** If the following flag is set, then command execution stops ** at an error if we are not interactive. @@ -1212,6 +350,7 @@ static char *local_getline(char *zPrompt, FILE *in){ while( zLine[n] ){ n++; } if( n>0 && zLine[n-1]=='\n' ){ n--; + if( n>0 && zLine[n-1]=='\r' ) n--; zLine[n] = 0; eol = 1; } @@ -1276,6 +415,7 @@ struct callback_data { char outfile[FILENAME_MAX]; /* Filename for *out */ const char *zDbFilename; /* name of the database file */ sqlite3_stmt *pStmt; /* Current statement if any. */ + FILE *pLog; /* Write log output here */ }; /* @@ -1318,6 +458,16 @@ static int strlen30(const char *z){ return 0x3fffffff & (int)(z2 - z); } +/* +** A callback for the sqlite3_log() interface. +*/ +static void shellLog(void *pArg, int iErrCode, const char *zMsg){ + struct callback_data *p = (struct callback_data*)pArg; + if( p->pLog==0 ) return; + fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + fflush(p->pLog); +} + /* ** Output the given string as a hex-encoded blob (eg. X'1234' ) */ @@ -1500,10 +650,6 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int int i; struct callback_data *p = (struct callback_data*)pArg; - if( p->echoOn && p->cnt==0 && p->pStmt){ - printf("%s\n", sqlite3_sql(p->pStmt)); - } - switch( p->mode ){ case MODE_Line: { int w = 5; @@ -1832,10 +978,9 @@ static int shell_exec( struct callback_data *pArg, /* Pointer to struct callback_data */ char **pzErrMsg /* Error msg written here */ ){ - sqlite3_stmt *pStmt = NULL; - int rc = SQLITE_OK; - int rc2; - const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3_stmt *pStmt = NULL; /* Statement to execute. */ + int rc = SQLITE_OK; /* Return Code */ + const char *zLeftover; /* Tail of unprocessed SQL */ if( pzErrMsg ){ *pzErrMsg = NULL; @@ -1855,6 +1000,12 @@ static int shell_exec( continue; } + /* echo the sql statement if echo on */ + if( pArg->echoOn ){ + const char *zStmtSql = sqlite3_sql(pStmt); + fprintf(pArg->out,"%s\n", zStmtSql ? zStmtSql : zSql); + } + /* perform the first step. this will tell us if we ** have a result set or not and how wide it is. */ @@ -1916,28 +1067,15 @@ static int shell_exec( } } - /* if the last sqlite3_step() didn't complete successfully... */ - if( (SQLITE_OK != rc) && (SQLITE_DONE != rc) ){ - if( pzErrMsg ){ - *pzErrMsg = save_err_msg(db); - } - }else{ - rc = SQLITE_OK; - } - - rc2 = sqlite3_finalize(pStmt); - /* if the last sqlite3_finalize() didn't complete successfully - ** AND we don't have a saved error from sqlite3_step ... */ - if( (SQLITE_OK != rc2) && (SQLITE_OK == rc) ){ - rc = rc2; - if( pzErrMsg ){ - *pzErrMsg = save_err_msg(db); - } - } - - if( SQLITE_OK == rc ){ + /* Finalize the statement just executed. If this fails, save a + ** copy of the error message. Otherwise, set zSql to point to the + ** next statement to execute. */ + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ zSql = zLeftover; while( isspace(zSql[0]) ) zSql++; + }else if( pzErrMsg ){ + *pzErrMsg = save_err_msg(db); } } } /* end while */ @@ -2070,62 +1208,6 @@ static int run_schema_dump_query( return rc; } -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_SUBQUERY) -struct GenfkeyCmd { - sqlite3 *db; /* Database handle */ - struct callback_data *pCb; /* Callback data */ - int isIgnoreErrors; /* True for --ignore-errors */ - int isExec; /* True for --exec */ - int isNoDrop; /* True for --no-drop */ - int nErr; /* Number of errors seen so far */ -}; -typedef struct GenfkeyCmd GenfkeyCmd; - -static int genfkeyParseArgs(GenfkeyCmd *p, char **azArg, int nArg){ - int ii; - memset(p, 0, sizeof(GenfkeyCmd)); - - for(ii=0; ii2 && n<10 && 0==strncmp(azArg[ii], "--no-drop", n) ){ - p->isNoDrop = 1; - }else if( n>2 && n<16 && 0==strncmp(azArg[ii], "--ignore-errors", n) ){ - p->isIgnoreErrors = 1; - }else if( n>2 && n<7 && 0==strncmp(azArg[ii], "--exec", n) ){ - p->isExec = 1; - }else{ - fprintf(stderr, "unknown option: %s\n", azArg[ii]); - return -1; - } - } - - return SQLITE_OK; -} - -static int genfkeyCmdCb(void *pCtx, int eType, const char *z){ - GenfkeyCmd *p = (GenfkeyCmd *)pCtx; - if( eType==GENFKEY_ERROR && !p->isIgnoreErrors ){ - p->nErr++; - fprintf(stderr, "%s\n", z); - } - - if( p->nErr==0 && ( - (eType==GENFKEY_CREATETRIGGER) - || (eType==GENFKEY_DROPTRIGGER && !p->isNoDrop) - )){ - if( p->isExec ){ - sqlite3_exec(p->db, z, 0, 0, 0); - }else{ - char *zCol = "sql"; - callback((void *)p->pCb, 1, (char **)&z, (char **)&zCol); - } - } - - return SQLITE_OK; -} -#endif - /* ** Text of a help message */ @@ -2140,14 +1222,6 @@ static char zHelp[] = ".exit Exit this program\n" ".explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off.\n" " With no args, it turns EXPLAIN on.\n" -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_SUBQUERY) - ".genfkey ?OPTIONS? Options are:\n" - " --no-drop: Do not drop old fkey triggers.\n" - " --ignore-errors: Ignore tables with fkey errors\n" - " --exec: Execute generated SQL immediately\n" - " See file tool/genfkey.README in the source \n" - " distribution for further information.\n" -#endif ".header(s) ON|OFF Turn display of headers on or off\n" ".help Show this message\n" ".import FILE TABLE Import data from FILE into TABLE\n" @@ -2160,6 +1234,7 @@ static char zHelp[] = #ifndef SQLITE_OMIT_LOAD_EXTENSION ".load FILE ?ENTRY? Load an extension library\n" #endif + ".log FILE|off Turn logging on or off. FILE can be stderr/stdout\n" ".mode MODE ?TABLE? Set output mode where MODE is one of:\n" " csv Comma-separated values\n" " column Left-aligned columns. (See .width)\n" @@ -2467,17 +1542,6 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_SUBQUERY) - if( c=='g' && strncmp(azArg[0], "genfkey", n)==0 ){ - GenfkeyCmd cmd; - if( 0==genfkeyParseArgs(&cmd, &azArg[1], nArg-1) ){ - cmd.db = p->db; - cmd.pCb = p; - genfkey_create_triggers(p->db, "main", (void *)&cmd, genfkeyCmdCb); - } - }else -#endif - if( c=='h' && (strncmp(azArg[0], "header", n)==0 || strncmp(azArg[0], "headers", n)==0) && nArg>1 && nArg<3 ){ p->showHeader = booleanValue(azArg[1]); @@ -2686,6 +1750,26 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else #endif + if( c=='l' && strncmp(azArg[0], "log", n)==0 && nArg>=1 ){ + const char *zFile = azArg[1]; + if( p->pLog && p->pLog!=stdout && p->pLog!=stderr ){ + fclose(p->pLog); + p->pLog = 0; + } + if( strcmp(zFile,"stdout")==0 ){ + p->pLog = stdout; + }else if( strcmp(zFile, "stderr")==0 ){ + p->pLog = stderr; + }else if( strcmp(zFile, "off")==0 ){ + p->pLog = 0; + }else{ + p->pLog = fopen(zFile, "w"); + if( p->pLog==0 ){ + fprintf(stderr, "Error: cannot open \"%s\"\n", zFile); + } + } + }else + if( c=='m' && strncmp(azArg[0], "mode", n)==0 && nArg==2 ){ int n2 = strlen30(azArg[1]); if( (n2==4 && strncmp(azArg[1],"line",n2)==0) @@ -3331,8 +2415,10 @@ static void main_init(struct callback_data *data) { data->mode = MODE_List; memcpy(data->separator,"|", 2); data->showHeader = 0; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); + sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } int main(int argc, char **argv){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2f21c1e7..f831496a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -76,55 +76,43 @@ extern "C" { #endif /* -** CAPI3REF: Compile-Time Library Version Numbers {H10010} +** CAPI3REF: Compile-Time Library Version Numbers ** -** The SQLITE_VERSION and SQLITE_VERSION_NUMBER #defines in -** the sqlite3.h file specify the version of SQLite with which -** that header file is associated. -** -** The "version" of SQLite is a string of the form "W.X.Y" or "W.X.Y.Z". -** The W value is major version number and is always 3 in SQLite3. -** The W value only changes when backwards compatibility is -** broken and we intend to never break backwards compatibility. -** The X value is the minor version number and only changes when -** there are major feature enhancements that are forwards compatible -** but not backwards compatible. -** The Y value is the release number and is incremented with -** each release but resets back to 0 whenever X is incremented. -** The Z value only appears on branch releases. -** -** The SQLITE_VERSION_NUMBER is an integer that is computed as -** follows: -** -** 
-** SQLITE_VERSION_NUMBER = W*1000000 + X*1000 + Y
-**
+** ^(The [SQLITE_VERSION] C preprocessor macro in the sqlite3.h header +** evaluates to a string literal that is the SQLite version in the +** format "X.Y.Z" where X is the major version number (always 3 for +** SQLite3) and Y is the minor version number and Z is the release number.)^ +** ^(The [SQLITE_VERSION_NUMBER] C preprocessor macro resolves to an integer +** with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z are the same +** numbers used in [SQLITE_VERSION].)^ +** The SQLITE_VERSION_NUMBER for any given release of SQLite will also +** be larger than the release from which it is derived. Either Y will +** be held constant and Z will be incremented or else Y will be incremented +** and Z will be reset to zero. ** ** Since version 3.6.18, SQLite source code has been stored in the -** fossil configuration management -** system. The SQLITE_SOURCE_ID -** macro is a string which identifies a particular check-in of SQLite -** within its configuration management system. The string contains the -** date and time of the check-in (UTC) and an SHA1 hash of the entire -** source tree. +** Fossil configuration management +** system. ^The SQLITE_SOURCE_ID macro evalutes to +** a string which identifies a particular check-in of SQLite +** within its configuration management system. ^The SQLITE_SOURCE_ID +** string contains the date and time of the check-in (UTC) and an SHA1 +** hash of the entire source tree. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. -** -** Requirements: [H10011] [H10014] */ #define SQLITE_VERSION "--VERS--" #define SQLITE_VERSION_NUMBER --VERSION-NUMBER-- #define SQLITE_SOURCE_ID "--SOURCE-ID--" /* -** CAPI3REF: Run-Time Library Version Numbers {H10020} -** KEYWORDS: sqlite3_version +** CAPI3REF: Run-Time Library Version Numbers +** KEYWORDS: sqlite3_version, sqlite3_sourceid ** ** These interfaces provide the same information as the [SQLITE_VERSION], -** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] #defines in the header, -** but are associated with the library instead of the header file. Cautious +** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros +** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in ** the header, and thus insure that the application is @@ -133,27 +121,59 @@ extern "C" { ** 
 ** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
 ** assert( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)==0 );
-** assert( strcmp(sqlite3_libversion,SQLITE_VERSION)==0 );
-**
+** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 ); +** )^ ** -** The sqlite3_libversion() function returns the same information as is -** in the sqlite3_version[] string constant. The function is provided -** for use in DLLs since DLL users usually do not have direct access to string -** constants within the DLL. Similarly, the sqlite3_sourceid() function -** returns the same information as is in the [SQLITE_SOURCE_ID] #define of -** the header file. +** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] +** macro. ^The sqlite3_libversion() function returns a pointer to the +** to the sqlite3_version[] string constant. The sqlite3_libversion() +** function is provided for use in DLLs since DLL users usually do not have +** direct access to string constants within the DLL. ^The +** sqlite3_libversion_number() function returns an integer equal to +** [SQLITE_VERSION_NUMBER]. ^The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. ** ** See also: [sqlite_version()] and [sqlite_source_id()]. -** -** Requirements: [H10021] [H10022] [H10023] */ SQLITE_EXTERN const char sqlite3_version[]; const char *sqlite3_libversion(void); const char *sqlite3_sourceid(void); int sqlite3_libversion_number(void); +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* -** CAPI3REF: Test To See If The Library Is Threadsafe {H10100} +** CAPI3REF: Run-Time Library Compilation Options Diagnostics +** +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). +** +** ^The sqlite3_compileoption_get() function allows interating +** over the list of options that were defined at compile time by +** returning the N-th compile time option string. ^If N is out of range, +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by +** sqlite3_compileoption_get(). +** +** ^Support for the diagnostic functions sqlite3_compileoption_used() +** and sqlite3_compileoption_get() may be omitted by specifing the +** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. +** +** See also: SQL functions [sqlite_compileoption_used()] and +** [sqlite_compileoption_get()] and the [compile_options pragma]. +*/ +int sqlite3_compileoption_used(const char *zOptName); +const char *sqlite3_compileoption_get(int N); +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* +** CAPI3REF: Test To See If The Library Is Threadsafe +** +** ^The sqlite3_threadsafe() function returns zero if and only if +** SQLite was compiled mutexing code omitted due to the +** [SQLITE_THREADSAFE] compile-time option being set to 0. ** ** SQLite can be compiled with or without mutexes. When ** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes @@ -165,7 +185,7 @@ int sqlite3_libversion_number(void); ** Enabling mutexes incurs a measurable performance penalty. ** So if speed is of utmost importance, it makes sense to disable ** the mutexes. But for maximum safety, mutexes should be enabled. -** The default behavior is for mutexes to be enabled. +** ^The default behavior is for mutexes to be enabled. ** ** This interface can be used by an application to make sure that the ** version of SQLite that it is linking against was compiled with @@ -173,21 +193,21 @@ int sqlite3_libversion_number(void); ** ** This interface only reports on the compile-time mutex setting ** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with -** SQLITE_THREADSAFE=1 then mutexes are enabled by default but +** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but ** can be fully or partially disabled using a call to [sqlite3_config()] ** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], -** or [SQLITE_CONFIG_MUTEX]. The return value of this function shows -** only the default compile-time setting, not any run-time changes -** to that setting. +** or [SQLITE_CONFIG_MUTEX]. ^(The return value of the +** sqlite3_threadsafe() function shows only the compile-time setting of +** thread safety, not any run-time changes to that setting made by +** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() +** is unchanged by calls to sqlite3_config().)^ ** ** See the [threading mode] documentation for additional information. -** -** Requirements: [H10101] [H10102] */ int sqlite3_threadsafe(void); /* -** CAPI3REF: Database Connection Handle {H12000} +** CAPI3REF: Database Connection Handle ** KEYWORDS: {database connection} {database connections} ** ** Each open SQLite database is represented by a pointer to an instance of @@ -202,7 +222,7 @@ int sqlite3_threadsafe(void); typedef struct sqlite3 sqlite3; /* -** CAPI3REF: 64-Bit Integer Types {H10200} +** CAPI3REF: 64-Bit Integer Types ** KEYWORDS: sqlite_int64 sqlite_uint64 ** ** Because there is no cross-platform way to specify 64-bit integer types @@ -212,7 +232,10 @@ typedef struct sqlite3 sqlite3; ** The sqlite_int64 and sqlite_uint64 types are supported for backwards ** compatibility only. ** -** Requirements: [H10201] [H10202] +** ^The sqlite3_int64 and sqlite_int64 types can store integer values +** between -9223372036854775808 and +9223372036854775807 inclusive. ^The +** sqlite3_uint64 and sqlite_uint64 types can store integer values +** between 0 and +18446744073709551615 inclusive. */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; @@ -236,24 +259,28 @@ typedef sqlite_uint64 sqlite3_uint64; #endif /* -** CAPI3REF: Closing A Database Connection {H12010} +** CAPI3REF: Closing A Database Connection ** -** This routine is the destructor for the [sqlite3] object. +** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. +** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is +** successfullly destroyed and all associated resources are deallocated. ** ** Applications must [sqlite3_finalize | finalize] all [prepared statements] ** and [sqlite3_blob_close | close] all [BLOB handles] associated with -** the [sqlite3] object prior to attempting to close the object. +** the [sqlite3] object prior to attempting to close the object. ^If +** sqlite3_close() is called on a [database connection] that still has +** outstanding [prepared statements] or [BLOB handles], then it returns +** SQLITE_BUSY. ** -** If [sqlite3_close()] is invoked while a transaction is open, +** ^If [sqlite3_close()] is invoked while a transaction is open, ** the transaction is automatically rolled back. ** ** The C parameter to [sqlite3_close(C)] must be either a NULL ** pointer or an [sqlite3] object pointer obtained ** from [sqlite3_open()], [sqlite3_open16()], or ** [sqlite3_open_v2()], and not previously closed. -** -** Requirements: -** [H12011] [H12012] [H12013] [H12014] [H12015] [H12019] +** ^Calling sqlite3_close() with a NULL pointer argument is a +** harmless no-op. */ int sqlite3_close(sqlite3 *); @@ -265,48 +292,65 @@ int sqlite3_close(sqlite3 *); typedef int (*sqlite3_callback)(void*,int,char**, char**); /* -** CAPI3REF: One-Step Query Execution Interface {H12100} +** CAPI3REF: One-Step Query Execution Interface ** -** The sqlite3_exec() interface is a convenient way of running one or more -** SQL statements without having to write a lot of C code. The UTF-8 encoded -** SQL statements are passed in as the second parameter to sqlite3_exec(). -** The statements are evaluated one by one until either an error or -** an interrupt is encountered, or until they are all done. The 3rd parameter -** is an optional callback that is invoked once for each row of any query -** results produced by the SQL statements. The 5th parameter tells where -** to write any error messages. +** The sqlite3_exec() interface is a convenience wrapper around +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], +** that allows an application to run multiple statements of SQL +** without having to use a lot of C code. ** -** The error message passed back through the 5th parameter is held -** in memory obtained from [sqlite3_malloc()]. To avoid a memory leak, -** the calling application should call [sqlite3_free()] on any error -** message returned through the 5th parameter when it has finished using -** the error message. +** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, +** semicolon-separate SQL statements passed into its 2nd argument, +** in the context of the [database connection] passed in as its 1st +** argument. ^If the callback function of the 3rd argument to +** sqlite3_exec() is not NULL, then it is invoked for each result row +** coming out of the evaluated SQL statements. ^The 4th argument to +** to sqlite3_exec() is relayed through to the 1st argument of each +** callback invocation. ^If the callback pointer to sqlite3_exec() +** is NULL, then no callback is ever invoked and result rows are +** ignored. ** -** If the SQL statement in the 2nd parameter is NULL or an empty string -** or a string containing only whitespace and comments, then no SQL -** statements are evaluated and the database is not changed. +** ^If an error occurs while evaluating the SQL statements passed into +** sqlite3_exec(), then execution of the current statement stops and +** subsequent statements are skipped. ^If the 5th parameter to sqlite3_exec() +** is not NULL then any error message is written into memory obtained +** from [sqlite3_malloc()] and passed back through the 5th parameter. +** To avoid memory leaks, the application should invoke [sqlite3_free()] +** on error message strings returned through the 5th parameter of +** of sqlite3_exec() after the error message string is no longer needed. +** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors +** occur, then sqlite3_exec() sets the pointer in its 5th parameter to +** NULL before returning. ** -** The sqlite3_exec() interface is implemented in terms of -** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()]. -** The sqlite3_exec() routine does nothing to the database that cannot be done -** by [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()]. +** ^If an sqlite3_exec() callback returns non-zero, the sqlite3_exec() +** routine returns SQLITE_ABORT without invoking the callback again and +** without running any subsequent SQL statements. ** -** The first parameter to [sqlite3_exec()] must be an valid and open -** [database connection]. +** ^The 2nd argument to the sqlite3_exec() callback function is the +** number of columns in the result. ^The 3rd argument to the sqlite3_exec() +** callback is an array of pointers to strings obtained as if from +** [sqlite3_column_text()], one for each column. ^If an element of a +** result row is NULL then the corresponding string pointer for the +** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the +** sqlite3_exec() callback is an array of pointers to strings where each +** entry represents the name of corresponding result column as obtained +** from [sqlite3_column_name()]. ** -** The database connection must not be closed while -** [sqlite3_exec()] is running. +** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer +** to an empty string, or a pointer that contains only whitespace and/or +** SQL comments, then no SQL statements are evaluated and the database +** is not changed. ** -** The calling function should use [sqlite3_free()] to free -** the memory that *errmsg is left pointing at once the error -** message is no longer needed. +** Restrictions: ** -** The SQL statement text in the 2nd parameter to [sqlite3_exec()] -** must remain unchanged while [sqlite3_exec()] is running. -** -** Requirements: -** [H12101] [H12102] [H12104] [H12105] [H12107] [H12110] [H12113] [H12116] -** [H12119] [H12122] [H12125] [H12131] [H12134] [H12137] [H12138] +**
    +**
  • The application must insure that the 1st parameter to sqlite3_exec() +** is a valid and open [database connection]. +**
  • The application must not close [database connection] specified by +** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not modify the SQL statement text passed into +** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
*/ int sqlite3_exec( sqlite3*, /* An open database */ @@ -317,7 +361,7 @@ int sqlite3_exec( ); /* -** CAPI3REF: Result Codes {H10210} +** CAPI3REF: Result Codes ** KEYWORDS: SQLITE_OK {error code} {error codes} ** KEYWORDS: {result code} {result codes} ** @@ -361,7 +405,7 @@ int sqlite3_exec( /* end-of-error-codes */ /* -** CAPI3REF: Extended Result Codes {H10220} +** CAPI3REF: Extended Result Codes ** KEYWORDS: {extended error code} {extended error codes} ** KEYWORDS: {extended result code} {extended result codes} ** @@ -403,7 +447,7 @@ int sqlite3_exec( #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8) ) /* -** CAPI3REF: Flags For File Open Operations {H10230} +** CAPI3REF: Flags For File Open Operations ** ** These bit values are intended for use in the ** 3rd parameter to the [sqlite3_open_v2()] interface and @@ -415,6 +459,7 @@ int sqlite3_exec( #define SQLITE_OPEN_CREATE 0x00000004 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ #define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ +#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ #define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ #define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ #define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ @@ -428,7 +473,7 @@ int sqlite3_exec( #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ /* -** CAPI3REF: Device Characteristics {H10240} +** CAPI3REF: Device Characteristics ** ** The xDeviceCapabilities method of the [sqlite3_io_methods] ** object returns an integer which is a vector of the these @@ -460,7 +505,7 @@ int sqlite3_exec( #define SQLITE_IOCAP_SEQUENTIAL 0x00000400 /* -** CAPI3REF: File Locking Levels {H10250} +** CAPI3REF: File Locking Levels ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods @@ -473,7 +518,7 @@ int sqlite3_exec( #define SQLITE_LOCK_EXCLUSIVE 4 /* -** CAPI3REF: Synchronization Type Flags {H10260} +** CAPI3REF: Synchronization Type Flags ** ** When SQLite invokes the xSync() method of an ** [sqlite3_io_methods] object it uses a combination of @@ -491,7 +536,7 @@ int sqlite3_exec( #define SQLITE_SYNC_DATAONLY 0x00010 /* -** CAPI3REF: OS Interface Open File Handle {H11110} +** CAPI3REF: OS Interface Open File Handle ** ** An [sqlite3_file] object represents an open file in the ** [sqlite3_vfs | OS interface layer]. Individual OS interface @@ -507,7 +552,7 @@ struct sqlite3_file { }; /* -** CAPI3REF: OS Interface File Virtual Methods Object {H11120} +** CAPI3REF: OS Interface File Virtual Methods Object ** ** Every file opened by the [sqlite3_vfs] xOpen method populates an ** [sqlite3_file] object (or, more commonly, a subclass of the @@ -612,7 +657,7 @@ struct sqlite3_io_methods { }; /* -** CAPI3REF: Standard File Control Opcodes {H11310} +** CAPI3REF: Standard File Control Opcodes ** ** These integer constants are opcodes for the xFileControl method ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] @@ -632,7 +677,7 @@ struct sqlite3_io_methods { #define SQLITE_LAST_ERRNO 4 /* -** CAPI3REF: Mutex Handle {H17110} +** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an ** abstract type for a mutex object. The SQLite core never looks @@ -644,7 +689,7 @@ struct sqlite3_io_methods { typedef struct sqlite3_mutex sqlite3_mutex; /* -** CAPI3REF: OS Interface Object {H11140} +** CAPI3REF: OS Interface Object ** ** An instance of the sqlite3_vfs object defines the interface between ** the SQLite core and the underlying operating system. The "vfs" @@ -798,10 +843,10 @@ struct sqlite3_vfs { }; /* -** CAPI3REF: Flags for the xAccess VFS method {H11190} +** CAPI3REF: Flags for the xAccess VFS method ** ** These integer constants can be used as the third parameter to -** the xAccess method of an [sqlite3_vfs] object. {END} They determine +** the xAccess method of an [sqlite3_vfs] object. They determine ** what kind of permissions the xAccess method is looking for. ** With SQLITE_ACCESS_EXISTS, the xAccess method ** simply checks whether the file exists. @@ -815,10 +860,10 @@ struct sqlite3_vfs { #define SQLITE_ACCESS_READ 2 /* -** CAPI3REF: Initialize The SQLite Library {H10130} +** CAPI3REF: Initialize The SQLite Library ** -** The sqlite3_initialize() routine initializes the -** SQLite library. The sqlite3_shutdown() routine +** ^The sqlite3_initialize() routine initializes the +** SQLite library. ^The sqlite3_shutdown() routine ** deallocates any resources that were allocated by sqlite3_initialize(). ** These routines are designed to aid in process initialization and ** shutdown on embedded systems. Workstation applications using @@ -827,14 +872,14 @@ struct sqlite3_vfs { ** A call to sqlite3_initialize() is an "effective" call if it is ** the first time sqlite3_initialize() is invoked during the lifetime of ** the process, or if it is the first time sqlite3_initialize() is invoked -** following a call to sqlite3_shutdown(). Only an effective call +** following a call to sqlite3_shutdown(). ^(Only an effective call ** of sqlite3_initialize() does any initialization. All other calls -** are harmless no-ops. +** are harmless no-ops.)^ ** ** A call to sqlite3_shutdown() is an "effective" call if it is the first -** call to sqlite3_shutdown() since the last sqlite3_initialize(). Only +** call to sqlite3_shutdown() since the last sqlite3_initialize(). ^(Only ** an effective call to sqlite3_shutdown() does any deinitialization. -** All other valid calls to sqlite3_shutdown() are harmless no-ops. +** All other valid calls to sqlite3_shutdown() are harmless no-ops.)^ ** ** The sqlite3_initialize() interface is threadsafe, but sqlite3_shutdown() ** is not. The sqlite3_shutdown() interface must only be called from a @@ -842,21 +887,21 @@ struct sqlite3_vfs { ** other SQLite resources must be deallocated prior to invoking ** sqlite3_shutdown(). ** -** Among other things, sqlite3_initialize() will invoke -** sqlite3_os_init(). Similarly, sqlite3_shutdown() +** Among other things, ^sqlite3_initialize() will invoke +** sqlite3_os_init(). Similarly, ^sqlite3_shutdown() ** will invoke sqlite3_os_end(). ** -** The sqlite3_initialize() routine returns [SQLITE_OK] on success. -** If for some reason, sqlite3_initialize() is unable to initialize +** ^The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** ^If for some reason, sqlite3_initialize() is unable to initialize ** the library (perhaps it is unable to allocate a needed resource such ** as a mutex) it returns an [error code] other than [SQLITE_OK]. ** -** The sqlite3_initialize() routine is called internally by many other +** ^The sqlite3_initialize() routine is called internally by many other ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically ** initialized when [sqlite3_open()] is called if it has not be initialized -** already. However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly ** prior to using any other SQLite interface. For maximum portability, @@ -895,8 +940,7 @@ int sqlite3_os_init(void); int sqlite3_os_end(void); /* -** CAPI3REF: Configuring The SQLite Library {H14100} -** EXPERIMENTAL +** CAPI3REF: Configuring The SQLite Library ** ** The sqlite3_config() interface is used to make global configuration ** changes to SQLite in order to tune SQLite to the specific needs of @@ -909,7 +953,9 @@ int sqlite3_os_end(void); ** threads while sqlite3_config() is running. Furthermore, sqlite3_config() ** may only be invoked prior to library initialization using ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** Note, however, that sqlite3_config() can be called as part of the +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the ** implementation of an application-defined [sqlite3_os_init()]. ** ** The first argument to sqlite3_config() is an integer @@ -918,26 +964,21 @@ int sqlite3_os_end(void); ** vary depending on the [SQLITE_CONFIG_SINGLETHREAD | configuration option] ** in the first argument. ** -** When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. -** If the option is unknown or SQLite is unable to set the option +** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. -** -** Requirements: -** [H14103] [H14106] [H14120] [H14123] [H14126] [H14129] [H14132] [H14135] -** [H14138] [H14141] [H14144] [H14147] [H14150] [H14153] [H14156] [H14159] -** [H14162] [H14165] [H14168] */ SQLITE_EXPERIMENTAL int sqlite3_config(int, ...); /* -** CAPI3REF: Configure database connections {H14200} +** CAPI3REF: Configure database connections ** EXPERIMENTAL ** ** The sqlite3_db_config() interface is used to make configuration ** changes to a [database connection]. The interface is similar to ** [sqlite3_config()] except that the changes apply to a single ** [database connection] (specified in the first argument). The -** sqlite3_db_config() interface can only be used immediately after +** sqlite3_db_config() interface should only be used immediately after ** the database connection is created using [sqlite3_open()], ** [sqlite3_open16()], or [sqlite3_open_v2()]. ** @@ -948,13 +989,13 @@ SQLITE_EXPERIMENTAL int sqlite3_config(int, ...); ** New verbs are likely to be added in future releases of SQLite. ** Additional arguments depend on the verb. ** -** Requirements: -** [H14203] [H14206] [H14209] [H14212] [H14215] +** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if +** the call is considered successful. */ SQLITE_EXPERIMENTAL int sqlite3_db_config(sqlite3*, int op, ...); /* -** CAPI3REF: Memory Allocation Routines {H10155} +** CAPI3REF: Memory Allocation Routines ** EXPERIMENTAL ** ** An instance of this object defines the interface between SQLite @@ -984,7 +1025,7 @@ SQLITE_EXPERIMENTAL int sqlite3_db_config(sqlite3*, int op, ...); ** The xRealloc method must work like realloc() from the standard C library ** with the exception that if the second argument to xRealloc is zero, ** xRealloc must be a no-op - it must not perform any allocation or -** deallocation. SQLite guaranteeds that the second argument to +** deallocation. ^SQLite guarantees that the second argument to ** xRealloc is always a value returned by a prior call to xRoundup. ** And so in cases where xRoundup always returns a positive number, ** xRealloc can perform exactly as the standard library realloc() and @@ -1036,7 +1077,7 @@ struct sqlite3_mem_methods { }; /* -** CAPI3REF: Configuration Options {H10160} +** CAPI3REF: Configuration Options ** EXPERIMENTAL ** ** These constants are the available integer configuration options that @@ -1051,22 +1092,33 @@ struct sqlite3_mem_methods { ** **
**
SQLITE_CONFIG_SINGLETHREAD
-**
There are no arguments to this option. This option disables +**
There are no arguments to this option. ^This option sets the +** [threading mode] to Single-thread. In other words, it disables ** all mutexing and puts SQLite into a mode where it can only be used -** by a single thread.
+** by a single thread. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to change the [threading mode] from its default +** value of Single-thread and so [sqlite3_config()] will return +** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD +** configuration option. ** **
SQLITE_CONFIG_MULTITHREAD
-**
There are no arguments to this option. This option disables +**
There are no arguments to this option. ^This option sets the +** [threading mode] to Multi-thread. In other words, it disables ** mutexing on [database connection] and [prepared statement] objects. ** The application is responsible for serializing access to ** [database connections] and [prepared statements]. But other mutexes ** are enabled so that SQLite will be safe to use in a multi-threaded ** environment as long as no two threads attempt to use the same -** [database connection] at the same time. See the [threading mode] -** documentation for additional information.
+** [database connection] at the same time. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Multi-thread [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_MULTITHREAD configuration option. ** **
SQLITE_CONFIG_SERIALIZED
-**
There are no arguments to this option. This option enables +**
There are no arguments to this option. ^This option sets the +** [threading mode] to Serialized. In other words, this option enables ** all mutexes including the recursive ** mutexes on [database connection] and [prepared statement] objects. ** In this mode (which is the default when SQLite is compiled with @@ -1074,55 +1126,63 @@ struct sqlite3_mem_methods { ** to [database connections] and [prepared statements] so that the ** application is free to use the same [database connection] or the ** same [prepared statement] in different threads at the same time. -** See the [threading mode] documentation for additional information.
+** ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Serialized [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_SERIALIZED configuration option. ** **
SQLITE_CONFIG_MALLOC
-**
This option takes a single argument which is a pointer to an +**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The argument specifies ** alternative low-level memory allocation routines to be used in place of -** the memory allocation routines built into SQLite.
+** the memory allocation routines built into SQLite.)^ ^SQLite makes +** its own private copy of the content of the [sqlite3_mem_methods] structure +** before the [sqlite3_config()] call returns. ** **
SQLITE_CONFIG_GETMALLOC
-**
This option takes a single argument which is a pointer to an +**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] -** structure is filled with the currently defined memory allocation routines. +** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or -** tracks memory usage, for example.
+** tracks memory usage, for example. ** **
SQLITE_CONFIG_MEMSTATUS
-**
This option takes single argument of type int, interpreted as a +**
^This option takes single argument of type int, interpreted as a ** boolean, which enables or disables the collection of memory allocation -** statistics. When disabled, the following SQLite interfaces become -** non-operational: +** statistics. ^(When memory allocation statistics are disabled, the +** following SQLite interfaces become non-operational: **
    **
  • [sqlite3_memory_used()] **
  • [sqlite3_memory_highwater()] **
  • [sqlite3_soft_heap_limit()] **
  • [sqlite3_status()] -**
+** )^ +** ^Memory allocation statistics are enabled by default unless SQLite is +** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory +** allocation statistics are disabled by default. **
** **
SQLITE_CONFIG_SCRATCH
-**
This option specifies a static memory buffer that SQLite can use for +**
^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz ** argument must be a multiple of 16. The sz parameter should be a few bytes ** larger than the actual scratch space required due to internal overhead. -** The first argument should pointer to an 8-byte aligned buffer +** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** SQLite will use no more than one scratch buffer at once per thread, so -** N should be set to the expected maximum number of threads. The sz -** parameter should be 6 times the size of the largest database page size. -** Scratch buffers are used as part of the btree balance operation. If -** The btree balancer needs additional memory beyond what is provided by -** scratch buffers or if no scratch buffer space is specified, then SQLite -** goes to [sqlite3_malloc()] to obtain the memory it needs.
+** ^SQLite will use no more than one scratch buffer per thread. So +** N should be set to the expected maximum number of threads. ^SQLite will +** never require a scratch buffer that is more than 6 times the database +** page size. ^If SQLite needs needs additional scratch memory beyond +** what is provided by this configuration option, then +** [sqlite3_malloc()] will be used to obtain the memory needed. ** **
SQLITE_CONFIG_PAGECACHE
-**
This option specifies a static memory buffer that SQLite can use for +**
^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implemenation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the SQLITE_CONFIG_PCACHE option. @@ -1130,28 +1190,28 @@ struct sqlite3_mem_methods { ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus a little extra for each -** page header. The page header size is 20 to 40 bytes depending on -** the host architecture. It is harmless, apart from the wasted memory, +** page header. ^The page header size is 20 to 40 bytes depending on +** the host architecture. ^It is harmless, apart from the wasted memory, ** to make sz a little too large. The first ** argument should point to an allocation of at least sz*N bytes of memory. -** SQLite will use the memory provided by the first argument to satisfy its -** memory needs for the first N pages that it adds to cache. If additional +** ^SQLite will use the memory provided by the first argument to satisfy its +** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** The implementation might use one or more of the N buffers to hold +** ^The implementation might use one or more of the N buffers to hold ** memory accounting information. The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
** **
SQLITE_CONFIG_HEAP
-**
This option specifies a static memory buffer that SQLite will use +**
^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. ** There are three arguments: An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. -** If the first pointer (the memory pointer) is NULL, then SQLite reverts +** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), -** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. If the +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the ** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or ** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. @@ -1159,39 +1219,50 @@ struct sqlite3_mem_methods { ** boundary or subsequent behavior of SQLite will be undefined.
** **
SQLITE_CONFIG_MUTEX
-**
This option takes a single argument which is a pointer to an +**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The argument specifies ** alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.
+** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the +** content of the [sqlite3_mutex_methods] structure before the call to +** [sqlite3_config()] returns. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will +** return [SQLITE_ERROR]. ** **
SQLITE_CONFIG_GETMUTEX
-**
This option takes a single argument which is a pointer to an +**
^(This option takes a single argument which is a pointer to an ** instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] -** structure is filled with the currently defined mutex routines. +** structure is filled with the currently defined mutex routines.)^ ** This option can be used to overload the default mutex allocation ** routines with a wrapper used to track mutex usage for performance -** profiling or testing, for example.
+** profiling or testing, for example. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will +** return [SQLITE_ERROR]. ** **
SQLITE_CONFIG_LOOKASIDE
-**
This option takes two arguments that determine the default -** memory allocation lookaside optimization. The first argument is the +**
^(This option takes two arguments that determine the default +** memory allocation for the lookaside memory allocator on each +** [database connection]. The first argument is the ** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection. This option sets the -** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** slots allocated to each database connection.)^ ^(This option sets the +** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] ** verb to [sqlite3_db_config()] can be used to change the lookaside -** configuration on individual connections.
+** configuration on individual connections.)^ ** **
SQLITE_CONFIG_PCACHE
-**
This option takes a single argument which is a pointer to +**
^(This option takes a single argument which is a pointer to ** an [sqlite3_pcache_methods] object. This object specifies the interface -** to a custom page cache implementation. SQLite makes a copy of the +** to a custom page cache implementation.)^ ^SQLite makes a copy of the ** object and uses it for page cache memory allocations.
** **
SQLITE_CONFIG_GETPCACHE
-**
This option takes a single argument which is a pointer to an +**
^(This option takes a single argument which is a pointer to an ** [sqlite3_pcache_methods] object. SQLite copies of the current -** page cache implementation into that object.
+** page cache implementation into that object.)^ ** **
*/ @@ -1210,9 +1281,10 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ #define SQLITE_CONFIG_PCACHE 14 /* sqlite3_pcache_methods* */ #define SQLITE_CONFIG_GETPCACHE 15 /* sqlite3_pcache_methods* */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ /* -** CAPI3REF: Configuration Options {H10170} +** CAPI3REF: Configuration Options ** EXPERIMENTAL ** ** These constants are the available integer configuration options that @@ -1221,23 +1293,25 @@ struct sqlite3_mem_methods { ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_db_config()] to make sure that -** the call worked. The [sqlite3_db_config()] interface will return a +** the call worked. ^The [sqlite3_db_config()] interface will return a ** non-zero [error code] if a discontinued or unsupported configuration option ** is invoked. ** **
**
SQLITE_DBCONFIG_LOOKASIDE
-**
This option takes three additional arguments that determine the +**
^This option takes three additional arguments that determine the ** [lookaside memory allocator] configuration for the [database connection]. -** The first argument (the third parameter to [sqlite3_db_config()] is a +** ^The first argument (the third parameter to [sqlite3_db_config()] is a ** pointer to an memory buffer to use for lookaside memory. -** The first argument may be NULL in which case SQLite will allocate the -** lookaside buffer itself using [sqlite3_malloc()]. The second argument is the -** size of each lookaside buffer slot and the third argument is the number of +** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb +** may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the +** size of each lookaside buffer slot. ^The third argument is the number of ** slots. The size of the buffer in the first argument must be greater than ** or equal to the product of the second and third arguments. The buffer -** must be aligned to an 8-byte boundary. If the second argument is not -** a multiple of 8, it is internally rounded down to the next smaller +** must be aligned to an 8-byte boundary. ^If the second argument to +** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally +** rounded down to the next smaller ** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE]
** **
@@ -1246,56 +1320,50 @@ struct sqlite3_mem_methods { /* -** CAPI3REF: Enable Or Disable Extended Result Codes {H12200} +** CAPI3REF: Enable Or Disable Extended Result Codes ** -** The sqlite3_extended_result_codes() routine enables or disables the -** [extended result codes] feature of SQLite. The extended result -** codes are disabled by default for historical compatibility considerations. -** -** Requirements: -** [H12201] [H12202] +** ^The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. ^The extended result +** codes are disabled by default for historical compatibility. */ int sqlite3_extended_result_codes(sqlite3*, int onoff); /* -** CAPI3REF: Last Insert Rowid {H12220} +** CAPI3REF: Last Insert Rowid ** -** Each entry in an SQLite table has a unique 64-bit signed -** integer key called the [ROWID | "rowid"]. The rowid is always available +** ^Each entry in an SQLite table has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available ** as an undeclared column named ROWID, OID, or _ROWID_ as long as those -** names are not also used by explicitly declared columns. If +** names are not also used by explicitly declared columns. ^If ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** This routine returns the [rowid] of the most recent +** ^This routine returns the [rowid] of the most recent ** successful [INSERT] into the database from the [database connection] -** in the first argument. If no successful [INSERT]s +** in the first argument. ^If no successful [INSERT]s ** have ever occurred on that database connection, zero is returned. ** -** If an [INSERT] occurs within a trigger, then the [rowid] of the inserted +** ^(If an [INSERT] occurs within a trigger, then the [rowid] of the inserted ** row is returned by this routine as long as the trigger is running. ** But once the trigger terminates, the value returned by this routine -** reverts to the last value inserted before the trigger fired. +** reverts to the last value inserted before the trigger fired.)^ ** -** An [INSERT] that fails due to a constraint violation is not a +** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this -** routine. Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, ** and INSERT OR ABORT make no changes to the return value of this -** routine when their insertion fails. When INSERT OR REPLACE +** routine when their insertion fails. ^(When INSERT OR REPLACE ** encounters a constraint violation, it does not fail. The ** INSERT continues to completion after deleting rows that caused ** the constraint problem so INSERT OR REPLACE will always change -** the return value of this interface. +** the return value of this interface.)^ ** -** For the purposes of this routine, an [INSERT] is considered to +** ^For the purposes of this routine, an [INSERT] is considered to ** be successful even if it is subsequently rolled back. ** ** This function is accessible to SQL statements via the ** [last_insert_rowid() SQL function]. ** -** Requirements: -** [H12221] [H12223] -** ** If a separate thread performs a new [INSERT] on the same ** database connection while the [sqlite3_last_insert_rowid()] ** function is running and thus changes the last insert [rowid], @@ -1306,25 +1374,25 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); /* -** CAPI3REF: Count The Number Of Rows Modified {H12240} +** CAPI3REF: Count The Number Of Rows Modified ** -** This function returns the number of database rows that were changed +** ^This function returns the number of database rows that were changed ** or inserted or deleted by the most recently completed SQL statement ** on the [database connection] specified by the first parameter. -** Only changes that are directly specified by the [INSERT], [UPDATE], +** ^(Only changes that are directly specified by the [INSERT], [UPDATE], ** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted. Use the +** triggers or [foreign key actions] are not counted.)^ Use the ** [sqlite3_total_changes()] function to find the total number of changes ** including changes caused by triggers and foreign key actions. ** -** Changes to a view that are simulated by an [INSTEAD OF trigger] +** ^Changes to a view that are simulated by an [INSTEAD OF trigger] ** are not counted. Only real table changes are counted. ** -** A "row change" is a change to a single row of a single table +** ^(A "row change" is a change to a single row of a single table ** caused by an INSERT, DELETE, or UPDATE statement. Rows that ** are changed as side effects of [REPLACE] constraint resolution, ** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes. +** mechanisms do not count as direct row changes.)^ ** ** A "trigger context" is a scope of execution that begins and ** ends with the script of a [CREATE TRIGGER | trigger]. @@ -1334,28 +1402,25 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); ** new trigger context is entered for the duration of that one ** trigger. Subtriggers create subcontexts for their duration. ** -** Calling [sqlite3_exec()] or [sqlite3_step()] recursively does +** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does ** not create a new trigger context. ** -** This function returns the number of direct row changes in the +** ^This function returns the number of direct row changes in the ** most recent INSERT, UPDATE, or DELETE statement within the same ** trigger context. ** -** Thus, when called from the top level, this function returns the +** ^Thus, when called from the top level, this function returns the ** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. Within the body of a trigger, +** that also occurred at the top level. ^(Within the body of a trigger, ** the sqlite3_changes() interface can be called to find the number of ** changes in the most recently completed INSERT, UPDATE, or DELETE ** statement within the body of the same trigger. ** However, the number returned does not include changes -** caused by subtriggers since those have their own context. +** caused by subtriggers since those have their own context.)^ ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. ** -** Requirements: -** [H12241] [H12243] -** ** If a separate thread makes changes on the same database connection ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. @@ -1363,27 +1428,25 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); int sqlite3_changes(sqlite3*); /* -** CAPI3REF: Total Number Of Rows Modified {H12260} +** CAPI3REF: Total Number Of Rows Modified ** -** This function returns the number of row changes caused by [INSERT], +** ^This function returns the number of row changes caused by [INSERT], ** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** The count includes all changes from all [CREATE TRIGGER | trigger] -** contexts and changes made by [foreign key actions]. However, +** ^(The count returned by sqlite3_total_changes() includes all changes +** from all [CREATE TRIGGER | trigger] contexts and changes made by +** [foreign key actions]. However, ** the count does not include changes used to implement [REPLACE] constraints, ** do rollbacks or ABORT processing, or [DROP TABLE] processing. The ** count does not include rows of views that fire an [INSTEAD OF trigger], ** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted. -** The changes are counted as soon as the statement that makes them is -** completed (when the statement handle is passed to [sqlite3_reset()] or -** [sqlite3_finalize()]). +** are counted.)^ +** ^The sqlite3_total_changes() function counts the changes as soon as +** the statement that makes them is completed (when the statement handle +** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). ** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** -** Requirements: -** [H12261] [H12263] -** ** If a separate thread makes changes on the same database connection ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. @@ -1391,75 +1454,70 @@ int sqlite3_changes(sqlite3*); int sqlite3_total_changes(sqlite3*); /* -** CAPI3REF: Interrupt A Long-Running Query {H12270} +** CAPI3REF: Interrupt A Long-Running Query ** -** This function causes any pending database operation to abort and +** ^This function causes any pending database operation to abort and ** return at its earliest opportunity. This routine is typically ** called in response to a user action such as pressing "Cancel" ** or Ctrl-C where the user wants a long query operation to halt ** immediately. ** -** It is safe to call this routine from a thread different from the +** ^It is safe to call this routine from a thread different from the ** thread that is currently running the database operation. But it ** is not safe to call this routine with a [database connection] that ** is closed or might close before sqlite3_interrupt() returns. ** -** If an SQL operation is very nearly finished at the time when +** ^If an SQL operation is very nearly finished at the time when ** sqlite3_interrupt() is called, then it might not have an opportunity ** to be interrupted and might continue to completion. ** -** An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. -** If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** ^An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** ^If the interrupted SQL operation is an INSERT, UPDATE, or DELETE ** that is inside an explicit transaction, then the entire transaction ** will be rolled back automatically. ** -** The sqlite3_interrupt(D) call is in effect until all currently running -** SQL statements on [database connection] D complete. Any new SQL statements +** ^The sqlite3_interrupt(D) call is in effect until all currently running +** SQL statements on [database connection] D complete. ^Any new SQL statements ** that are started after the sqlite3_interrupt() call and before the ** running statements reaches zero are interrupted as if they had been -** running prior to the sqlite3_interrupt() call. New SQL statements +** running prior to the sqlite3_interrupt() call. ^New SQL statements ** that are started after the running statement count reaches zero are ** not effected by the sqlite3_interrupt(). -** A call to sqlite3_interrupt(D) that occurs when there are no running +** ^A call to sqlite3_interrupt(D) that occurs when there are no running ** SQL statements is a no-op and has no effect on SQL statements ** that are started after the sqlite3_interrupt() call returns. ** -** Requirements: -** [H12271] [H12272] -** ** If the database connection closes while [sqlite3_interrupt()] ** is running then bad things will likely happen. */ void sqlite3_interrupt(sqlite3*); /* -** CAPI3REF: Determine If An SQL Statement Is Complete {H10510} +** CAPI3REF: Determine If An SQL Statement Is Complete ** ** These routines are useful during command-line input to determine if the ** currently entered text seems to form a complete SQL statement or ** if additional input is needed before sending the text into -** SQLite for parsing. These routines return 1 if the input string -** appears to be a complete SQL statement. A statement is judged to be +** SQLite for parsing. ^These routines return 1 if the input string +** appears to be a complete SQL statement. ^A statement is judged to be ** complete if it ends with a semicolon token and is not a prefix of a -** well-formed CREATE TRIGGER statement. Semicolons that are embedded within +** well-formed CREATE TRIGGER statement. ^Semicolons that are embedded within ** string literals or quoted identifier names or comments are not ** independent tokens (they are part of the token in which they are -** embedded) and thus do not count as a statement terminator. Whitespace +** embedded) and thus do not count as a statement terminator. ^Whitespace ** and comments that follow the final semicolon are ignored. ** -** These routines return 0 if the statement is incomplete. If a +** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements thus ** will not detect syntactically incorrect SQL. ** -** If SQLite has not been initialized using [sqlite3_initialize()] prior +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior ** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked ** automatically by sqlite3_complete16(). If that initialization fails, ** then the return value from sqlite3_complete16() will be non-zero -** regardless of whether or not the input SQL is complete. -** -** Requirements: [H10511] [H10512] +** regardless of whether or not the input SQL is complete.)^ ** ** The input to [sqlite3_complete()] must be a zero-terminated ** UTF-8 string. @@ -1471,27 +1529,27 @@ int sqlite3_complete(const char *sql); int sqlite3_complete16(const void *sql); /* -** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors {H12310} +** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors ** -** This routine sets a callback function that might be invoked whenever +** ^This routine sets a callback function that might be invoked whenever ** an attempt is made to open a database table that another thread ** or process has locked. ** -** If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] -** is returned immediately upon encountering the lock. If the busy callback -** is not NULL, then the callback will be invoked with two arguments. +** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] +** is returned immediately upon encountering the lock. ^If the busy callback +** is not NULL, then the callback might be invoked with two arguments. ** -** The first argument to the handler is a copy of the void* pointer which -** is the third argument to sqlite3_busy_handler(). The second argument to -** the handler callback is the number of times that the busy handler has -** been invoked for this locking event. If the +** ^The first argument to the busy handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). ^The second argument to +** the busy handler callback is the number of times that the busy handler has +** been invoked for this locking event. ^If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. -** If the callback returns non-zero, then another attempt +** ^If the callback returns non-zero, then another attempt ** is made to open the database for reading and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked -** when there is lock contention. If SQLite determines that invoking the busy +** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] ** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. ** Consider a scenario where one process is holding a read lock that @@ -1505,65 +1563,59 @@ int sqlite3_complete16(const void *sql); ** will induce the first process to release its read lock and allow ** the second process to proceed. ** -** The default busy callback is NULL. +** ^The default busy callback is NULL. ** -** The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] +** ^The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] ** when SQLite is in the middle of a large transaction where all the ** changes will not fit into the in-memory cache. SQLite will ** already hold a RESERVED lock on the database file, but it needs ** to promote this lock to EXCLUSIVE so that it can spill cache ** pages into the database file without harm to concurrent -** readers. If it is unable to promote the lock, then the in-memory +** readers. ^If it is unable to promote the lock, then the in-memory ** cache will be left in an inconsistent state and so the error ** code is promoted from the relatively benign [SQLITE_BUSY] to -** the more severe [SQLITE_IOERR_BLOCKED]. This error code promotion +** the more severe [SQLITE_IOERR_BLOCKED]. ^This error code promotion ** forces an automatic rollback of the changes. See the ** ** CorruptionFollowingBusyError wiki page for a discussion of why ** this is important. ** -** There can only be a single busy handler defined for each +** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any -** previously set handler. Note that calling [sqlite3_busy_timeout()] +** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] ** will also set or clear the busy handler. ** ** The busy callback should not take any actions which modify the ** database connection that invoked the busy handler. Any such actions ** result in undefined behavior. ** -** Requirements: -** [H12311] [H12312] [H12314] [H12316] [H12318] -** ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); /* -** CAPI3REF: Set A Busy Timeout {H12340} +** CAPI3REF: Set A Busy Timeout ** -** This routine sets a [sqlite3_busy_handler | busy handler] that sleeps -** for a specified amount of time when a table is locked. The handler +** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. ^The handler ** will sleep multiple times until at least "ms" milliseconds of sleeping -** have accumulated. {H12343} After "ms" milliseconds of sleeping, +** have accumulated. ^After at least "ms" milliseconds of sleeping, ** the handler returns 0 which causes [sqlite3_step()] to return ** [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. ** -** Calling this routine with an argument less than or equal to zero +** ^Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. ** -** There can only be a single busy handler for a particular +** ^(There can only be a single busy handler for a particular ** [database connection] any any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling -** this routine, that other busy handler is cleared. -** -** Requirements: -** [H12341] [H12343] [H12344] +** this routine, that other busy handler is cleared.)^ */ int sqlite3_busy_timeout(sqlite3*, int ms); /* -** CAPI3REF: Convenience Routines For Running Queries {H12370} +** CAPI3REF: Convenience Routines For Running Queries ** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the @@ -1611,27 +1663,25 @@ int sqlite3_busy_timeout(sqlite3*, int ms); ** azResult[7] = "21"; ** ** -** The sqlite3_get_table() function evaluates one or more +** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 -** string of its 2nd parameter. It returns a result table to the +** string of its 2nd parameter and returns a result table to the ** pointer given in its 3rd parameter. ** -** After the calling function has finished using the result, it should -** pass the pointer to the result table to sqlite3_free_table() in order to +** After the application has finished with the result from sqlite3_get_table(), +** it should pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** The sqlite3_get_table() interface is implemented as a wrapper around +** ^(The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not -** reflected in subsequent calls to [sqlite3_errcode()] or [sqlite3_errmsg()]. -** -** Requirements: -** [H12371] [H12373] [H12374] [H12376] [H12379] [H12382] +** reflected in subsequent calls to [sqlite3_errcode()] or +** [sqlite3_errmsg()].)^ */ int sqlite3_get_table( sqlite3 *db, /* An open database */ @@ -1644,33 +1694,33 @@ int sqlite3_get_table( void sqlite3_free_table(char **result); /* -** CAPI3REF: Formatted String Printing Functions {H17400} +** CAPI3REF: Formatted String Printing Functions ** ** These routines are work-alikes of the "printf()" family of functions ** from the standard C library. ** -** The sqlite3_mprintf() and sqlite3_vmprintf() routines write their +** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their ** results into memory obtained from [sqlite3_malloc()]. ** The strings returned by these two routines should be -** released by [sqlite3_free()]. Both routines return a +** released by [sqlite3_free()]. ^Both routines return a ** NULL pointer if [sqlite3_malloc()] is unable to allocate enough ** memory to hold the resulting string. ** -** In sqlite3_snprintf() routine is similar to "snprintf()" from +** ^(In sqlite3_snprintf() routine is similar to "snprintf()" from ** the standard C library. The result is written into the ** buffer supplied as the second parameter whose size is given by ** the first parameter. Note that the order of the -** first two parameters is reversed from snprintf(). This is an +** first two parameters is reversed from snprintf().)^ This is an ** historical accident that cannot be fixed without breaking -** backwards compatibility. Note also that sqlite3_snprintf() +** backwards compatibility. ^(Note also that sqlite3_snprintf() ** returns a pointer to its buffer instead of the number of -** characters actually written into the buffer. We admit that +** characters actually written into the buffer.)^ We admit that ** the number of characters written would be a more useful return ** value but we cannot change the implementation of sqlite3_snprintf() ** now without breaking compatibility. ** -** As long as the buffer size is greater than zero, sqlite3_snprintf() -** guarantees that the buffer is always zero-terminated. The first +** ^As long as the buffer size is greater than zero, sqlite3_snprintf() +** guarantees that the buffer is always zero-terminated. ^The first ** parameter "n" is the total size of the buffer, including space for ** the zero terminator. So the longest string that can be completely ** written will be n-1 characters. @@ -1680,9 +1730,9 @@ void sqlite3_free_table(char **result); ** All of the usual printf() formatting options apply. In addition, there ** is are "%q", "%Q", and "%z" options. ** -** The %q option works like %s in that it substitutes a null-terminated +** ^(The %q option works like %s in that it substitutes a null-terminated ** string from the argument list. But %q also doubles every '\'' character. -** %q is designed for use inside a string literal. By doubling each '\'' +** %q is designed for use inside a string literal.)^ By doubling each '\'' ** character it escapes that character and allows it to be inserted into ** the string. ** @@ -1717,10 +1767,10 @@ void sqlite3_free_table(char **result); ** This second example is an SQL syntax error. As a general rule you should ** always use %q instead of %s when inserting text into a string literal. ** -** The %Q option works like %q except it also adds single quotes around +** ^(The %Q option works like %q except it also adds single quotes around ** the outside of the total string. Additionally, if the parameter in the ** argument list is a NULL pointer, %Q substitutes the text "NULL" (without -** single quotes) in place of the %Q option. So, for example, one could say: +** single quotes).)^ So, for example, one could say: ** ** 
 **  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
@@ -1731,35 +1781,32 @@ void sqlite3_free_table(char **result);
 ** The code above will render a correct SQL statement in the zSQL
 ** variable even if the zText variable is a NULL pointer.
 **
-** The "%z" formatting option works exactly like "%s" with the
+** ^(The "%z" formatting option works like "%s" but with the
 ** addition that after the string has been read and copied into
-** the result, [sqlite3_free()] is called on the input string. {END}
-**
-** Requirements:
-** [H17403] [H17406] [H17407]
+** the result, [sqlite3_free()] is called on the input string.)^
 */
 char *sqlite3_mprintf(const char*,...);
 char *sqlite3_vmprintf(const char*, va_list);
 char *sqlite3_snprintf(int,char*,const char*, ...);
 
 /*
-** CAPI3REF: Memory Allocation Subsystem {H17300} 
+** CAPI3REF: Memory Allocation Subsystem
 **
-** The SQLite core  uses these three routines for all of its own
+** The SQLite core uses these three routines for all of its own
 ** internal memory allocation needs. "Core" in the previous sentence
 ** does not include operating-system specific VFS implementation.  The
 ** Windows VFS uses native malloc() and free() for some operations.
 **
-** The sqlite3_malloc() routine returns a pointer to a block
+** ^The sqlite3_malloc() routine returns a pointer to a block
 ** of memory at least N bytes in length, where N is the parameter.
-** If sqlite3_malloc() is unable to obtain sufficient free
-** memory, it returns a NULL pointer.  If the parameter N to
+** ^If sqlite3_malloc() is unable to obtain sufficient free
+** memory, it returns a NULL pointer.  ^If the parameter N to
 ** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns
 ** a NULL pointer.
 **
-** Calling sqlite3_free() with a pointer previously returned
+** ^Calling sqlite3_free() with a pointer previously returned
 ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so
-** that it might be reused.  The sqlite3_free() routine is
+** that it might be reused.  ^The sqlite3_free() routine is
 ** a no-op if is called with a NULL pointer.  Passing a NULL pointer
 ** to sqlite3_free() is harmless.  After being freed, memory
 ** should neither be read nor written.  Even reading previously freed
@@ -1768,34 +1815,25 @@ char *sqlite3_snprintf(int,char*,const char*, ...);
 ** might result if sqlite3_free() is called with a non-NULL pointer that
 ** was not obtained from sqlite3_malloc() or sqlite3_realloc().
 **
-** The sqlite3_realloc() interface attempts to resize a
+** ^(The sqlite3_realloc() interface attempts to resize a
 ** prior memory allocation to be at least N bytes, where N is the
 ** second parameter.  The memory allocation to be resized is the first
-** parameter.  If the first parameter to sqlite3_realloc()
+** parameter.)^ ^ If the first parameter to sqlite3_realloc()
 ** is a NULL pointer then its behavior is identical to calling
 ** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc().
-** If the second parameter to sqlite3_realloc() is zero or
+** ^If the second parameter to sqlite3_realloc() is zero or
 ** negative then the behavior is exactly the same as calling
 ** sqlite3_free(P) where P is the first parameter to sqlite3_realloc().
-** sqlite3_realloc() returns a pointer to a memory allocation
+** ^sqlite3_realloc() returns a pointer to a memory allocation
 ** of at least N bytes in size or NULL if sufficient memory is unavailable.
-** If M is the size of the prior allocation, then min(N,M) bytes
+** ^If M is the size of the prior allocation, then min(N,M) bytes
 ** of the prior allocation are copied into the beginning of buffer returned
 ** by sqlite3_realloc() and the prior allocation is freed.
-** If sqlite3_realloc() returns NULL, then the prior allocation
+** ^If sqlite3_realloc() returns NULL, then the prior allocation
 ** is not freed.
 **
-** The memory returned by sqlite3_malloc() and sqlite3_realloc()
-** is always aligned to at least an 8 byte boundary. {END}
-**
-** The default implementation of the memory allocation subsystem uses
-** the malloc(), realloc() and free() provided by the standard C library.
-** {H17382} However, if SQLite is compiled with the
-** SQLITE_MEMORY_SIZE=NNN C preprocessor macro (where NNN
-** is an integer), then SQLite create a static array of at least
-** NNN bytes in size and uses that array for all of its dynamic
-** memory allocation needs. {END}  Additional memory allocator options
-** may be added in future releases.
+** ^The memory returned by sqlite3_malloc() and sqlite3_realloc()
+** is always aligned to at least an 8 byte boundary.
 **
 ** In SQLite version 3.5.0 and 3.5.1, it was possible to define
 ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in
@@ -1810,10 +1848,6 @@ char *sqlite3_snprintf(int,char*,const char*, ...);
 ** they are reported back as [SQLITE_CANTOPEN] or
 ** [SQLITE_IOERR] rather than [SQLITE_NOMEM].
 **
-** Requirements:
-** [H17303] [H17304] [H17305] [H17306] [H17310] [H17312] [H17315] [H17318]
-** [H17321] [H17322] [H17323]
-**
 ** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()]
 ** must be either NULL or else pointers obtained from a prior
 ** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have
@@ -1828,20 +1862,33 @@ void *sqlite3_realloc(void*, int);
 void sqlite3_free(void*);
 
 /*
-** CAPI3REF: Memory Allocator Statistics {H17370} 
+** CAPI3REF: Memory Allocator Statistics
 **
 ** SQLite provides these two interfaces for reporting on the status
 ** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()]
 ** routines, which form the built-in memory allocation subsystem.
 **
-** Requirements:
-** [H17371] [H17373] [H17374] [H17375]
+** ^The [sqlite3_memory_used()] routine returns the number of bytes
+** of memory currently outstanding (malloced but not freed).
+** ^The [sqlite3_memory_highwater()] routine returns the maximum
+** value of [sqlite3_memory_used()] since the high-water mark
+** was last reset.  ^The values returned by [sqlite3_memory_used()] and
+** [sqlite3_memory_highwater()] include any overhead
+** added by SQLite in its implementation of [sqlite3_malloc()],
+** but not overhead added by the any underlying system library
+** routines that [sqlite3_malloc()] may call.
+**
+** ^The memory high-water mark is reset to the current value of
+** [sqlite3_memory_used()] if and only if the parameter to
+** [sqlite3_memory_highwater()] is true.  ^The value returned
+** by [sqlite3_memory_highwater(1)] is the high-water mark
+** prior to the reset.
 */
 sqlite3_int64 sqlite3_memory_used(void);
 sqlite3_int64 sqlite3_memory_highwater(int resetFlag);
 
 /*
-** CAPI3REF: Pseudo-Random Number Generator {H17390} 
+** CAPI3REF: Pseudo-Random Number Generator
 **
 ** SQLite contains a high-quality pseudo-random number generator (PRNG) used to
 ** select random [ROWID | ROWIDs] when inserting new records into a table that
@@ -1849,60 +1896,57 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag);
 ** the build-in random() and randomblob() SQL functions.  This interface allows
 ** applications to access the same PRNG for other purposes.
 **
-** A call to this routine stores N bytes of randomness into buffer P.
+** ^A call to this routine stores N bytes of randomness into buffer P.
 **
-** The first time this routine is invoked (either internally or by
+** ^The first time this routine is invoked (either internally or by
 ** the application) the PRNG is seeded using randomness obtained
 ** from the xRandomness method of the default [sqlite3_vfs] object.
-** On all subsequent invocations, the pseudo-randomness is generated
+** ^On all subsequent invocations, the pseudo-randomness is generated
 ** internally and without recourse to the [sqlite3_vfs] xRandomness
 ** method.
-**
-** Requirements:
-** [H17392]
 */
 void sqlite3_randomness(int N, void *P);
 
 /*
-** CAPI3REF: Compile-Time Authorization Callbacks {H12500} 
+** CAPI3REF: Compile-Time Authorization Callbacks
 **
-** This routine registers a authorizer callback with a particular
+** ^This routine registers a authorizer callback with a particular
 ** [database connection], supplied in the first argument.
-** The authorizer callback is invoked as SQL statements are being compiled
+** ^The authorizer callback is invoked as SQL statements are being compiled
 ** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()],
-** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()].  At various
+** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()].  ^At various
 ** points during the compilation process, as logic is being created
 ** to perform various actions, the authorizer callback is invoked to
-** see if those actions are allowed.  The authorizer callback should
+** see if those actions are allowed.  ^The authorizer callback should
 ** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the
 ** specific action but allow the SQL statement to continue to be
 ** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be
-** rejected with an error.  If the authorizer callback returns
+** rejected with an error.  ^If the authorizer callback returns
 ** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY]
 ** then the [sqlite3_prepare_v2()] or equivalent call that triggered
 ** the authorizer will fail with an error message.
 **
 ** When the callback returns [SQLITE_OK], that means the operation
-** requested is ok.  When the callback returns [SQLITE_DENY], the
+** requested is ok.  ^When the callback returns [SQLITE_DENY], the
 ** [sqlite3_prepare_v2()] or equivalent call that triggered the
 ** authorizer will fail with an error message explaining that
 ** access is denied. 
 **
-** The first parameter to the authorizer callback is a copy of the third
-** parameter to the sqlite3_set_authorizer() interface. The second parameter
+** ^The first parameter to the authorizer callback is a copy of the third
+** parameter to the sqlite3_set_authorizer() interface. ^The second parameter
 ** to the callback is an integer [SQLITE_COPY | action code] that specifies
-** the particular action to be authorized. The third through sixth parameters
+** the particular action to be authorized. ^The third through sixth parameters
 ** to the callback are zero-terminated strings that contain additional
 ** details about the action to be authorized.
 **
-** If the action code is [SQLITE_READ]
+** ^If the action code is [SQLITE_READ]
 ** and the callback returns [SQLITE_IGNORE] then the
 ** [prepared statement] statement is constructed to substitute
 ** a NULL value in place of the table column that would have
 ** been read if [SQLITE_OK] had been returned.  The [SQLITE_IGNORE]
 ** return can be used to deny an untrusted user access to individual
 ** columns of a table.
-** If the action code is [SQLITE_DELETE] and the callback returns
+** ^If the action code is [SQLITE_DELETE] and the callback returns
 ** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the
 ** [truncate optimization] is disabled and all rows are deleted individually.
 **
@@ -1922,9 +1966,9 @@ void sqlite3_randomness(int N, void *P);
 ** and limiting database size using the [max_page_count] [PRAGMA]
 ** in addition to using an authorizer.
 **
-** Only a single authorizer can be in place on a database connection
+** ^(Only a single authorizer can be in place on a database connection
 ** at a time.  Each call to sqlite3_set_authorizer overrides the
-** previous call.  Disable the authorizer by installing a NULL callback.
+** previous call.)^  ^Disable the authorizer by installing a NULL callback.
 ** The authorizer is disabled by default.
 **
 ** The authorizer callback must not do anything that will modify
@@ -1932,20 +1976,16 @@ void sqlite3_randomness(int N, void *P);
 ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their
 ** database connections for the meaning of "modify" in this paragraph.
 **
-** When [sqlite3_prepare_v2()] is used to prepare a statement, the
+** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the
 ** statement might be re-prepared during [sqlite3_step()] due to a 
 ** schema change.  Hence, the application should ensure that the
 ** correct authorizer callback remains in place during the [sqlite3_step()].
 **
-** Note that the authorizer callback is invoked only during
+** ^Note that the authorizer callback is invoked only during
 ** [sqlite3_prepare()] or its variants.  Authorization is not
 ** performed during statement evaluation in [sqlite3_step()], unless
 ** as stated in the previous paragraph, sqlite3_step() invokes
 ** sqlite3_prepare_v2() to reprepare a statement after a schema change.
-**
-** Requirements:
-** [H12501] [H12502] [H12503] [H12504] [H12505] [H12506] [H12507] [H12510]
-** [H12511] [H12512] [H12520] [H12521] [H12522]
 */
 int sqlite3_set_authorizer(
   sqlite3*,
@@ -1954,7 +1994,7 @@ int sqlite3_set_authorizer(
 );
 
 /*
-** CAPI3REF: Authorizer Return Codes {H12590} 
+** CAPI3REF: Authorizer Return Codes
 **
 ** The [sqlite3_set_authorizer | authorizer callback function] must
 ** return either [SQLITE_OK] or one of these two constants in order
@@ -1966,7 +2006,7 @@ int sqlite3_set_authorizer(
 #define SQLITE_IGNORE 2   /* Don't allow access, but don't generate an error */
 
 /*
-** CAPI3REF: Authorizer Action Codes {H12550} 
+** CAPI3REF: Authorizer Action Codes
 **
 ** The [sqlite3_set_authorizer()] interface registers a callback function
 ** that is invoked to authorize certain SQL statement actions.  The
@@ -1977,15 +2017,12 @@ int sqlite3_set_authorizer(
 ** These action code values signify what kind of operation is to be
 ** authorized.  The 3rd and 4th parameters to the authorization
 ** callback function will be parameters or NULL depending on which of these
-** codes is used as the second parameter.  The 5th parameter to the
+** codes is used as the second parameter.  ^(The 5th parameter to the
 ** authorizer callback is the name of the database ("main", "temp",
-** etc.) if applicable.  The 6th parameter to the authorizer callback
+** etc.) if applicable.)^  ^The 6th parameter to the authorizer callback
 ** is the name of the inner-most trigger or view that is responsible for
 ** the access attempt or NULL if this access attempt is directly from
 ** top-level SQL code.
-**
-** Requirements:
-** [H12551] [H12552] [H12553] [H12554]
 */
 /******************************************* 3rd ************ 4th ***********/
 #define SQLITE_CREATE_INDEX          1   /* Index Name      Table Name      */
@@ -2023,42 +2060,39 @@ int sqlite3_set_authorizer(
 #define SQLITE_COPY                  0   /* No longer used */
 
 /*
-** CAPI3REF: Tracing And Profiling Functions {H12280} 
+** CAPI3REF: Tracing And Profiling Functions
 ** EXPERIMENTAL
 **
 ** These routines register callback functions that can be used for
 ** tracing and profiling the execution of SQL statements.
 **
-** The callback function registered by sqlite3_trace() is invoked at
+** ^The callback function registered by sqlite3_trace() is invoked at
 ** various times when an SQL statement is being run by [sqlite3_step()].
-** The callback returns a UTF-8 rendering of the SQL statement text
-** as the statement first begins executing.  Additional callbacks occur
+** ^The sqlite3_trace() callback is invoked with a UTF-8 rendering of the
+** SQL statement text as the statement first begins executing.
+** ^(Additional sqlite3_trace() callbacks might occur
 ** as each triggered subprogram is entered.  The callbacks for triggers
-** contain a UTF-8 SQL comment that identifies the trigger.
+** contain a UTF-8 SQL comment that identifies the trigger.)^
 **
-** The callback function registered by sqlite3_profile() is invoked
-** as each SQL statement finishes.  The profile callback contains
+** ^The callback function registered by sqlite3_profile() is invoked
+** as each SQL statement finishes.  ^The profile callback contains
 ** the original statement text and an estimate of wall-clock time
 ** of how long that statement took to run.
-**
-** Requirements:
-** [H12281] [H12282] [H12283] [H12284] [H12285] [H12287] [H12288] [H12289]
-** [H12290]
 */
 SQLITE_EXPERIMENTAL void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*);
 SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*,
    void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
 
 /*
-** CAPI3REF: Query Progress Callbacks {H12910} 
+** CAPI3REF: Query Progress Callbacks
 **
-** This routine configures a callback function - the
+** ^This routine configures a callback function - the
 ** progress callback - that is invoked periodically during long
 ** running calls to [sqlite3_exec()], [sqlite3_step()] and
 ** [sqlite3_get_table()].  An example use for this
 ** interface is to keep a GUI updated during a large query.
 **
-** If the progress callback returns non-zero, the operation is
+** ^If the progress callback returns non-zero, the operation is
 ** interrupted.  This feature can be used to implement a
 ** "Cancel" button on a GUI progress dialog box.
 **
@@ -2067,28 +2101,26 @@ SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*,
 ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their
 ** database connections for the meaning of "modify" in this paragraph.
 **
-** Requirements:
-** [H12911] [H12912] [H12913] [H12914] [H12915] [H12916] [H12917] [H12918]
-**
 */
 void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 
 /*
-** CAPI3REF: Opening A New Database Connection {H12700} 
+** CAPI3REF: Opening A New Database Connection
 **
-** These routines open an SQLite database file whose name is given by the
-** filename argument. The filename argument is interpreted as UTF-8 for
+** ^These routines open an SQLite database file whose name is given by the
+** filename argument. ^The filename argument is interpreted as UTF-8 for
 ** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte
-** order for sqlite3_open16(). A [database connection] handle is usually
+** order for sqlite3_open16(). ^(A [database connection] handle is usually
 ** returned in *ppDb, even if an error occurs.  The only exception is that
 ** if SQLite is unable to allocate memory to hold the [sqlite3] object,
 ** a NULL will be written into *ppDb instead of a pointer to the [sqlite3]
-** object. If the database is opened (and/or created) successfully, then
-** [SQLITE_OK] is returned.  Otherwise an [error code] is returned.  The
+** object.)^ ^(If the database is opened (and/or created) successfully, then
+** [SQLITE_OK] is returned.  Otherwise an [error code] is returned.)^ ^The
 ** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain
-** an English language description of the error.
+** an English language description of the error following a failure of any
+** of the sqlite3_open() routines.
 **
-** The default encoding for the database will be UTF-8 if
+** ^The default encoding for the database will be UTF-8 if
 ** sqlite3_open() or sqlite3_open_v2() is called and
 ** UTF-16 in the native byte order if sqlite3_open16() is used.
 **
@@ -2098,25 +2130,26 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **
 ** The sqlite3_open_v2() interface works like sqlite3_open()
 ** except that it accepts two additional parameters for additional control
-** over the new database connection.  The flags parameter can take one of
+** over the new database connection.  ^(The flags parameter to
+** sqlite3_open_v2() can take one of
 ** the following three values, optionally combined with the 
 ** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE],
-** and/or [SQLITE_OPEN_PRIVATECACHE] flags:
+** and/or [SQLITE_OPEN_PRIVATECACHE] flags:)^
 **
 ** 

-** 
[SQLITE_OPEN_READONLY]

+** ^(
[SQLITE_OPEN_READONLY]

 ** 
The database is opened in read-only mode.  If the database does not
-** already exist, an error is returned.

+** already exist, an error is returned.)^
 **
-** 
[SQLITE_OPEN_READWRITE]

+** ^(
[SQLITE_OPEN_READWRITE]

 ** 
The database is opened for reading and writing if possible, or reading
 ** only if the file is write protected by the operating system.  In either
-** case the database must already exist, otherwise an error is returned.

+** case the database must already exist, otherwise an error is returned.)^
 **
-** 
[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]

+** ^(
[SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]

 ** 
The database is opened for reading and writing, and is creates it if
 ** it does not already exist. This is the behavior that is always used for
-** sqlite3_open() and sqlite3_open16().

+** sqlite3_open() and sqlite3_open16().)^
 ** 

 **
 ** If the 3rd parameter to sqlite3_open_v2() is not one of the
@@ -2125,33 +2158,33 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags,
 ** then the behavior is undefined.
 **
-** If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection
+** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection
 ** opens in the multi-thread [threading mode] as long as the single-thread
-** mode has not been set at compile-time or start-time.  If the
+** mode has not been set at compile-time or start-time.  ^If the
 ** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens
 ** in the serialized [threading mode] unless single-thread was
 ** previously selected at compile-time or start-time.
-** The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be
+** ^The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be
 ** eligible to use [shared cache mode], regardless of whether or not shared
-** cache is enabled using [sqlite3_enable_shared_cache()].  The
+** cache is enabled using [sqlite3_enable_shared_cache()].  ^The
 ** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not
 ** participate in [shared cache mode] even if it is enabled.
 **
-** If the filename is ":memory:", then a private, temporary in-memory database
-** is created for the connection.  This in-memory database will vanish when
+** ^If the filename is ":memory:", then a private, temporary in-memory database
+** is created for the connection.  ^This in-memory database will vanish when
 ** the database connection is closed.  Future versions of SQLite might
 ** make use of additional special filenames that begin with the ":" character.
 ** It is recommended that when a database filename actually does begin with
 ** a ":" character you should prefix the filename with a pathname such as
 ** "./" to avoid ambiguity.
 **
-** If the filename is an empty string, then a private, temporary
-** on-disk database will be created.  This private database will be
+** ^If the filename is an empty string, then a private, temporary
+** on-disk database will be created.  ^This private database will be
 ** automatically deleted as soon as the database connection is closed.
 **
-** The fourth parameter to sqlite3_open_v2() is the name of the
+** ^The fourth parameter to sqlite3_open_v2() is the name of the
 ** [sqlite3_vfs] object that defines the operating system interface that
-** the new database connection should use.  If the fourth parameter is
+** the new database connection should use.  ^If the fourth parameter is
 ** a NULL pointer then the default [sqlite3_vfs] object is used.
 **
 ** Note to Windows users:  The encoding used for the filename argument
@@ -2159,10 +2192,6 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** codepage is currently defined.  Filenames containing international
 ** characters must be converted to UTF-8 prior to passing them into
 ** sqlite3_open() or sqlite3_open_v2().
-**
-** Requirements:
-** [H12701] [H12702] [H12703] [H12704] [H12706] [H12707] [H12709] [H12711]
-** [H12712] [H12713] [H12714] [H12717] [H12719] [H12721] [H12723]
 */
 int sqlite3_open(
   const char *filename,   /* Database filename (UTF-8) */
@@ -2180,23 +2209,23 @@ int sqlite3_open_v2(
 );
 
 /*
-** CAPI3REF: Error Codes And Messages {H12800} 
+** CAPI3REF: Error Codes And Messages
 **
-** The sqlite3_errcode() interface returns the numeric [result code] or
+** ^The sqlite3_errcode() interface returns the numeric [result code] or
 ** [extended result code] for the most recent failed sqlite3_* API call
 ** associated with a [database connection]. If a prior API call failed
 ** but the most recent API call succeeded, the return value from
-** sqlite3_errcode() is undefined.  The sqlite3_extended_errcode()
+** sqlite3_errcode() is undefined.  ^The sqlite3_extended_errcode()
 ** interface is the same except that it always returns the 
 ** [extended result code] even when extended result codes are
 ** disabled.
 **
-** The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
+** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
 ** text that describes the error, as either UTF-8 or UTF-16 respectively.
-** Memory to hold the error message string is managed internally.
+** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
-** subsequent calls to other SQLite interface functions.
+** subsequent calls to other SQLite interface functions.)^
 **
 ** When the serialized [threading mode] is in use, it might be the
 ** case that a second error occurs on a separate thread in between
@@ -2211,9 +2240,6 @@ int sqlite3_open_v2(
 ** If an interface fails with SQLITE_MISUSE, that means the interface
 ** was invoked incorrectly by the application.  In that case, the
 ** error code and message may or may not be set.
-**
-** Requirements:
-** [H12801] [H12802] [H12803] [H12807] [H12808] [H12809]
 */
 int sqlite3_errcode(sqlite3 *db);
 int sqlite3_extended_errcode(sqlite3 *db);
@@ -2221,7 +2247,7 @@ const char *sqlite3_errmsg(sqlite3*);
 const void *sqlite3_errmsg16(sqlite3*);
 
 /*
-** CAPI3REF: SQL Statement Object {H13000} 
+** CAPI3REF: SQL Statement Object
 ** KEYWORDS: {prepared statement} {prepared statements}
 **
 ** An instance of this object represents a single SQL statement.
@@ -2247,25 +2273,25 @@ const void *sqlite3_errmsg16(sqlite3*);
 typedef struct sqlite3_stmt sqlite3_stmt;
 
 /*
-** CAPI3REF: Run-time Limits {H12760} 
+** CAPI3REF: Run-time Limits
 **
-** This interface allows the size of various constructs to be limited
+** ^(This interface allows the size of various constructs to be limited
 ** on a connection by connection basis.  The first parameter is the
 ** [database connection] whose limit is to be set or queried.  The
 ** second parameter is one of the [limit categories] that define a
 ** class of constructs to be size limited.  The third parameter is the
-** new limit for that construct.  The function returns the old limit.
+** new limit for that construct.  The function returns the old limit.)^
 **
-** If the new limit is a negative number, the limit is unchanged.
-** For the limit category of SQLITE_LIMIT_XYZ there is a 
+** ^If the new limit is a negative number, the limit is unchanged.
+** ^(For the limit category of SQLITE_LIMIT_XYZ there is a 
 ** [limits | hard upper bound]
 ** set by a compile-time C preprocessor macro named 
 ** [limits | SQLITE_MAX_XYZ].
-** (The "_LIMIT_" in the name is changed to "_MAX_".)
-** Attempts to increase a limit above its hard upper bound are
-** silently truncated to the hard upper limit.
+** (The "_LIMIT_" in the name is changed to "_MAX_".))^
+** ^Attempts to increase a limit above its hard upper bound are
+** silently truncated to the hard upper bound.
 **
-** Run time limits are intended for use in applications that manage
+** Run-time limits are intended for use in applications that manage
 ** both their own internal database and also databases that are controlled
 ** by untrusted external sources.  An example application might be a
 ** web browser that has its own databases for storing history and
@@ -2279,14 +2305,11 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 ** [max_page_count] [PRAGMA].
 **
 ** New run-time limit categories may be added in future releases.
-**
-** Requirements:
-** [H12762] [H12766] [H12769]
 */
 int sqlite3_limit(sqlite3*, int id, int newVal);
 
 /*
-** CAPI3REF: Run-Time Limit Categories {H12790} 
+** CAPI3REF: Run-Time Limit Categories
 ** KEYWORDS: {limit category} {*limit categories}
 **
 ** These constants define various performance limits
@@ -2295,43 +2318,43 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** Additional information is available at [limits | Limits in SQLite].
 **
 ** 

-** 
SQLITE_LIMIT_LENGTH

-** 
The maximum size of any string or BLOB or table row.

+** ^(
SQLITE_LIMIT_LENGTH

+** 
The maximum size of any string or BLOB or table row.
)^
 **
-** 
SQLITE_LIMIT_SQL_LENGTH

-** 
The maximum length of an SQL statement.

+** ^(
SQLITE_LIMIT_SQL_LENGTH

+** 
The maximum length of an SQL statement, in bytes.
)^
 **
-** 
SQLITE_LIMIT_COLUMN

+** ^(
SQLITE_LIMIT_COLUMN

 ** 
The maximum number of columns in a table definition or in the
 ** result set of a [SELECT] or the maximum number of columns in an index
-** or in an ORDER BY or GROUP BY clause.

+** or in an ORDER BY or GROUP BY clause.)^
 **
-** 
SQLITE_LIMIT_EXPR_DEPTH

-** 
The maximum depth of the parse tree on any expression.

+** ^(
SQLITE_LIMIT_EXPR_DEPTH

+** 
The maximum depth of the parse tree on any expression.
)^
 **
-** 
SQLITE_LIMIT_COMPOUND_SELECT

-** 
The maximum number of terms in a compound SELECT statement.

+** ^(
SQLITE_LIMIT_COMPOUND_SELECT

+** 
The maximum number of terms in a compound SELECT statement.
)^
 **
-** 
SQLITE_LIMIT_VDBE_OP

+** ^(
SQLITE_LIMIT_VDBE_OP

 ** 
The maximum number of instructions in a virtual machine program
-** used to implement an SQL statement.

+** used to implement an SQL statement.)^
 **
-** 
SQLITE_LIMIT_FUNCTION_ARG

-** 
The maximum number of arguments on a function.

+** ^(
SQLITE_LIMIT_FUNCTION_ARG

+** 
The maximum number of arguments on a function.
)^
 **
-** 
SQLITE_LIMIT_ATTACHED

-** 
The maximum number of [ATTACH | attached databases].

+** ^(
SQLITE_LIMIT_ATTACHED

+** 
The maximum number of [ATTACH | attached databases].)^

 **
-** 
SQLITE_LIMIT_LIKE_PATTERN_LENGTH

+** ^(
SQLITE_LIMIT_LIKE_PATTERN_LENGTH

 ** 
The maximum length of the pattern argument to the [LIKE] or
-** [GLOB] operators.

+** [GLOB] operators.)^
 **
-** 
SQLITE_LIMIT_VARIABLE_NUMBER

+** ^(
SQLITE_LIMIT_VARIABLE_NUMBER

 ** 
The maximum number of variables in an SQL statement that can
-** be bound.

+** be bound.)^
 **
-** 
SQLITE_LIMIT_TRIGGER_DEPTH

-** 
The maximum depth of recursion for triggers.

+** ^(
SQLITE_LIMIT_TRIGGER_DEPTH

+** 
The maximum depth of recursion for triggers.
)^
 ** 

 */
 #define SQLITE_LIMIT_LENGTH                    0
@@ -2347,7 +2370,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 #define SQLITE_LIMIT_TRIGGER_DEPTH            10
 
 /*
-** CAPI3REF: Compiling An SQL Statement {H13010} 
+** CAPI3REF: Compiling An SQL Statement
 ** KEYWORDS: {SQL statement compiler}
 **
 ** To execute an SQL query, it must first be compiled into a byte-code
@@ -2362,9 +2385,9 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2()
 ** use UTF-16.
 **
-** If the nByte argument is less than zero, then zSql is read up to the
-** first zero terminator. If nByte is non-negative, then it is the maximum
-** number of  bytes read from zSql.  When nByte is non-negative, the
+** ^If the nByte argument is less than zero, then zSql is read up to the
+** first zero terminator. ^If nByte is non-negative, then it is the maximum
+** number of  bytes read from zSql.  ^When nByte is non-negative, the
 ** zSql string ends at either the first '\000' or '\u0000' character or
 ** the nByte-th byte, whichever comes first. If the caller knows
 ** that the supplied string is nul-terminated, then there is a small
@@ -2372,34 +2395,35 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** is equal to the number of bytes in the input string including
 ** the nul-terminator bytes.
 **
-** If pzTail is not NULL then *pzTail is made to point to the first byte
+** ^If pzTail is not NULL then *pzTail is made to point to the first byte
 ** past the end of the first SQL statement in zSql.  These routines only
 ** compile the first statement in zSql, so *pzTail is left pointing to
 ** what remains uncompiled.
 **
-** *ppStmt is left pointing to a compiled [prepared statement] that can be
-** executed using [sqlite3_step()].  If there is an error, *ppStmt is set
-** to NULL.  If the input text contains no SQL (if the input is an empty
+** ^*ppStmt is left pointing to a compiled [prepared statement] that can be
+** executed using [sqlite3_step()].  ^If there is an error, *ppStmt is set
+** to NULL.  ^If the input text contains no SQL (if the input is an empty
 ** string or a comment) then *ppStmt is set to NULL.
 ** The calling procedure is responsible for deleting the compiled
 ** SQL statement using [sqlite3_finalize()] after it has finished with it.
 ** ppStmt may not be NULL.
 **
-** On success, [SQLITE_OK] is returned, otherwise an [error code] is returned.
+** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK];
+** otherwise an [error code] is returned.
 **
 ** The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are
 ** recommended for all new programs. The two older interfaces are retained
 ** for backwards compatibility, but their use is discouraged.
-** In the "v2" interfaces, the prepared statement
+** ^In the "v2" interfaces, the prepared statement
 ** that is returned (the [sqlite3_stmt] object) contains a copy of the
 ** original SQL text. This causes the [sqlite3_step()] interface to
 ** behave differently in three ways:
 **
 ** 
    
 ** 
  1. 
-** If the database schema changes, instead of returning [SQLITE_SCHEMA] as it
+** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it
 ** always used to do, [sqlite3_step()] will automatically recompile the SQL
-** statement and try to run it again.  If the schema has changed in
+** statement and try to run it again.  ^If the schema has changed in
 ** a way that makes the statement no longer valid, [sqlite3_step()] will still
 ** return [SQLITE_SCHEMA].  But unlike the legacy behavior, [SQLITE_SCHEMA] is
 ** now a fatal error.  Calling [sqlite3_prepare_v2()] again will not make the
@@ -2408,11 +2432,11 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** 
    2. 
 **
 ** 
  2. 
-** When an error occurs, [sqlite3_step()] will return one of the detailed
-** [error codes] or [extended error codes].  The legacy behavior was that
+** ^When an error occurs, [sqlite3_step()] will return one of the detailed
+** [error codes] or [extended error codes].  ^The legacy behavior was that
 ** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code
-** and you would have to make a second call to [sqlite3_reset()] in order
-** to find the underlying cause of the problem. With the "v2" prepare
+** and the application would have to make a second call to [sqlite3_reset()]
+** in order to find the underlying cause of the problem. With the "v2" prepare
 ** interfaces, the underlying reason for the error is returned immediately.
 ** 
    3. 
 **
@@ -2424,10 +2448,6 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** [sqlite3_bind_text | bindings] of the [parameter]. 
 ** 
 ** 

-**
-** Requirements:
-** [H13011] [H13012] [H13013] [H13014] [H13015] [H13016] [H13019] [H13021]
-**
 */
 int sqlite3_prepare(
   sqlite3 *db,            /* Database handle */
@@ -2459,24 +2479,21 @@ int sqlite3_prepare16_v2(
 );
 
 /*
-** CAPI3REF: Retrieving Statement SQL {H13100} 
+** CAPI3REF: Retrieving Statement SQL
 **
-** This interface can be used to retrieve a saved copy of the original
+** ^This interface can be used to retrieve a saved copy of the original
 ** SQL text used to create a [prepared statement] if that statement was
 ** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()].
-**
-** Requirements:
-** [H13101] [H13102] [H13103]
 */
 const char *sqlite3_sql(sqlite3_stmt *pStmt);
 
 /*
-** CAPI3REF: Dynamically Typed Value Object {H15000} 
+** CAPI3REF: Dynamically Typed Value Object
 ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value}
 **
 ** SQLite uses the sqlite3_value object to represent all values
 ** that can be stored in a database table. SQLite uses dynamic typing
-** for the values it stores. Values stored in sqlite3_value objects
+** for the values it stores.  ^Values stored in sqlite3_value objects
 ** can be integers, floating point values, strings, BLOBs, or NULL.
 **
 ** An sqlite3_value object may be either "protected" or "unprotected".
@@ -2498,9 +2515,9 @@ const char *sqlite3_sql(sqlite3_stmt *pStmt);
 ** still make the distinction between between protected and unprotected
 ** sqlite3_value objects even when not strictly required.
 **
-** The sqlite3_value objects that are passed as parameters into the
+** ^The sqlite3_value objects that are passed as parameters into the
 ** implementation of [application-defined SQL functions] are protected.
-** The sqlite3_value object returned by
+** ^The sqlite3_value object returned by
 ** [sqlite3_column_value()] is unprotected.
 ** Unprotected sqlite3_value objects may only be used with
 ** [sqlite3_result_value()] and [sqlite3_bind_value()].
@@ -2510,10 +2527,10 @@ const char *sqlite3_sql(sqlite3_stmt *pStmt);
 typedef struct Mem sqlite3_value;
 
 /*
-** CAPI3REF: SQL Function Context Object {H16001} 
+** CAPI3REF: SQL Function Context Object
 **
 ** The context in which an SQL function executes is stored in an
-** sqlite3_context object.  A pointer to an sqlite3_context object
+** sqlite3_context object.  ^A pointer to an sqlite3_context object
 ** is always first parameter to [application-defined SQL functions].
 ** The application-defined SQL function implementation will pass this
 ** pointer through into calls to [sqlite3_result_int | sqlite3_result()],
@@ -2524,11 +2541,11 @@ typedef struct Mem sqlite3_value;
 typedef struct sqlite3_context sqlite3_context;
 
 /*
-** CAPI3REF: Binding Values To Prepared Statements {H13500} 
+** CAPI3REF: Binding Values To Prepared Statements
 ** KEYWORDS: {host parameter} {host parameters} {host parameter name}
 ** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding}
 **
-** In the SQL strings input to [sqlite3_prepare_v2()] and its variants,
+** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants,
 ** literals may be replaced by a [parameter] that matches one of following
 ** templates:
 **
@@ -2541,72 +2558,66 @@ typedef struct sqlite3_context sqlite3_context;
 ** 
 **
 ** In the templates above, NNN represents an integer literal,
-** and VVV represents an alphanumeric identifer.  The values of these
+** and VVV represents an alphanumeric identifer.)^  ^The values of these
 ** parameters (also called "host parameter names" or "SQL parameters")
 ** can be set using the sqlite3_bind_*() routines defined here.
 **
-** The first argument to the sqlite3_bind_*() routines is always
+** ^The first argument to the sqlite3_bind_*() routines is always
 ** a pointer to the [sqlite3_stmt] object returned from
 ** [sqlite3_prepare_v2()] or its variants.
 **
-** The second argument is the index of the SQL parameter to be set.
-** The leftmost SQL parameter has an index of 1.  When the same named
+** ^The second argument is the index of the SQL parameter to be set.
+** ^The leftmost SQL parameter has an index of 1.  ^When the same named
 ** SQL parameter is used more than once, second and subsequent
 ** occurrences have the same index as the first occurrence.
-** The index for named parameters can be looked up using the
-** [sqlite3_bind_parameter_index()] API if desired.  The index
+** ^The index for named parameters can be looked up using the
+** [sqlite3_bind_parameter_index()] API if desired.  ^The index
 ** for "?NNN" parameters is the value of NNN.
-** The NNN value must be between 1 and the [sqlite3_limit()]
+** ^The NNN value must be between 1 and the [sqlite3_limit()]
 ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999).
 **
-** The third argument is the value to bind to the parameter.
+** ^The third argument is the value to bind to the parameter.
 **
-** In those routines that have a fourth argument, its value is the
+** ^(In those routines that have a fourth argument, its value is the
 ** number of bytes in the parameter.  To be clear: the value is the
-** number of bytes in the value, not the number of characters.
-** If the fourth parameter is negative, the length of the string is
+** number of bytes in the value, not the number of characters.)^
+** ^If the fourth parameter is negative, the length of the string is
 ** the number of bytes up to the first zero terminator.
 **
-** The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
+** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
 ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
-** string after SQLite has finished with it. If the fifth argument is
+** string after SQLite has finished with it. ^If the fifth argument is
 ** the special value [SQLITE_STATIC], then SQLite assumes that the
 ** information is in static, unmanaged space and does not need to be freed.
-** If the fifth argument has the value [SQLITE_TRANSIENT], then
+** ^If the fifth argument has the value [SQLITE_TRANSIENT], then
 ** SQLite makes its own private copy of the data immediately, before
 ** the sqlite3_bind_*() routine returns.
 **
-** The sqlite3_bind_zeroblob() routine binds a BLOB of length N that
-** is filled with zeroes.  A zeroblob uses a fixed amount of memory
+** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that
+** is filled with zeroes.  ^A zeroblob uses a fixed amount of memory
 ** (just an integer to hold its size) while it is being processed.
 ** Zeroblobs are intended to serve as placeholders for BLOBs whose
 ** content is later written using
 ** [sqlite3_blob_open | incremental BLOB I/O] routines.
-** A negative value for the zeroblob results in a zero-length BLOB.
+** ^A negative value for the zeroblob results in a zero-length BLOB.
 **
-** The sqlite3_bind_*() routines must be called after
-** [sqlite3_prepare_v2()] (and its variants) or [sqlite3_reset()] and
-** before [sqlite3_step()].
-** Bindings are not cleared by the [sqlite3_reset()] routine.
-** Unbound parameters are interpreted as NULL.
+** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer
+** for the [prepared statement] or with a prepared statement for which
+** [sqlite3_step()] has been called more recently than [sqlite3_reset()],
+** then the call will return [SQLITE_MISUSE].  If any sqlite3_bind_()
+** routine is passed a [prepared statement] that has been finalized, the
+** result is undefined and probably harmful.
 **
-** These routines return [SQLITE_OK] on success or an error code if
-** anything goes wrong.  [SQLITE_RANGE] is returned if the parameter
-** index is out of range.  [SQLITE_NOMEM] is returned if malloc() fails.
-** [SQLITE_MISUSE] might be returned if these routines are called on a
-** virtual machine that is the wrong state or which has already been finalized.
-** Detection of misuse is unreliable.  Applications should not depend
-** on SQLITE_MISUSE returns.  SQLITE_MISUSE is intended to indicate a
-** a logic error in the application.  Future versions of SQLite might
-** panic rather than return SQLITE_MISUSE.
+** ^Bindings are not cleared by the [sqlite3_reset()] routine.
+** ^Unbound parameters are interpreted as NULL.
+**
+** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an
+** [error code] if anything goes wrong.
+** ^[SQLITE_RANGE] is returned if the parameter
+** index is out of range.  ^[SQLITE_NOMEM] is returned if malloc() fails.
 **
 ** See also: [sqlite3_bind_parameter_count()],
 ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()].
-**
-** Requirements:
-** [H13506] [H13509] [H13512] [H13515] [H13518] [H13521] [H13524] [H13527]
-** [H13530] [H13533] [H13536] [H13539] [H13542] [H13545] [H13548] [H13551]
-**
 */
 int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
 int sqlite3_bind_double(sqlite3_stmt*, int, double);
@@ -2619,45 +2630,42 @@ int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
 int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
 
 /*
-** CAPI3REF: Number Of SQL Parameters {H13600} 
+** CAPI3REF: Number Of SQL Parameters
 **
-** This routine can be used to find the number of [SQL parameters]
+** ^This routine can be used to find the number of [SQL parameters]
 ** in a [prepared statement].  SQL parameters are tokens of the
 ** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as
 ** placeholders for values that are [sqlite3_bind_blob | bound]
 ** to the parameters at a later time.
 **
-** This routine actually returns the index of the largest (rightmost)
+** ^(This routine actually returns the index of the largest (rightmost)
 ** parameter. For all forms except ?NNN, this will correspond to the
-** number of unique parameters.  If parameters of the ?NNN are used,
-** there may be gaps in the list.
+** number of unique parameters.  If parameters of the ?NNN form are used,
+** there may be gaps in the list.)^
 **
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_name()], and
 ** [sqlite3_bind_parameter_index()].
-**
-** Requirements:
-** [H13601]
 */
 int sqlite3_bind_parameter_count(sqlite3_stmt*);
 
 /*
-** CAPI3REF: Name Of A Host Parameter {H13620} 
+** CAPI3REF: Name Of A Host Parameter
 **
-** This routine returns a pointer to the name of the n-th
-** [SQL parameter] in a [prepared statement].
-** SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA"
+** ^The sqlite3_bind_parameter_name(P,N) interface returns
+** the name of the N-th [SQL parameter] in the [prepared statement] P.
+** ^(SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA"
 ** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA"
 ** respectively.
 ** In other words, the initial ":" or "$" or "@" or "?"
-** is included as part of the name.
-** Parameters of the form "?" without a following integer have no name
-** and are also referred to as "anonymous parameters".
+** is included as part of the name.)^
+** ^Parameters of the form "?" without a following integer have no name
+** and are referred to as "nameless" or "anonymous parameters".
 **
-** The first host parameter has an index of 1, not 0.
+** ^The first host parameter has an index of 1, not 0.
 **
-** If the value n is out of range or if the n-th parameter is
-** nameless, then NULL is returned.  The returned string is
+** ^If the value N is out of range or if the N-th parameter is
+** nameless, then NULL is returned.  ^The returned string is
 ** always in UTF-8 encoding even if the named parameter was
 ** originally specified as UTF-16 in [sqlite3_prepare16()] or
 ** [sqlite3_prepare16_v2()].
@@ -2665,125 +2673,108 @@ int sqlite3_bind_parameter_count(sqlite3_stmt*);
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_count()], and
 ** [sqlite3_bind_parameter_index()].
-**
-** Requirements:
-** [H13621]
 */
 const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
 
 /*
-** CAPI3REF: Index Of A Parameter With A Given Name {H13640} 
+** CAPI3REF: Index Of A Parameter With A Given Name
 **
-** Return the index of an SQL parameter given its name.  The
+** ^Return the index of an SQL parameter given its name.  ^The
 ** index value returned is suitable for use as the second
-** parameter to [sqlite3_bind_blob|sqlite3_bind()].  A zero
-** is returned if no matching parameter is found.  The parameter
+** parameter to [sqlite3_bind_blob|sqlite3_bind()].  ^A zero
+** is returned if no matching parameter is found.  ^The parameter
 ** name must be given in UTF-8 even if the original statement
 ** was prepared from UTF-16 text using [sqlite3_prepare16_v2()].
 **
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_count()], and
 ** [sqlite3_bind_parameter_index()].
-**
-** Requirements:
-** [H13641]
 */
 int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName);
 
 /*
-** CAPI3REF: Reset All Bindings On A Prepared Statement {H13660} 
+** CAPI3REF: Reset All Bindings On A Prepared Statement
 **
-** Contrary to the intuition of many, [sqlite3_reset()] does not reset
+** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset
 ** the [sqlite3_bind_blob | bindings] on a [prepared statement].
-** Use this routine to reset all host parameters to NULL.
-**
-** Requirements:
-** [H13661]
+** ^Use this routine to reset all host parameters to NULL.
 */
 int sqlite3_clear_bindings(sqlite3_stmt*);
 
 /*
-** CAPI3REF: Number Of Columns In A Result Set {H13710} 
+** CAPI3REF: Number Of Columns In A Result Set
 **
-** Return the number of columns in the result set returned by the
-** [prepared statement]. This routine returns 0 if pStmt is an SQL
+** ^Return the number of columns in the result set returned by the
+** [prepared statement]. ^This routine returns 0 if pStmt is an SQL
 ** statement that does not return data (for example an [UPDATE]).
-**
-** Requirements:
-** [H13711]
 */
 int sqlite3_column_count(sqlite3_stmt *pStmt);
 
 /*
-** CAPI3REF: Column Names In A Result Set {H13720} 
+** CAPI3REF: Column Names In A Result Set
 **
-** These routines return the name assigned to a particular column
-** in the result set of a [SELECT] statement.  The sqlite3_column_name()
+** ^These routines return the name assigned to a particular column
+** in the result set of a [SELECT] statement.  ^The sqlite3_column_name()
 ** interface returns a pointer to a zero-terminated UTF-8 string
 ** and sqlite3_column_name16() returns a pointer to a zero-terminated
-** UTF-16 string.  The first parameter is the [prepared statement]
-** that implements the [SELECT] statement. The second parameter is the
-** column number.  The leftmost column is number 0.
+** UTF-16 string.  ^The first parameter is the [prepared statement]
+** that implements the [SELECT] statement. ^The second parameter is the
+** column number.  ^The leftmost column is number 0.
 **
-** The returned string pointer is valid until either the [prepared statement]
+** ^The returned string pointer is valid until either the [prepared statement]
 ** is destroyed by [sqlite3_finalize()] or until the next call to
 ** sqlite3_column_name() or sqlite3_column_name16() on the same column.
 **
-** If sqlite3_malloc() fails during the processing of either routine
+** ^If sqlite3_malloc() fails during the processing of either routine
 ** (for example during a conversion from UTF-8 to UTF-16) then a
 ** NULL pointer is returned.
 **
-** The name of a result column is the value of the "AS" clause for
+** ^The name of a result column is the value of the "AS" clause for
 ** that column, if there is an AS clause.  If there is no AS clause
 ** then the name of the column is unspecified and may change from
 ** one release of SQLite to the next.
-**
-** Requirements:
-** [H13721] [H13723] [H13724] [H13725] [H13726] [H13727]
 */
 const char *sqlite3_column_name(sqlite3_stmt*, int N);
 const void *sqlite3_column_name16(sqlite3_stmt*, int N);
 
 /*
-** CAPI3REF: Source Of Data In A Query Result {H13740} 
+** CAPI3REF: Source Of Data In A Query Result
 **
-** These routines provide a means to determine what column of what
-** table in which database a result of a [SELECT] statement comes from.
-** The name of the database or table or column can be returned as
-** either a UTF-8 or UTF-16 string.  The _database_ routines return
+** ^These routines provide a means to determine the database, table, and
+** table column that is the origin of a particular result column in
+** [SELECT] statement.
+** ^The name of the database or table or column can be returned as
+** either a UTF-8 or UTF-16 string.  ^The _database_ routines return
 ** the database name, the _table_ routines return the table name, and
 ** the origin_ routines return the column name.
-** The returned string is valid until the [prepared statement] is destroyed
+** ^The returned string is valid until the [prepared statement] is destroyed
 ** using [sqlite3_finalize()] or until the same information is requested
 ** again in a different encoding.
 **
-** The names returned are the original un-aliased names of the
+** ^The names returned are the original un-aliased names of the
 ** database, table, and column.
 **
-** The first argument to the following calls is a [prepared statement].
-** These functions return information about the Nth column returned by
+** ^The first argument to these interfaces is a [prepared statement].
+** ^These functions return information about the Nth result column returned by
 ** the statement, where N is the second function argument.
+** ^The left-most column is column 0 for these routines.
 **
-** If the Nth column returned by the statement is an expression or
+** ^If the Nth column returned by the statement is an expression or
 ** subquery and is not a column value, then all of these functions return
-** NULL.  These routine might also return NULL if a memory allocation error
-** occurs.  Otherwise, they return the name of the attached database, table
-** and column that query result column was extracted from.
+** NULL.  ^These routine might also return NULL if a memory allocation error
+** occurs.  ^Otherwise, they return the name of the attached database, table,
+** or column that query result column was extracted from.
 **
-** As with all other SQLite APIs, those postfixed with "16" return
-** UTF-16 encoded strings, the other functions return UTF-8. {END}
+** ^As with all other SQLite APIs, those whose names end with "16" return
+** UTF-16 encoded strings and the other functions return UTF-8.
 **
-** These APIs are only available if the library was compiled with the
-** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined.
+** ^These APIs are only available if the library was compiled with the
+** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol.
 **
-** {A13751}
 ** If two or more threads call one or more of these routines against the same
 ** prepared statement and column at the same time then the results are
 ** undefined.
 **
-** Requirements:
-** [H13741] [H13742] [H13743] [H13744] [H13745] [H13746] [H13748]
-**
 ** If two or more threads call one or more
 ** [sqlite3_column_database_name | column metadata interfaces]
 ** for the same [prepared statement] and result column
@@ -2797,17 +2788,17 @@ const char *sqlite3_column_origin_name(sqlite3_stmt*,int);
 const void *sqlite3_column_origin_name16(sqlite3_stmt*,int);
 
 /*
-** CAPI3REF: Declared Datatype Of A Query Result {H13760} 
+** CAPI3REF: Declared Datatype Of A Query Result
 **
-** The first parameter is a [prepared statement].
+** ^(The first parameter is a [prepared statement].
 ** If this statement is a [SELECT] statement and the Nth column of the
 ** returned result set of that [SELECT] is a table column (not an
 ** expression or subquery) then the declared type of the table
-** column is returned.  If the Nth column of the result set is an
+** column is returned.)^  ^If the Nth column of the result set is an
 ** expression or subquery, then a NULL pointer is returned.
-** The returned string is always UTF-8 encoded. {END}
+** ^The returned string is always UTF-8 encoded.
 **
-** For example, given the database schema:
+** ^(For example, given the database schema:
 **
 ** CREATE TABLE t1(c1 VARIANT);
 **
@@ -2816,23 +2807,20 @@ const void *sqlite3_column_origin_name16(sqlite3_stmt*,int);
 ** SELECT c1 + 1, c1 FROM t1;
 **
 ** this routine would return the string "VARIANT" for the second result
-** column (i==1), and a NULL pointer for the first result column (i==0).
+** column (i==1), and a NULL pointer for the first result column (i==0).)^
 **
-** SQLite uses dynamic run-time typing.  So just because a column
+** ^SQLite uses dynamic run-time typing.  ^So just because a column
 ** is declared to contain a particular type does not mean that the
 ** data stored in that column is of the declared type.  SQLite is
-** strongly typed, but the typing is dynamic not static.  Type
+** strongly typed, but the typing is dynamic not static.  ^Type
 ** is associated with individual values, not with the containers
 ** used to hold those values.
-**
-** Requirements:
-** [H13761] [H13762] [H13763]
 */
 const char *sqlite3_column_decltype(sqlite3_stmt*,int);
 const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 
 /*
-** CAPI3REF: Evaluate An SQL Statement {H13200} 
+** CAPI3REF: Evaluate An SQL Statement
 **
 ** After a [prepared statement] has been prepared using either
 ** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy
@@ -2846,35 +2834,35 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 ** new "v2" interface is recommended for new applications but the legacy
 ** interface will continue to be supported.
 **
-** In the legacy interface, the return value will be either [SQLITE_BUSY],
+** ^In the legacy interface, the return value will be either [SQLITE_BUSY],
 ** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE].
-** With the "v2" interface, any of the other [result codes] or
+** ^With the "v2" interface, any of the other [result codes] or
 ** [extended result codes] might be returned as well.
 **
-** [SQLITE_BUSY] means that the database engine was unable to acquire the
-** database locks it needs to do its job.  If the statement is a [COMMIT]
+** ^[SQLITE_BUSY] means that the database engine was unable to acquire the
+** database locks it needs to do its job.  ^If the statement is a [COMMIT]
 ** or occurs outside of an explicit transaction, then you can retry the
 ** statement.  If the statement is not a [COMMIT] and occurs within a
 ** explicit transaction then you should rollback the transaction before
 ** continuing.
 **
-** [SQLITE_DONE] means that the statement has finished executing
+** ^[SQLITE_DONE] means that the statement has finished executing
 ** successfully.  sqlite3_step() should not be called again on this virtual
 ** machine without first calling [sqlite3_reset()] to reset the virtual
 ** machine back to its initial state.
 **
-** If the SQL statement being executed returns any data, then [SQLITE_ROW]
+** ^If the SQL statement being executed returns any data, then [SQLITE_ROW]
 ** is returned each time a new row of data is ready for processing by the
 ** caller. The values may be accessed using the [column access functions].
 ** sqlite3_step() is called again to retrieve the next row of data.
 **
-** [SQLITE_ERROR] means that a run-time error (such as a constraint
+** ^[SQLITE_ERROR] means that a run-time error (such as a constraint
 ** violation) has occurred.  sqlite3_step() should not be called again on
 ** the VM. More information may be found by calling [sqlite3_errmsg()].
-** With the legacy interface, a more specific error code (for example,
+** ^With the legacy interface, a more specific error code (for example,
 ** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth)
 ** can be obtained by calling [sqlite3_reset()] on the
-** [prepared statement].  In the "v2" interface,
+** [prepared statement].  ^In the "v2" interface,
 ** the more specific error code is returned directly by sqlite3_step().
 **
 ** [SQLITE_MISUSE] means that the this routine was called inappropriately.
@@ -2895,27 +2883,22 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 ** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces,
 ** then the more specific [error codes] are returned directly
 ** by sqlite3_step().  The use of the "v2" interface is recommended.
-**
-** Requirements:
-** [H13202] [H15304] [H15306] [H15308] [H15310]
 */
 int sqlite3_step(sqlite3_stmt*);
 
 /*
-** CAPI3REF: Number of columns in a result set {H13770} 
+** CAPI3REF: Number of columns in a result set
 **
-** Returns the number of values in the current row of the result set.
-**
-** Requirements:
-** [H13771] [H13772]
+** ^The sqlite3_data_count(P) the number of columns in the
+** of the result set of [prepared statement] P.
 */
 int sqlite3_data_count(sqlite3_stmt *pStmt);
 
 /*
-** CAPI3REF: Fundamental Datatypes {H10265} 
+** CAPI3REF: Fundamental Datatypes
 ** KEYWORDS: SQLITE_TEXT
 **
-** {H10266} Every value in SQLite has one of five fundamental datatypes:
+** ^(Every value in SQLite has one of five fundamental datatypes:
 **
 ** 
    
 ** 
  •  64-bit signed integer
@@ -2923,7 +2906,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** 
  •  string
 ** 
  •  BLOB
 ** 
  •  NULL
-** 
 {END}
+** )^
 **
 ** These constants are codes for each of those types.
 **
@@ -2944,18 +2927,18 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 #define SQLITE3_TEXT     3
 
 /*
-** CAPI3REF: Result Values From A Query {H13800} 
+** CAPI3REF: Result Values From A Query
 ** KEYWORDS: {column access functions}
 **
-** These routines form the "result set query" interface.
+** These routines form the "result set" interface.
 **
-** These routines return information about a single column of the current
-** result row of a query.  In every case the first argument is a pointer
+** ^These routines return information about a single column of the current
+** result row of a query.  ^In every case the first argument is a pointer
 ** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*]
 ** that was returned from [sqlite3_prepare_v2()] or one of its variants)
 ** and the second argument is the index of the column for which information
-** should be returned.  The leftmost column of the result set has the index 0.
-** The number of columns in the result can be determined using
+** should be returned. ^The leftmost column of the result set has the index 0.
+** ^The number of columns in the result can be determined using
 ** [sqlite3_column_count()].
 **
 ** If the SQL statement does not currently point to a valid row, or if the
@@ -2970,9 +2953,9 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** are called from a different thread while any of these routines
 ** are pending, then the results are undefined.
 **
-** The sqlite3_column_type() routine returns the
+** ^The sqlite3_column_type() routine returns the
 ** [SQLITE_INTEGER | datatype code] for the initial data type
-** of the result column.  The returned value is one of [SQLITE_INTEGER],
+** of the result column.  ^The returned value is one of [SQLITE_INTEGER],
 ** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].  The value
 ** returned by sqlite3_column_type() is only meaningful if no type
 ** conversions have occurred as described below.  After a type conversion,
@@ -2980,27 +2963,27 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** versions of SQLite may change the behavior of sqlite3_column_type()
 ** following a type conversion.
 **
-** If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes()
+** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes()
 ** routine returns the number of bytes in that BLOB or string.
-** If the result is a UTF-16 string, then sqlite3_column_bytes() converts
+** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts
 ** the string to UTF-8 and then returns the number of bytes.
-** If the result is a numeric value then sqlite3_column_bytes() uses
+** ^If the result is a numeric value then sqlite3_column_bytes() uses
 ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns
 ** the number of bytes in that string.
-** The value returned does not include the zero terminator at the end
-** of the string.  For clarity: the value returned is the number of
+** ^The value returned does not include the zero terminator at the end
+** of the string.  ^For clarity: the value returned is the number of
 ** bytes in the string, not the number of characters.
 **
-** Strings returned by sqlite3_column_text() and sqlite3_column_text16(),
-** even empty strings, are always zero terminated.  The return
+** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(),
+** even empty strings, are always zero terminated.  ^The return
 ** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary
 ** pointer, possibly even a NULL pointer.
 **
-** The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes()
+** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes()
 ** but leaves the result in UTF-16 in native byte order instead of UTF-8.
-** The zero terminator is not included in this count.
+** ^The zero terminator is not included in this count.
 **
-** The object returned by [sqlite3_column_value()] is an
+** ^The object returned by [sqlite3_column_value()] is an
 ** [unprotected sqlite3_value] object.  An unprotected sqlite3_value object
 ** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()].
 ** If the [unprotected sqlite3_value] object returned by
@@ -3008,10 +2991,10 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** to routines like [sqlite3_value_int()], [sqlite3_value_text()],
 ** or [sqlite3_value_bytes()], then the behavior is undefined.
 **
-** These routines attempt to convert the value where appropriate.  For
+** These routines attempt to convert the value where appropriate.  ^For
 ** example, if the internal representation is FLOAT and a text result
 ** is requested, [sqlite3_snprintf()] is used internally to perform the
-** conversion automatically.  The following table details the conversions
+** conversion automatically.  ^(The following table details the conversions
 ** that are applied:
 **
 ** 

@@ -3035,7 +3018,7 @@ int sqlite3_data_count(sqlite3_stmt *pStmt);
 **   BLOB      FLOAT     Convert to TEXT then use atof()
 **   BLOB       TEXT     Add a zero terminator if needed
 ** 
-** 

+**
)^ ** ** The table above makes reference to standard C library functions atoi() ** and atof(). SQLite does not really use these functions. It has its @@ -3043,10 +3026,10 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** Note that when type conversions occur, pointers returned by prior +** ^Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** Type conversions and pointer invalidations might occur +** ^(Type conversions and pointer invalidations might occur ** in the following cases: ** **
    @@ -3059,22 +3042,22 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); **
  • The initial content is UTF-16 text and sqlite3_column_bytes() or ** sqlite3_column_text() is called. The content must be converted ** to UTF-8.
    • -**
+** )^ ** -** Conversions between UTF-16be and UTF-16le are always done in place and do +** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer ** that the prior pointer points to will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** ^(The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** **
    **
  • sqlite3_column_text() followed by sqlite3_column_bytes()
    • **
  • sqlite3_column_blob() followed by sqlite3_column_bytes()
    • **
  • sqlite3_column_text16() followed by sqlite3_column_bytes16()
    • -**
+** )^ ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result @@ -3084,22 +3067,18 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); ** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() ** with calls to sqlite3_column_bytes(). ** -** The pointers returned are valid until a type conversion occurs as +** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or -** [sqlite3_finalize()] is called. The memory space used to hold strings +** [sqlite3_finalize()] is called. ^The memory space used to hold strings ** and BLOBs is freed automatically. Do not pass the pointers returned ** [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** -** If a memory allocation error occurs during the evaluation of any +** ^(If a memory allocation error occurs during the evaluation of any ** of these routines, a default value is returned. The default value ** is either the integer 0, the floating point number 0.0, or a NULL ** pointer. Subsequent calls to [sqlite3_errcode()] will return -** [SQLITE_NOMEM]. -** -** Requirements: -** [H13803] [H13806] [H13809] [H13812] [H13815] [H13818] [H13821] [H13824] -** [H13827] [H13830] +** [SQLITE_NOMEM].)^ */ const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); int sqlite3_column_bytes(sqlite3_stmt*, int iCol); @@ -3113,79 +3092,76 @@ int sqlite3_column_type(sqlite3_stmt*, int iCol); sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); /* -** CAPI3REF: Destroy A Prepared Statement Object {H13300} +** CAPI3REF: Destroy A Prepared Statement Object ** -** The sqlite3_finalize() function is called to delete a [prepared statement]. -** If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. If execution of the statement failed then an +** ^The sqlite3_finalize() function is called to delete a [prepared statement]. +** ^If the statement was executed successfully or not executed at all, then +** SQLITE_OK is returned. ^If execution of the statement failed then an ** [error code] or [extended error code] is returned. ** -** This routine can be called at any point during the execution of the -** [prepared statement]. If the virtual machine has not +** ^This routine can be called at any point during the execution of the +** [prepared statement]. ^If the virtual machine has not ** completed execution when this routine is called, that is like ** encountering an error or an [sqlite3_interrupt | interrupt]. -** Incomplete updates may be rolled back and transactions canceled, +** ^Incomplete updates may be rolled back and transactions canceled, ** depending on the circumstances, and the ** [error code] returned will be [SQLITE_ABORT]. -** -** Requirements: -** [H11302] [H11304] */ int sqlite3_finalize(sqlite3_stmt *pStmt); /* -** CAPI3REF: Reset A Prepared Statement Object {H13330} +** CAPI3REF: Reset A Prepared Statement Object ** ** The sqlite3_reset() function is called to reset a [prepared statement] ** object back to its initial state, ready to be re-executed. -** Any SQL statement variables that had values bound to them using +** ^Any SQL statement variables that had values bound to them using ** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. ** Use [sqlite3_clear_bindings()] to reset the bindings. ** -** {H11332} The [sqlite3_reset(S)] interface resets the [prepared statement] S -** back to the beginning of its program. +** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. ** -** {H11334} If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], -** or if [sqlite3_step(S)] has never before been called on S, -** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], +** or if [sqlite3_step(S)] has never before been called on S, +** then [sqlite3_reset(S)] returns [SQLITE_OK]. ** -** {H11336} If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S indicated an error, then -** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. ** -** {H11338} The [sqlite3_reset(S)] interface does not change the values -** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. +** ^The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ int sqlite3_reset(sqlite3_stmt *pStmt); /* -** CAPI3REF: Create Or Redefine SQL Functions {H16100} +** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** These two functions (collectively known as "function creation routines") +** ^These two functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior ** of existing SQL functions or aggregates. The only difference between the ** two is that the second parameter, the name of the (scalar) function or ** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 ** for sqlite3_create_function16(). ** -** The first parameter is the [database connection] to which the SQL -** function is to be added. If a single program uses more than one database -** connection internally, then SQL functions must be added individually to -** each database connection. +** ^The first parameter is the [database connection] to which the SQL +** function is to be added. ^If an application uses more than one database +** connection then application-defined SQL functions must be added +** to each database connection separately. ** ** The second parameter is the name of the SQL function to be created or -** redefined. The length of the name is limited to 255 bytes, exclusive of +** redefined. ^The length of the name is limited to 255 bytes, exclusive of ** the zero-terminator. Note that the name length limit is in bytes, not -** characters. Any attempt to create a function with a longer name +** characters. ^Any attempt to create a function with a longer name ** will result in [SQLITE_ERROR] being returned. ** -** The third parameter (nArg) +** ^The third parameter (nArg) ** is the number of arguments that the SQL function or -** aggregate takes. If this parameter is -1, then the SQL function or +** aggregate takes. ^If this parameter is -1, then the SQL function or ** aggregate may take any number of arguments between 0 and the limit ** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third ** parameter is less than -1 or greater than 127 then the behavior is @@ -3195,53 +3171,49 @@ int sqlite3_reset(sqlite3_stmt *pStmt); ** [SQLITE_UTF8 | text encoding] this SQL function prefers for ** its parameters. Any SQL function implementation should be able to work ** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be -** more efficient with one encoding than another. An application may +** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. -** When multiple implementations of the same function are available, SQLite +** ^When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. ** If there is only a single implementation which does not care what text ** encoding is used, then the fourth argument should be [SQLITE_ANY]. ** -** The fifth parameter is an arbitrary pointer. The implementation of the -** function can gain access to this pointer using [sqlite3_user_data()]. +** ^(The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()].)^ ** ** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or -** aggregate. A scalar SQL function requires an implementation of the xFunc -** callback only, NULL pointers should be passed as the xStep and xFinal -** parameters. An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. To delete an existing +** aggregate. ^A scalar SQL function requires an implementation of the xFunc +** callback only; NULL pointers should be passed as the xStep and xFinal +** parameters. ^An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL should be passed for xFunc. ^To delete an existing ** SQL function or aggregate, pass NULL for all three function callbacks. ** -** It is permitted to register multiple implementations of the same +** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of -** arguments or differing preferred text encodings. SQLite will use +** arguments or differing preferred text encodings. ^SQLite will use ** the implementation that most closely matches the way in which the -** SQL function is used. A function implementation with a non-negative +** SQL function is used. ^A function implementation with a non-negative ** nArg parameter is a better match than a function implementation with -** a negative nArg. A function where the preferred text encoding +** a negative nArg. ^A function where the preferred text encoding ** matches the database encoding is a better ** match than a function where the encoding is different. -** A function where the encoding difference is between UTF16le and UTF16be +** ^A function where the encoding difference is between UTF16le and UTF16be ** is a closer match than a function where the encoding difference is ** between UTF8 and UTF16. ** -** Built-in functions may be overloaded by new application-defined functions. -** The first application-defined function with a given name overrides all +** ^Built-in functions may be overloaded by new application-defined functions. +** ^The first application-defined function with a given name overrides all ** built-in functions in the same [database connection] with the same name. -** Subsequent application-defined functions of the same name only override +** ^Subsequent application-defined functions of the same name only override ** prior application-defined functions that are an exact match for the ** number of parameters and preferred encoding. ** -** An application-defined function is permitted to call other +** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. -** -** Requirements: -** [H16103] [H16106] [H16109] [H16112] [H16118] [H16121] [H16127] -** [H16130] [H16133] [H16136] [H16139] [H16142] */ int sqlite3_create_function( sqlite3 *db, @@ -3265,7 +3237,7 @@ int sqlite3_create_function16( ); /* -** CAPI3REF: Text Encodings {H10267} +** CAPI3REF: Text Encodings ** ** These constant define integer codes that represent the various ** text encodings supported by SQLite. @@ -3297,7 +3269,7 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void #endif /* -** CAPI3REF: Obtaining SQL Function Parameter Values {H15100} +** CAPI3REF: Obtaining SQL Function Parameter Values ** ** The C-language implementation of SQL functions and aggregates uses ** this set of interface routines to access the parameter values on @@ -3315,22 +3287,22 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void ** Any attempt to use these routines on an [unprotected sqlite3_value] ** object results in undefined behavior. ** -** These routines work just like the corresponding [column access functions] +** ^These routines work just like the corresponding [column access functions] ** except that these routines take a single [protected sqlite3_value] object ** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. ** -** The sqlite3_value_text16() interface extracts a UTF-16 string -** in the native byte-order of the host machine. The +** ^The sqlite3_value_text16() interface extracts a UTF-16 string +** in the native byte-order of the host machine. ^The ** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces ** extract UTF-16 strings as big-endian and little-endian respectively. ** -** The sqlite3_value_numeric_type() interface attempts to apply +** ^(The sqlite3_value_numeric_type() interface attempts to apply ** numeric affinity to the value. This means that an attempt is ** made to convert the value to an integer or floating point. If ** such a conversion is possible without loss of information (in other ** words, if the value is a string that looks like a number) ** then the conversion is performed. Otherwise no conversion occurs. -** The [SQLITE_INTEGER | datatype] after conversion is returned. +** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** ** Please pay particular attention to the fact that the pointer returned ** from [sqlite3_value_blob()], [sqlite3_value_text()], or @@ -3340,10 +3312,6 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void ** ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. -** -** Requirements: -** [H15103] [H15106] [H15109] [H15112] [H15115] [H15118] [H15121] [H15124] -** [H15127] [H15130] [H15133] [H15136] */ const void *sqlite3_value_blob(sqlite3_value*); int sqlite3_value_bytes(sqlite3_value*); @@ -3359,66 +3327,73 @@ int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); /* -** CAPI3REF: Obtain Aggregate Function Context {H16210} +** CAPI3REF: Obtain Aggregate Function Context ** -** The implementation of aggregate SQL functions use this routine to allocate -** a structure for storing their state. +** Implementions of aggregate SQL functions use this +** routine to allocate memory for storing their state. ** -** The first time the sqlite3_aggregate_context() routine is called for a -** particular aggregate, SQLite allocates nBytes of memory, zeroes out that -** memory, and returns a pointer to it. On second and subsequent calls to -** sqlite3_aggregate_context() for the same aggregate function index, -** the same buffer is returned. The implementation of the aggregate can use -** the returned buffer to accumulate data. +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite +** allocates N of memory, zeroes out that memory, and returns a pointer +** to the new memory. ^On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function instance, +** the same buffer is returned. Sqlite3_aggregate_context() is normally +** called once for each invocation of the xStep callback and then one +** last time when the xFinal callback is invoked. ^(When no rows match +** an aggregate query, the xStep() callback of the aggregate function +** implementation is never called and xFinal() is called exactly once. +** In those cases, sqlite3_aggregate_context() might be called for the +** first time from within xFinal().)^ ** -** SQLite automatically frees the allocated buffer when the aggregate -** query concludes. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is +** less than or equal to zero or if a memory allocate error occurs. ** -** The first parameter should be a copy of the +** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is +** determined by the N parameter on first successful call. Changing the +** value of N in subsequent call to sqlite3_aggregate_context() within +** the same aggregate function instance will not resize the memory +** allocation.)^ +** +** ^SQLite automatically frees the memory allocated by +** sqlite3_aggregate_context() when the aggregate query concludes. +** +** The first parameter must be a copy of the ** [sqlite3_context | SQL function context] that is the first parameter -** to the callback routine that implements the aggregate function. +** to the xStep or xFinal callback routine that implements the aggregate +** function. ** ** This routine must be called from the same thread in which ** the aggregate SQL function is running. -** -** Requirements: -** [H16211] [H16213] [H16215] [H16217] */ void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* -** CAPI3REF: User Data For Functions {H16240} +** CAPI3REF: User Data For Functions ** -** The sqlite3_user_data() interface returns a copy of +** ^The sqlite3_user_data() interface returns a copy of ** the pointer that was the pUserData parameter (the 5th parameter) ** of the [sqlite3_create_function()] ** and [sqlite3_create_function16()] routines that originally -** registered the application defined function. {END} -** -** This routine must be called from the same thread in which -** the application-defined function is running. -** -** Requirements: -** [H16243] -*/ -void *sqlite3_user_data(sqlite3_context*); - -/* -** CAPI3REF: Database Connection For Functions {H16250} -** -** The sqlite3_context_db_handle() interface returns a copy of -** the pointer to the [database connection] (the 1st parameter) -** of the [sqlite3_create_function()] -** and [sqlite3_create_function16()] routines that originally ** registered the application defined function. ** -** Requirements: -** [H16253] +** This routine must be called from the same thread in which +** the application-defined function is running. +*/ +void *sqlite3_user_data(sqlite3_context*); + +/* +** CAPI3REF: Database Connection For Functions +** +** ^The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. */ sqlite3 *sqlite3_context_db_handle(sqlite3_context*); /* -** CAPI3REF: Function Auxiliary Data {H16270} +** CAPI3REF: Function Auxiliary Data ** ** The following two functions may be used by scalar SQL functions to ** associate metadata with argument values. If the same value is passed to @@ -3431,48 +3406,45 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** invocations of the same function so that the original pattern string ** does not need to be recompiled on each invocation. ** -** The sqlite3_get_auxdata() interface returns a pointer to the metadata +** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata ** associated by the sqlite3_set_auxdata() function with the Nth argument -** value to the application-defined function. If no metadata has been ever +** value to the application-defined function. ^If no metadata has been ever ** been set for the Nth argument of the function, or if the corresponding ** function parameter has changed since the meta-data was set, ** then sqlite3_get_auxdata() returns a NULL pointer. ** -** The sqlite3_set_auxdata() interface saves the metadata +** ^The sqlite3_set_auxdata() interface saves the metadata ** pointed to by its 3rd parameter as the metadata for the N-th ** argument of the application-defined function. Subsequent ** calls to sqlite3_get_auxdata() might return this data, if it has ** not been destroyed. -** If it is not NULL, SQLite will invoke the destructor +** ^If it is not NULL, SQLite will invoke the destructor ** function given by the 4th parameter to sqlite3_set_auxdata() on ** the metadata when the corresponding function parameter changes ** or when the SQL statement completes, whichever comes first. ** ** SQLite is free to call the destructor and drop metadata on any -** parameter of any function at any time. The only guarantee is that +** parameter of any function at any time. ^The only guarantee is that ** the destructor will be called before the metadata is dropped. ** -** In practice, metadata is preserved between function calls for +** ^(In practice, metadata is preserved between function calls for ** expressions that are constant at compile time. This includes literal -** values and SQL variables. +** values and [parameters].)^ ** ** These routines must be called from the same thread in which ** the SQL function is running. -** -** Requirements: -** [H16272] [H16274] [H16276] [H16277] [H16278] [H16279] */ void *sqlite3_get_auxdata(sqlite3_context*, int N); void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); /* -** CAPI3REF: Constants Defining Special Destructor Behavior {H10280} +** CAPI3REF: Constants Defining Special Destructor Behavior ** ** These are special values for the destructor that is passed in as the -** final argument to routines like [sqlite3_result_blob()]. If the destructor +** final argument to routines like [sqlite3_result_blob()]. ^If the destructor ** argument is SQLITE_STATIC, it means that the content pointer is constant -** and will never change. It does not need to be destroyed. The +** and will never change. It does not need to be destroyed. ^The ** SQLITE_TRANSIENT value means that the content will likely change in ** the near future and that SQLite should make its own private copy of ** the content before returning. @@ -3485,7 +3457,7 @@ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_TRANSIENT ((sqlite3_destructor_type)-1) /* -** CAPI3REF: Setting The Result Of An SQL Function {H16400} +** CAPI3REF: Setting The Result Of An SQL Function ** ** These routines are used by the xFunc or xFinal callbacks that ** implement SQL functions and aggregates. See @@ -3496,103 +3468,98 @@ typedef void (*sqlite3_destructor_type)(void*); ** functions used to bind values to host parameters in prepared statements. ** Refer to the [SQL parameter] documentation for additional information. ** -** The sqlite3_result_blob() interface sets the result from +** ^The sqlite3_result_blob() interface sets the result from ** an application-defined function to be the BLOB whose content is pointed ** to by the second parameter and which is N bytes long where N is the ** third parameter. ** -** The sqlite3_result_zeroblob() interfaces set the result of +** ^The sqlite3_result_zeroblob() interfaces set the result of ** the application-defined function to be a BLOB containing all zero ** bytes and N bytes in size, where N is the value of the 2nd parameter. ** -** The sqlite3_result_double() interface sets the result from +** ^The sqlite3_result_double() interface sets the result from ** an application-defined function to be a floating point value specified ** by its 2nd argument. ** -** The sqlite3_result_error() and sqlite3_result_error16() functions +** ^The sqlite3_result_error() and sqlite3_result_error16() functions ** cause the implemented SQL function to throw an exception. -** SQLite uses the string pointed to by the +** ^SQLite uses the string pointed to by the ** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() -** as the text of an error message. SQLite interprets the error -** message string from sqlite3_result_error() as UTF-8. SQLite +** as the text of an error message. ^SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. ^SQLite ** interprets the string from sqlite3_result_error16() as UTF-16 in native -** byte order. If the third parameter to sqlite3_result_error() +** byte order. ^If the third parameter to sqlite3_result_error() ** or sqlite3_result_error16() is negative then SQLite takes as the error ** message all text up through the first zero character. -** If the third parameter to sqlite3_result_error() or +** ^If the third parameter to sqlite3_result_error() or ** sqlite3_result_error16() is non-negative then SQLite takes that many ** bytes (not characters) from the 2nd parameter as the error message. -** The sqlite3_result_error() and sqlite3_result_error16() +** ^The sqlite3_result_error() and sqlite3_result_error16() ** routines make a private copy of the error message text before ** they return. Hence, the calling function can deallocate or ** modify the text after they return without harm. -** The sqlite3_result_error_code() function changes the error code -** returned by SQLite as a result of an error in a function. By default, -** the error code is SQLITE_ERROR. A subsequent call to sqlite3_result_error() +** ^The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. ^By default, +** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() ** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. ** -** The sqlite3_result_toobig() interface causes SQLite to throw an error -** indicating that a string or BLOB is to long to represent. +** ^The sqlite3_result_toobig() interface causes SQLite to throw an error +** indicating that a string or BLOB is too long to represent. ** -** The sqlite3_result_nomem() interface causes SQLite to throw an error +** ^The sqlite3_result_nomem() interface causes SQLite to throw an error ** indicating that a memory allocation failed. ** -** The sqlite3_result_int() interface sets the return value +** ^The sqlite3_result_int() interface sets the return value ** of the application-defined function to be the 32-bit signed integer ** value given in the 2nd argument. -** The sqlite3_result_int64() interface sets the return value +** ^The sqlite3_result_int64() interface sets the return value ** of the application-defined function to be the 64-bit signed integer ** value given in the 2nd argument. ** -** The sqlite3_result_null() interface sets the return value +** ^The sqlite3_result_null() interface sets the return value ** of the application-defined function to be NULL. ** -** The sqlite3_result_text(), sqlite3_result_text16(), +** ^The sqlite3_result_text(), sqlite3_result_text16(), ** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** SQLite takes the text result from the application from +** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** If the 3rd parameter to the sqlite3_result_text* interfaces +** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is negative, then SQLite takes result text from the 2nd parameter ** through the first zero character. -** If the 3rd parameter to the sqlite3_result_text* interfaces +** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined ** function result. -** If the 4th parameter to the sqlite3_result_text* interfaces +** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that ** function as the destructor on the text or BLOB result when it has ** finished using that result. -** If the 4th parameter to the sqlite3_result_text* interfaces or to +** ^If the 4th parameter to the sqlite3_result_text* interfaces or to ** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite ** assumes that the text or BLOB result is in constant space and does not ** copy the content of the parameter nor call a destructor on the content ** when it has finished using that result. -** If the 4th parameter to the sqlite3_result_text* interfaces +** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT ** then SQLite makes a copy of the result into space obtained from ** from [sqlite3_malloc()] before it returns. ** -** The sqlite3_result_value() interface sets the result of +** ^The sqlite3_result_value() interface sets the result of ** the application-defined function to be a copy the -** [unprotected sqlite3_value] object specified by the 2nd parameter. The +** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The ** sqlite3_result_value() interface makes a copy of the [sqlite3_value] ** so that the [sqlite3_value] specified in the parameter may change or ** be deallocated after sqlite3_result_value() returns without harm. -** A [protected sqlite3_value] object may always be used where an +** ^A [protected sqlite3_value] object may always be used where an ** [unprotected sqlite3_value] object is required, so either ** kind of [sqlite3_value] object can be used with this interface. ** ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. -** -** Requirements: -** [H16403] [H16406] [H16409] [H16412] [H16415] [H16418] [H16421] [H16424] -** [H16427] [H16430] [H16433] [H16436] [H16439] [H16442] [H16445] [H16448] -** [H16451] [H16454] [H16457] [H16460] [H16463] */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); @@ -3612,20 +3579,20 @@ void sqlite3_result_value(sqlite3_context*, sqlite3_value*); void sqlite3_result_zeroblob(sqlite3_context*, int n); /* -** CAPI3REF: Define New Collating Sequences {H16600} +** CAPI3REF: Define New Collating Sequences ** ** These functions are used to add new collation sequences to the ** [database connection] specified as the first argument. ** -** The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the new collation sequence is specified as a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). In all cases +** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases ** the name is passed as the second function argument. ** -** The third argument may be one of the constants [SQLITE_UTF8], +** ^The third argument may be one of the constants [SQLITE_UTF8], ** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied ** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. The +** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The ** third argument might also be [SQLITE_UTF16] to indicate that the routine ** expects pointers to be UTF-16 strings in the native byte order, or the ** argument can be [SQLITE_UTF16_ALIGNED] if the @@ -3633,33 +3600,29 @@ void sqlite3_result_zeroblob(sqlite3_context*, int n); ** of UTF-16 in the native byte order. ** ** A pointer to the user supplied routine must be passed as the fifth -** argument. If it is NULL, this is the same as deleting the collation +** argument. ^If it is NULL, this is the same as deleting the collation ** sequence (so that SQLite cannot call it anymore). -** Each time the application supplied function is invoked, it is passed +** ^Each time the application supplied function is invoked, it is passed ** as its first parameter a copy of the void* passed as the fourth argument ** to sqlite3_create_collation() or sqlite3_create_collation16(). ** -** The remaining arguments to the application-supplied routine are two strings, +** ^The remaining arguments to the application-supplied routine are two strings, ** each represented by a (length, data) pair and encoded in the encoding ** that was passed as the third argument when the collation sequence was -** registered. {END} The application defined collation routine should +** registered. The application defined collation routine should ** return negative, zero or positive if the first string is less than, ** equal to, or greater than the second string. i.e. (STRING1 - STRING2). ** -** The sqlite3_create_collation_v2() works like sqlite3_create_collation() +** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() ** except that it takes an extra argument which is a destructor for -** the collation. The destructor is called when the collation is +** the collation. ^The destructor is called when the collation is ** destroyed and is passed a copy of the fourth parameter void* pointer ** of the sqlite3_create_collation_v2(). -** Collations are destroyed when they are overridden by later calls to the +** ^Collations are destroyed when they are overridden by later calls to the ** collation creation functions or when the [database connection] is closed ** using [sqlite3_close()]. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. -** -** Requirements: -** [H16603] [H16604] [H16606] [H16609] [H16612] [H16615] [H16618] [H16621] -** [H16624] [H16627] [H16630] */ int sqlite3_create_collation( sqlite3*, @@ -3685,33 +3648,30 @@ int sqlite3_create_collation16( ); /* -** CAPI3REF: Collation Needed Callbacks {H16700} +** CAPI3REF: Collation Needed Callbacks ** -** To avoid having to register all collation sequences before a database +** ^To avoid having to register all collation sequences before a database ** can be used, a single callback function may be registered with the -** [database connection] to be called whenever an undefined collation +** [database connection] to be invoked whenever an undefined collation ** sequence is required. ** -** If the function is registered using the sqlite3_collation_needed() API, +** ^If the function is registered using the sqlite3_collation_needed() API, ** then it is passed the names of undefined collation sequences as strings -** encoded in UTF-8. {H16703} If sqlite3_collation_needed16() is used, +** encoded in UTF-8. ^If sqlite3_collation_needed16() is used, ** the names are passed as UTF-16 in machine native byte order. -** A call to either function replaces any existing callback. +** ^A call to either function replaces the existing collation-needed callback. ** -** When the callback is invoked, the first argument passed is a copy +** ^(When the callback is invoked, the first argument passed is a copy ** of the second argument to sqlite3_collation_needed() or ** sqlite3_collation_needed16(). The second argument is the database ** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], ** or [SQLITE_UTF16LE], indicating the most desirable form of the collation ** sequence function required. The fourth parameter is the name of the -** required collation sequence. +** required collation sequence.)^ ** ** The callback function should register the desired collation using ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. -** -** Requirements: -** [H16702] [H16704] [H16706] */ int sqlite3_collation_needed( sqlite3*, @@ -3724,6 +3684,7 @@ int sqlite3_collation_needed16( void(*)(void*,sqlite3*,int eTextRep,const void*) ); +#if SQLITE_HAS_CODEC /* ** Specify the key for an encrypted database. This routine should be ** called right after sqlite3_open(). @@ -3750,29 +3711,47 @@ int sqlite3_rekey( ); /* -** CAPI3REF: Suspend Execution For A Short Time {H10530} +** Specify the activation key for a SEE database. Unless +** activated, none of the SEE routines will work. +*/ +void sqlite3_activate_see( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +#ifdef SQLITE_ENABLE_CEROD +/* +** Specify the activation key for a CEROD database. Unless +** activated, none of the CEROD routines will work. +*/ +void sqlite3_activate_cerod( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +/* +** CAPI3REF: Suspend Execution For A Short Time ** -** The sqlite3_sleep() function causes the current thread to suspend execution +** ^The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** If the operating system does not support sleep requests with +** ^If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. The number of milliseconds of sleep actually +** the nearest second. ^The number of milliseconds of sleep actually ** requested from the operating system is returned. ** -** SQLite implements this interface by calling the xSleep() +** ^SQLite implements this interface by calling the xSleep() ** method of the default [sqlite3_vfs] object. -** -** Requirements: [H10533] [H10536] */ int sqlite3_sleep(int); /* -** CAPI3REF: Name Of The Folder Holding Temporary Files {H10310} +** CAPI3REF: Name Of The Folder Holding Temporary Files ** -** If this global variable is made to point to a string which is +** ^(If this global variable is made to point to a string which is ** the name of a folder (a.k.a. directory), then all temporary files -** created by SQLite will be placed in that directory. If this variable +** created by SQLite when using a built-in [sqlite3_vfs | VFS] +** will be placed in that directory.)^ ^If this variable ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** @@ -3785,8 +3764,8 @@ int sqlite3_sleep(int); ** routines have been called and that this variable remain unchanged ** thereafter. ** -** The [temp_store_directory pragma] may modify this variable and cause -** it to point to memory obtained from [sqlite3_malloc]. Furthermore, +** ^The [temp_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, ** the [temp_store_directory pragma] always assumes that any string ** that this variable points to is held in memory obtained from ** [sqlite3_malloc] and the pragma may attempt to free that memory @@ -3798,14 +3777,14 @@ int sqlite3_sleep(int); SQLITE_EXTERN char *sqlite3_temp_directory; /* -** CAPI3REF: Test For Auto-Commit Mode {H12930} +** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} ** -** The sqlite3_get_autocommit() interface returns non-zero or +** ^The sqlite3_get_autocommit() interface returns non-zero or ** zero if the given database connection is or is not in autocommit mode, -** respectively. Autocommit mode is on by default. -** Autocommit mode is disabled by a [BEGIN] statement. -** Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** respectively. ^Autocommit mode is on by default. +** ^Autocommit mode is disabled by a [BEGIN] statement. +** ^Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. ** ** If certain kinds of errors occur on a statement within a multi-statement ** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], @@ -3817,58 +3796,55 @@ SQLITE_EXTERN char *sqlite3_temp_directory; ** If another thread changes the autocommit status of the database ** connection while this routine is running, then the return value ** is undefined. -** -** Requirements: [H12931] [H12932] [H12933] [H12934] */ int sqlite3_get_autocommit(sqlite3*); /* -** CAPI3REF: Find The Database Handle Of A Prepared Statement {H13120} +** CAPI3REF: Find The Database Handle Of A Prepared Statement ** -** The sqlite3_db_handle interface returns the [database connection] handle -** to which a [prepared statement] belongs. The [database connection] -** returned by sqlite3_db_handle is the same [database connection] that was the first argument +** ^The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. ^The [database connection] +** returned by sqlite3_db_handle is the same [database connection] +** that was the first argument ** to the [sqlite3_prepare_v2()] call (or its variants) that was used to ** create the statement in the first place. -** -** Requirements: [H13123] */ sqlite3 *sqlite3_db_handle(sqlite3_stmt*); /* -** CAPI3REF: Find the next prepared statement {H13140} +** CAPI3REF: Find the next prepared statement ** -** This interface returns a pointer to the next [prepared statement] after -** pStmt associated with the [database connection] pDb. If pStmt is NULL +** ^This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. ^If pStmt is NULL ** then this interface returns a pointer to the first prepared statement -** associated with the database connection pDb. If no prepared statement +** associated with the database connection pDb. ^If no prepared statement ** satisfies the conditions of this routine, it returns NULL. ** ** The [database connection] pointer D in a call to ** [sqlite3_next_stmt(D,S)] must refer to an open database ** connection and in particular must not be a NULL pointer. -** -** Requirements: [H13143] [H13146] [H13149] [H13152] */ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* -** CAPI3REF: Commit And Rollback Notification Callbacks {H12950} +** CAPI3REF: Commit And Rollback Notification Callbacks ** -** The sqlite3_commit_hook() interface registers a callback +** ^The sqlite3_commit_hook() interface registers a callback ** function to be invoked whenever a transaction is [COMMIT | committed]. -** Any callback set by a previous call to sqlite3_commit_hook() +** ^Any callback set by a previous call to sqlite3_commit_hook() ** for the same database connection is overridden. -** The sqlite3_rollback_hook() interface registers a callback +** ^The sqlite3_rollback_hook() interface registers a callback ** function to be invoked whenever a transaction is [ROLLBACK | rolled back]. -** Any callback set by a previous call to sqlite3_commit_hook() +** ^Any callback set by a previous call to sqlite3_rollback_hook() ** for the same database connection is overridden. -** The pArg argument is passed through to the callback. -** If the callback on a commit hook function returns non-zero, +** ^The pArg argument is passed through to the callback. +** ^If the callback on a commit hook function returns non-zero, ** then the commit is converted into a rollback. ** -** If another function was previously registered, its -** pArg value is returned. Otherwise NULL is returned. +** ^The sqlite3_commit_hook(D,C,P) and sqlite3_rollback_hook(D,C,P) functions +** return the P argument from the previous call of the same function +** on the same [database connection] D, or NULL for +** the first call for each function on D. ** ** The callback implementation must not do anything that will modify ** the database connection that invoked the callback. Any actions @@ -3878,59 +3854,54 @@ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** -** Registering a NULL function disables the callback. +** ^Registering a NULL function disables the callback. ** -** When the commit hook callback routine returns zero, the [COMMIT] -** operation is allowed to continue normally. If the commit hook +** ^When the commit hook callback routine returns zero, the [COMMIT] +** operation is allowed to continue normally. ^If the commit hook ** returns non-zero, then the [COMMIT] is converted into a [ROLLBACK]. -** The rollback hook is invoked on a rollback that results from a commit +** ^The rollback hook is invoked on a rollback that results from a commit ** hook returning non-zero, just as it would be with any other rollback. ** -** For the purposes of this API, a transaction is said to have been +** ^For the purposes of this API, a transaction is said to have been ** rolled back if an explicit "ROLLBACK" statement is executed, or ** an error or constraint causes an implicit rollback to occur. -** The rollback callback is not invoked if a transaction is +** ^The rollback callback is not invoked if a transaction is ** automatically rolled back because the database connection is closed. -** The rollback callback is not invoked if a transaction is +** ^The rollback callback is not invoked if a transaction is ** rolled back because a commit callback returned non-zero. -** Check on this ** ** See also the [sqlite3_update_hook()] interface. -** -** Requirements: -** [H12951] [H12952] [H12953] [H12954] [H12955] -** [H12961] [H12962] [H12963] [H12964] */ void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); /* -** CAPI3REF: Data Change Notification Callbacks {H12970} +** CAPI3REF: Data Change Notification Callbacks ** -** The sqlite3_update_hook() interface registers a callback function +** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument ** to be invoked whenever a row is updated, inserted or deleted. -** Any callback set by a previous call to this function +** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** -** The second argument is a pointer to the function to invoke when a +** ^The second argument is a pointer to the function to invoke when a ** row is updated, inserted or deleted. -** The first argument to the callback is a copy of the third argument +** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). -** The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], ** or [SQLITE_UPDATE], depending on the operation that caused the callback ** to be invoked. -** The third and fourth arguments to the callback contain pointers to the +** ^The third and fourth arguments to the callback contain pointers to the ** database and table name containing the affected row. -** The final callback parameter is the [rowid] of the row. -** In the case of an update, this is the [rowid] after the update takes place. +** ^The final callback parameter is the [rowid] of the row. +** ^In the case of an update, this is the [rowid] after the update takes place. ** -** The update hook is not invoked when internal system tables are -** modified (i.e. sqlite_master and sqlite_sequence). +** ^(The update hook is not invoked when internal system tables are +** modified (i.e. sqlite_master and sqlite_sequence).)^ ** -** In the current implementation, the update hook +** ^In the current implementation, the update hook ** is not invoked when duplication rows are deleted because of an -** [ON CONFLICT | ON CONFLICT REPLACE] clause. Nor is the update hook +** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook ** invoked when rows are deleted using the [truncate optimization]. ** The exceptions defined in this paragraph might change in a future ** release of SQLite. @@ -3942,14 +3913,13 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** -** If another function was previously registered, its pArg value -** is returned. Otherwise NULL is returned. +** ^The sqlite3_update_hook(D,C,P) function +** returns the P argument from the previous call +** on the same [database connection] D, or NULL for +** the first call on D. ** ** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] ** interfaces. -** -** Requirements: -** [H12971] [H12973] [H12975] [H12977] [H12979] [H12981] [H12983] [H12986] */ void *sqlite3_update_hook( sqlite3*, @@ -3958,74 +3928,66 @@ void *sqlite3_update_hook( ); /* -** CAPI3REF: Enable Or Disable Shared Pager Cache {H10330} +** CAPI3REF: Enable Or Disable Shared Pager Cache ** KEYWORDS: {shared cache} ** -** This routine enables or disables the sharing of the database cache +** ^(This routine enables or disables the sharing of the database cache ** and schema data structures between [database connection | connections] ** to the same database. Sharing is enabled if the argument is true -** and disabled if the argument is false. +** and disabled if the argument is false.)^ ** -** Cache sharing is enabled and disabled for an entire process. +** ^Cache sharing is enabled and disabled for an entire process. ** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, ** sharing was enabled or disabled for each thread separately. ** -** The cache sharing mode set by this interface effects all subsequent +** ^(The cache sharing mode set by this interface effects all subsequent ** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. ** Existing database connections continue use the sharing mode -** that was in effect at the time they were opened. +** that was in effect at the time they were opened.)^ ** -** Virtual tables cannot be used with a shared cache. When shared -** cache is enabled, the [sqlite3_create_module()] API used to register -** virtual tables will always return an error. +** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise.)^ ** -** This routine returns [SQLITE_OK] if shared cache was enabled or disabled -** successfully. An [error code] is returned otherwise. -** -** Shared cache is disabled by default. But this might change in +** ^Shared cache is disabled by default. But this might change in ** future releases of SQLite. Applications that care about shared ** cache setting should set it explicitly. ** ** See Also: [SQLite Shared-Cache Mode] -** -** Requirements: [H10331] [H10336] [H10337] [H10339] */ int sqlite3_enable_shared_cache(int); /* -** CAPI3REF: Attempt To Free Heap Memory {H17340} +** CAPI3REF: Attempt To Free Heap Memory ** -** The sqlite3_release_memory() interface attempts to free N bytes +** ^The sqlite3_release_memory() interface attempts to free N bytes ** of heap memory by deallocating non-essential memory allocations -** held by the database library. {END} Memory used to cache database +** held by the database library. Memory used to cache database ** pages to improve performance is an example of non-essential memory. -** sqlite3_release_memory() returns the number of bytes actually freed, +** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. -** -** Requirements: [H17341] [H17342] */ int sqlite3_release_memory(int); /* -** CAPI3REF: Impose A Limit On Heap Size {H17350} +** CAPI3REF: Impose A Limit On Heap Size ** -** The sqlite3_soft_heap_limit() interface places a "soft" limit +** ^The sqlite3_soft_heap_limit() interface places a "soft" limit ** on the amount of heap memory that may be allocated by SQLite. -** If an internal allocation is requested that would exceed the +** ^If an internal allocation is requested that would exceed the ** soft heap limit, [sqlite3_release_memory()] is invoked one or ** more times to free up some space before the allocation is performed. ** -** The limit is called "soft", because if [sqlite3_release_memory()] +** ^The limit is called "soft" because if [sqlite3_release_memory()] ** cannot free sufficient memory to prevent the limit from being exceeded, ** the memory is allocated anyway and the current operation proceeds. ** -** A negative or zero value for N means that there is no soft heap limit and +** ^A negative or zero value for N means that there is no soft heap limit and ** [sqlite3_release_memory()] will only be called when memory is exhausted. -** The default value for the soft heap limit is zero. +** ^The default value for the soft heap limit is zero. ** -** SQLite makes a best effort to honor the soft heap limit. +** ^(SQLite makes a best effort to honor the soft heap limit. ** But if the soft heap limit cannot be honored, execution will -** continue without error or notification. This is why the limit is +** continue without error or notification.)^ This is why the limit is ** called a "soft" limit. It is advisory only. ** ** Prior to SQLite version 3.5.0, this routine only constrained the memory @@ -4035,35 +3997,32 @@ int sqlite3_release_memory(int); ** is an upper bound on the total memory allocation for all threads. In ** version 3.5.0 there is no mechanism for limiting the heap usage for ** individual threads. -** -** Requirements: -** [H16351] [H16352] [H16353] [H16354] [H16355] [H16358] */ void sqlite3_soft_heap_limit(int); /* -** CAPI3REF: Extract Metadata About A Column Of A Table {H12850} +** CAPI3REF: Extract Metadata About A Column Of A Table ** -** This routine returns metadata about a specific column of a specific +** ^This routine returns metadata about a specific column of a specific ** database table accessible using the [database connection] handle ** passed as the first function argument. ** -** The column is identified by the second, third and fourth parameters to -** this function. The second parameter is either the name of the database -** (i.e. "main", "temp" or an attached database) containing the specified -** table or NULL. If it is NULL, then all attached databases are searched +** ^The column is identified by the second, third and fourth parameters to +** this function. ^The second parameter is either the name of the database +** (i.e. "main", "temp", or an attached database) containing the specified +** table or NULL. ^If it is NULL, then all attached databases are searched ** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** -** The third and fourth parameters to this function are the table and column +** ^The third and fourth parameters to this function are the table and column ** name of the desired column, respectively. Neither of these parameters ** may be NULL. ** -** Metadata is returned by writing to the memory locations passed as the 5th -** and subsequent parameters to this function. Any of these arguments may be +** ^Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. ^Any of these arguments may be ** NULL, in which case the corresponding element of metadata is omitted. ** -**
+** ^(
** **
Parameter Output
Type 		Description ** @@ -4073,17 +4032,17 @@ void sqlite3_soft_heap_limit(int); **
8th int True if column is part of the PRIMARY KEY **
9th int True if column is [AUTOINCREMENT] **
-**
+**
)^ ** -** The memory pointed to by the character pointers returned for the +** ^The memory pointed to by the character pointers returned for the ** declaration type and collation sequence is valid only until the next ** call to any SQLite API function. ** -** If the specified table is actually a view, an [error code] is returned. +** ^If the specified table is actually a view, an [error code] is returned. ** -** If the specified column is "rowid", "oid" or "_rowid_" and an +** ^If the specified column is "rowid", "oid" or "_rowid_" and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output -** parameters are set for the explicitly declared column. If there is no +** parameters are set for the explicitly declared column. ^(If there is no ** explicitly declared [INTEGER PRIMARY KEY] column, then the output ** parameters are set as follows: ** @@ -4093,14 +4052,14 @@ void sqlite3_soft_heap_limit(int); ** not null: 0 ** primary key: 1 ** auto increment: 0 -** +** )^ ** -** This function may load one or more schemas from database files. If an +** ^(This function may load one or more schemas from database files. If an ** error occurs during this process, or if the requested table or column ** cannot be found, an [error code] is returned and an error message left -** in the [database connection] (to be retrieved using sqlite3_errmsg()). +** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ ** -** This API is only available if the library was compiled with the +** ^This API is only available if the library was compiled with the ** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. */ int sqlite3_table_column_metadata( @@ -4116,30 +4075,27 @@ int sqlite3_table_column_metadata( ); /* -** CAPI3REF: Load An Extension {H12600} +** CAPI3REF: Load An Extension ** -** This interface loads an SQLite extension library from the named file. +** ^This interface loads an SQLite extension library from the named file. ** -** {H12601} The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** ^The sqlite3_load_extension() interface attempts to load an +** SQLite extension library contained in the file zFile. ** -** {H12602} The entry point is zProc. +** ^The entry point is zProc. +** ^zProc may be 0, in which case the name of the entry point +** defaults to "sqlite3_extension_init". +** ^The sqlite3_load_extension() interface returns +** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^If an error occurs and pzErrMsg is not 0, then the +** [sqlite3_load_extension()] interface shall attempt to +** fill *pzErrMsg with error message text stored in memory +** obtained from [sqlite3_malloc()]. The calling function +** should free this memory by calling [sqlite3_free()]. ** -** {H12603} zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". -** -** {H12604} The sqlite3_load_extension() interface shall return -** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. -** -** {H12605} If an error occurs and pzErrMsg is not 0, then the -** [sqlite3_load_extension()] interface shall attempt to -** fill *pzErrMsg with error message text stored in memory -** obtained from [sqlite3_malloc()]. {END} The calling function -** should free this memory by calling [sqlite3_free()]. -** -** {H12606} Extension loading must be enabled using -** [sqlite3_enable_load_extension()] prior to calling this API, -** otherwise an error will be returned. +** ^Extension loading must be enabled using +** [sqlite3_enable_load_extension()] prior to calling this API, +** otherwise an error will be returned. ** ** See also the [load_extension() SQL function]. */ @@ -4151,61 +4107,49 @@ int sqlite3_load_extension( ); /* -** CAPI3REF: Enable Or Disable Extension Loading {H12620} +** CAPI3REF: Enable Or Disable Extension Loading ** -** So as not to open security holes in older applications that are +** ^So as not to open security holes in older applications that are ** unprepared to deal with extension loading, and as a means of disabling ** extension loading while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** Extension loading is off by default. See ticket #1863. -** -** {H12621} Call the sqlite3_enable_load_extension() routine with onoff==1 -** to turn extension loading on and call it with onoff==0 to turn -** it back off again. -** -** {H12622} Extension loading is off by default. +** ^Extension loading is off by default. See ticket #1863. +** ^Call the sqlite3_enable_load_extension() routine with onoff==1 +** to turn extension loading on and call it with onoff==0 to turn +** it back off again. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions {H12640} +** CAPI3REF: Automatically Load An Extensions ** -** This API can be invoked at program startup in order to register +** ^This API can be invoked at program startup in order to register ** one or more statically linked extensions that will be available -** to all new [database connections]. {END} +** to all new [database connections]. ** -** This routine stores a pointer to the extension in an array that is -** obtained from [sqlite3_malloc()]. If you run a memory leak checker -** on your program and it reports a leak because of this array, invoke -** [sqlite3_reset_auto_extension()] prior to shutdown to free the memory. +** ^(This routine stores a pointer to the extension entry point +** in an array that is obtained from [sqlite3_malloc()]. That memory +** is deallocated by [sqlite3_reset_auto_extension()].)^ ** -** {H12641} This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** -** {H12642} Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** -** {H12643} This routine stores a pointer to the extension in an array -** that is obtained from [sqlite3_malloc()]. -** -** {H12644} Automatic extensions apply across all threads. +** ^This function registers an extension entry point that is +** automatically invoked whenever a new [database connection] +** is opened using [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()]. +** ^Duplicate extensions are detected so calling this routine +** multiple times with the same extension is harmless. +** ^Automatic extensions apply across all threads. */ int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* -** CAPI3REF: Reset Automatic Extension Loading {H12660} +** CAPI3REF: Reset Automatic Extension Loading ** -** This function disables all previously registered automatic -** extensions. {END} It undoes the effect of all prior -** [sqlite3_auto_extension()] calls. +** ^(This function disables all previously registered automatic +** extensions. It undoes the effect of all prior +** [sqlite3_auto_extension()] calls.)^ ** -** {H12661} This function disables all previously registered -** automatic extensions. -** -** {H12662} This function disables automatic extensions in all threads. +** ^This function disables automatic extensions in all threads. */ void sqlite3_reset_auto_extension(void); @@ -4229,7 +4173,7 @@ typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor; typedef struct sqlite3_module sqlite3_module; /* -** CAPI3REF: Virtual Table Object {H18000} +** CAPI3REF: Virtual Table Object ** KEYWORDS: sqlite3_module {virtual table module} ** EXPERIMENTAL ** @@ -4237,10 +4181,10 @@ typedef struct sqlite3_module sqlite3_module; ** defines the implementation of a [virtual tables]. ** This structure consists mostly of methods for the module. ** -** A virtual table module is created by filling in a persistent +** ^A virtual table module is created by filling in a persistent ** instance of this structure and passing a pointer to that instance ** to [sqlite3_create_module()] or [sqlite3_create_module_v2()]. -** The registration remains valid until it is replaced by a different +** ^The registration remains valid until it is replaced by a different ** module or until the [database connection] closes. The content ** of this structure must not change while it is registered with ** any database connection. @@ -4276,7 +4220,7 @@ struct sqlite3_module { }; /* -** CAPI3REF: Virtual Table Indexing Information {H18100} +** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info ** EXPERIMENTAL ** @@ -4286,42 +4230,42 @@ struct sqlite3_module { ** inputs to xBestIndex and are read-only. xBestIndex inserts its ** results into the **Outputs** fields. ** -** The aConstraint[] array records WHERE clause constraints of the form: +** ^(The aConstraint[] array records WHERE clause constraints of the form: ** ** 
column OP expr
** -** where OP is =, <, <=, >, or >=. The particular operator is -** stored in aConstraint[].op. The index of the column is stored in -** aConstraint[].iColumn. aConstraint[].usable is TRUE if the +** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is +** stored in aConstraint[].op.)^ ^(The index of the column is stored in +** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint -** is usable) and false if it cannot. +** is usable) and false if it cannot.)^ ** -** The optimizer automatically inverts terms of the form "expr OP column" +** ^The optimizer automatically inverts terms of the form "expr OP column" ** and makes other simplifications to the WHERE clause in an attempt to ** get as many WHERE clause terms into the form shown above as possible. -** The aConstraint[] array only reports WHERE clause terms in the correct -** form that refer to the particular virtual table being queried. +** ^The aConstraint[] array only reports WHERE clause terms that are +** relevant to the particular virtual table being queried. ** -** Information about the ORDER BY clause is stored in aOrderBy[]. -** Each term of aOrderBy records a column of the ORDER BY clause. +** ^Information about the ORDER BY clause is stored in aOrderBy[]. +** ^Each term of aOrderBy records a column of the ORDER BY clause. ** ** The [xBestIndex] method must fill aConstraintUsage[] with information -** about what parameters to pass to xFilter. If argvIndex>0 then +** about what parameters to pass to xFilter. ^If argvIndex>0 then ** the right-hand side of the corresponding aConstraint[] is evaluated -** and becomes the argvIndex-th entry in argv. If aConstraintUsage[].omit +** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit ** is true, then the constraint is assumed to be fully handled by the -** virtual table and is not checked again by SQLite. +** virtual table and is not checked again by SQLite.)^ ** -** The idxNum and idxPtr values are recorded and passed into the +** ^The idxNum and idxPtr values are recorded and passed into the ** [xFilter] method. -** [sqlite3_free()] is used to free idxPtr if and only iff +** ^[sqlite3_free()] is used to free idxPtr if and only if ** needToFreeIdxPtr is true. ** -** The orderByConsumed means that output from [xFilter]/[xNext] will occur in +** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in ** the correct order to satisfy the ORDER BY clause so that no separate ** sorting step is required. ** -** The estimatedCost value is an estimate of the cost of doing the +** ^The estimatedCost value is an estimate of the cost of doing the ** particular lookup. A full scan of a table with N entries should have ** a cost of N. A binary search of a table of N entries should have a ** cost of approximately log(N). @@ -4359,24 +4303,28 @@ struct sqlite3_index_info { #define SQLITE_INDEX_CONSTRAINT_MATCH 64 /* -** CAPI3REF: Register A Virtual Table Implementation {H18200} +** CAPI3REF: Register A Virtual Table Implementation ** EXPERIMENTAL ** -** This routine is used to register a new [virtual table module] name. -** Module names must be registered before -** creating a new [virtual table] using the module, or before using a +** ^These routines are used to register a new [virtual table module] name. +** ^Module names must be registered before +** creating a new [virtual table] using the module and before using a ** preexisting [virtual table] for the module. ** -** The module name is registered on the [database connection] specified -** by the first parameter. The name of the module is given by the -** second parameter. The third parameter is a pointer to -** the implementation of the [virtual table module]. The fourth +** ^The module name is registered on the [database connection] specified +** by the first parameter. ^The name of the module is given by the +** second parameter. ^The third parameter is a pointer to +** the implementation of the [virtual table module]. ^The fourth ** parameter is an arbitrary client data pointer that is passed through ** into the [xCreate] and [xConnect] methods of the virtual table module ** when a new virtual table is be being created or reinitialized. ** -** This interface has exactly the same effect as calling -** [sqlite3_create_module_v2()] with a NULL client data destructor. +** ^The sqlite3_create_module_v2() interface has a fifth parameter which +** is a pointer to a destructor for the pClientData. ^SQLite will +** invoke the destructor function (if it is not NULL) when SQLite +** no longer needs the pClientData pointer. ^The sqlite3_create_module() +** interface is equivalent to sqlite3_create_module_v2() with a NULL +** destructor. */ SQLITE_EXPERIMENTAL int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ @@ -4384,17 +4332,6 @@ SQLITE_EXPERIMENTAL int sqlite3_create_module( const sqlite3_module *p, /* Methods for the module */ void *pClientData /* Client data for xCreate/xConnect */ ); - -/* -** CAPI3REF: Register A Virtual Table Implementation {H18210} -** EXPERIMENTAL -** -** This routine is identical to the [sqlite3_create_module()] method, -** except that it has an extra parameter to specify -** a destructor function for the client data pointer. SQLite will -** invoke the destructor function (if it is not NULL) when SQLite -** no longer needs the pClientData pointer. -*/ SQLITE_EXPERIMENTAL int sqlite3_create_module_v2( sqlite3 *db, /* SQLite connection to register module with */ const char *zName, /* Name of the module */ @@ -4404,21 +4341,21 @@ SQLITE_EXPERIMENTAL int sqlite3_create_module_v2( ); /* -** CAPI3REF: Virtual Table Instance Object {H18010} +** CAPI3REF: Virtual Table Instance Object ** KEYWORDS: sqlite3_vtab ** EXPERIMENTAL ** ** Every [virtual table module] implementation uses a subclass -** of the following structure to describe a particular instance +** of this object to describe a particular instance ** of the [virtual table]. Each subclass will ** be tailored to the specific needs of the module implementation. ** The purpose of this superclass is to define certain fields that are ** common to all module implementations. ** -** Virtual tables methods can set an error message by assigning a +** ^Virtual tables methods can set an error message by assigning a ** string obtained from [sqlite3_mprintf()] to zErrMsg. The method should ** take care that any prior string is freed by a call to [sqlite3_free()] -** prior to assigning a new string to zErrMsg. After the error message +** prior to assigning a new string to zErrMsg. ^After the error message ** is delivered up to the client application, the string will be automatically ** freed by sqlite3_free() and the zErrMsg field will be zeroed. */ @@ -4430,7 +4367,7 @@ struct sqlite3_vtab { }; /* -** CAPI3REF: Virtual Table Cursor Object {H18020} +** CAPI3REF: Virtual Table Cursor Object ** KEYWORDS: sqlite3_vtab_cursor {virtual table cursor} ** EXPERIMENTAL ** @@ -4439,7 +4376,7 @@ struct sqlite3_vtab { ** [virtual table] and are used ** to loop through the virtual table. Cursors are created using the ** [sqlite3_module.xOpen | xOpen] method of the module and are destroyed -** by the [sqlite3_module.xClose | xClose] method. Cussors are used +** by the [sqlite3_module.xClose | xClose] method. Cursors are used ** by the [xFilter], [xNext], [xEof], [xColumn], and [xRowid] methods ** of the module. Each module implementation will define ** the content of a cursor structure to suit its own needs. @@ -4453,10 +4390,10 @@ struct sqlite3_vtab_cursor { }; /* -** CAPI3REF: Declare The Schema Of A Virtual Table {H18280} +** CAPI3REF: Declare The Schema Of A Virtual Table ** EXPERIMENTAL ** -** The [xCreate] and [xConnect] methods of a +** ^The [xCreate] and [xConnect] methods of a ** [virtual table module] call this interface ** to declare the format (the names and datatypes of the columns) of ** the virtual tables they implement. @@ -4464,17 +4401,17 @@ struct sqlite3_vtab_cursor { SQLITE_EXPERIMENTAL int sqlite3_declare_vtab(sqlite3*, const char *zSQL); /* -** CAPI3REF: Overload A Function For A Virtual Table {H18300} +** CAPI3REF: Overload A Function For A Virtual Table ** EXPERIMENTAL ** -** Virtual tables can provide alternative implementations of functions +** ^(Virtual tables can provide alternative implementations of functions ** using the [xFindFunction] method of the [virtual table module]. ** But global versions of those functions -** must exist in order to be overloaded. +** must exist in order to be overloaded.)^ ** -** This API makes sure a global version of a function with a particular +** ^(This API makes sure a global version of a function with a particular ** name and number of parameters exists. If no such function exists -** before this API is called, a new function is created. The implementation +** before this API is called, a new function is created.)^ ^The implementation ** of the new function always causes an exception to be thrown. So ** the new function is not good for anything by itself. Its only ** purpose is to be a placeholder function that can be overloaded @@ -4495,77 +4432,74 @@ SQLITE_EXPERIMENTAL int sqlite3_overload_function(sqlite3*, const char *zFuncNam */ /* -** CAPI3REF: A Handle To An Open BLOB {H17800} +** CAPI3REF: A Handle To An Open BLOB ** KEYWORDS: {BLOB handle} {BLOB handles} ** ** An instance of this object represents an open BLOB on which ** [sqlite3_blob_open | incremental BLOB I/O] can be performed. -** Objects of this type are created by [sqlite3_blob_open()] +** ^Objects of this type are created by [sqlite3_blob_open()] ** and destroyed by [sqlite3_blob_close()]. -** The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces +** ^The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces ** can be used to read or write small subsections of the BLOB. -** The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. +** ^The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. */ typedef struct sqlite3_blob sqlite3_blob; /* -** CAPI3REF: Open A BLOB For Incremental I/O {H17810} +** CAPI3REF: Open A BLOB For Incremental I/O ** -** This interfaces opens a [BLOB handle | handle] to the BLOB located +** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located ** in row iRow, column zColumn, table zTable in database zDb; ** in other words, the same BLOB that would be selected by: ** ** 
 **     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
-**
{END} +** )^ ** -** If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. If it is zero, the BLOB is opened for read access. -** It is not possible to open a column that is part of an index or primary +** ^If the flags parameter is non-zero, then the BLOB is opened for read +** and write access. ^If it is zero, the BLOB is opened for read access. +** ^It is not possible to open a column that is part of an index or primary ** key for writing. ^If [foreign key constraints] are enabled, it is ** not possible to open a column that is part of a [child key] for writing. ** -** Note that the database name is not the filename that contains +** ^Note that the database name is not the filename that contains ** the database but rather the symbolic name of the database that -** is assigned when the database is connected using [ATTACH]. -** For the main database file, the database name is "main". -** For TEMP tables, the database name is "temp". +** appears after the AS keyword when the database is connected using [ATTACH]. +** ^For the main database file, the database name is "main". +** ^For TEMP tables, the database name is "temp". ** -** On success, [SQLITE_OK] is returned and the new [BLOB handle] is written +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written ** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer. -** This function sets the [database connection] error code and message +** to be a null pointer.)^ +** ^This function sets the [database connection] error code and message ** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. Note that the *ppBlob variable is always initialized in a +** functions. ^Note that the *ppBlob variable is always initialized in a ** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob ** regardless of the success or failure of this routine. ** -** If the row that a BLOB handle points to is modified by an +** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects ** then the BLOB handle is marked as "expired". ** This is true if any column of the row is changed, even a column -** other than the one the BLOB handle is open on. -** Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for +** other than the one the BLOB handle is open on.)^ +** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for ** a expired BLOB handle fail with an return code of [SQLITE_ABORT]. -** Changes written into a BLOB prior to the BLOB expiring are not -** rollback by the expiration of the BLOB. Such changes will eventually -** commit if the transaction continues to completion. +** ^(Changes written into a BLOB prior to the BLOB expiring are not +** rolled back by the expiration of the BLOB. Such changes will eventually +** commit if the transaction continues to completion.)^ ** -** Use the [sqlite3_blob_bytes()] interface to determine the size of -** the opened blob. The size of a blob may not be changed by this +** ^Use the [sqlite3_blob_bytes()] interface to determine the size of +** the opened blob. ^The size of a blob may not be changed by this ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces +** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces ** and the built-in [zeroblob] SQL function can be used, if desired, ** to create an empty, zero-filled blob in which to read or write using ** this interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. -** -** Requirements: -** [H17813] [H17814] [H17816] [H17819] [H17821] [H17824] */ int sqlite3_blob_open( sqlite3*, @@ -4578,37 +4512,34 @@ int sqlite3_blob_open( ); /* -** CAPI3REF: Close A BLOB Handle {H17830} +** CAPI3REF: Close A BLOB Handle ** -** Closes an open [BLOB handle]. +** ^Closes an open [BLOB handle]. ** -** Closing a BLOB shall cause the current transaction to commit +** ^Closing a BLOB shall cause the current transaction to commit ** if there are no other BLOBs, no pending prepared statements, and the ** database connection is in [autocommit mode]. -** If any writes were made to the BLOB, they might be held in cache +** ^If any writes were made to the BLOB, they might be held in cache ** until the close operation if they will fit. ** -** Closing the BLOB often forces the changes +** ^(Closing the BLOB often forces the changes ** out to disk and so if any I/O errors occur, they will likely occur ** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value. +** closing are reported as a non-zero return value.)^ ** -** The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed. +** ^(The BLOB is closed unconditionally. Even if this routine returns +** an error code, the BLOB is still closed.)^ ** -** Calling this routine with a null pointer (which as would be returned -** by failed call to [sqlite3_blob_open()]) is a harmless no-op. -** -** Requirements: -** [H17833] [H17836] [H17839] +** ^Calling this routine with a null pointer (such as would be returned +** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. */ int sqlite3_blob_close(sqlite3_blob *); /* -** CAPI3REF: Return The Size Of An Open BLOB {H17840} +** CAPI3REF: Return The Size Of An Open BLOB ** -** Returns the size in bytes of the BLOB accessible via the -** successfully opened [BLOB handle] in its only argument. The +** ^Returns the size in bytes of the BLOB accessible via the +** successfully opened [BLOB handle] in its only argument. ^The ** incremental blob I/O routines can only read or overwriting existing ** blob content; they cannot change the size of a blob. ** @@ -4616,30 +4547,27 @@ int sqlite3_blob_close(sqlite3_blob *); ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in ** to this routine results in undefined and probably undesirable behavior. -** -** Requirements: -** [H17843] */ int sqlite3_blob_bytes(sqlite3_blob *); /* -** CAPI3REF: Read Data From A BLOB Incrementally {H17850} +** CAPI3REF: Read Data From A BLOB Incrementally ** -** This function is used to read data from an open [BLOB handle] into a +** ^(This function is used to read data from an open [BLOB handle] into a ** caller-supplied buffer. N bytes of data are copied into buffer Z -** from the open BLOB, starting at offset iOffset. +** from the open BLOB, starting at offset iOffset.)^ ** -** If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is read. If N or iOffset is +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is read. ^If N or iOffset is ** less than zero, [SQLITE_ERROR] is returned and no data is read. -** The size of the blob (and hence the maximum value of N+iOffset) +** ^The size of the blob (and hence the maximum value of N+iOffset) ** can be determined using the [sqlite3_blob_bytes()] interface. ** -** An attempt to read from an expired [BLOB handle] fails with an +** ^An attempt to read from an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ** -** On success, SQLITE_OK is returned. -** Otherwise, an [error code] or an [extended error code] is returned. +** ^(On success, sqlite3_blob_read() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ ** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not @@ -4647,40 +4575,37 @@ int sqlite3_blob_bytes(sqlite3_blob *); ** to this routine results in undefined and probably undesirable behavior. ** ** See also: [sqlite3_blob_write()]. -** -** Requirements: -** [H17853] [H17856] [H17859] [H17862] [H17863] [H17865] [H17868] */ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* -** CAPI3REF: Write Data Into A BLOB Incrementally {H17870} +** CAPI3REF: Write Data Into A BLOB Incrementally ** -** This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. N bytes of data are copied from the buffer Z +** ^This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. ^N bytes of data are copied from the buffer Z ** into the open BLOB, starting at offset iOffset. ** -** If the [BLOB handle] passed as the first argument was not opened for +** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** This function may only modify the contents of the BLOB; it is +** ^This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. -** If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. If N is +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is written. ^If N is ** less than zero [SQLITE_ERROR] is returned and no data is written. ** The size of the BLOB (and hence the maximum value of N+iOffset) ** can be determined using the [sqlite3_blob_bytes()] interface. ** -** An attempt to write to an expired [BLOB handle] fails with an -** error code of [SQLITE_ABORT]. Writes to the BLOB that occurred +** ^An attempt to write to an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred ** before the [BLOB handle] expired are not rolled back by the ** expiration of the handle, though of course those changes might ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** On success, SQLITE_OK is returned. -** Otherwise, an [error code] or an [extended error code] is returned. +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ ** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not @@ -4688,15 +4613,11 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** to this routine results in undefined and probably undesirable behavior. ** ** See also: [sqlite3_blob_read()]. -** -** Requirements: -** [H17873] [H17874] [H17875] [H17876] [H17877] [H17879] [H17882] [H17885] -** [H17888] */ int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); /* -** CAPI3REF: Virtual File System Objects {H11200} +** CAPI3REF: Virtual File System Objects ** ** A virtual filesystem (VFS) is an [sqlite3_vfs] object ** that SQLite uses to interact @@ -4705,34 +4626,31 @@ int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); ** New VFSes can be registered and existing VFSes can be unregistered. ** The following interfaces are provided. ** -** The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. -** Names are case sensitive. -** Names are zero-terminated UTF-8 strings. -** If there is no match, a NULL pointer is returned. -** If zVfsName is NULL then the default VFS is returned. +** ^The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. +** ^Names are case sensitive. +** ^Names are zero-terminated UTF-8 strings. +** ^If there is no match, a NULL pointer is returned. +** ^If zVfsName is NULL then the default VFS is returned. ** -** New VFSes are registered with sqlite3_vfs_register(). -** Each new VFS becomes the default VFS if the makeDflt flag is set. -** The same VFS can be registered multiple times without injury. -** To make an existing VFS into the default VFS, register it again +** ^New VFSes are registered with sqlite3_vfs_register(). +** ^Each new VFS becomes the default VFS if the makeDflt flag is set. +** ^The same VFS can be registered multiple times without injury. +** ^To make an existing VFS into the default VFS, register it again ** with the makeDflt flag set. If two different VFSes with the ** same name are registered, the behavior is undefined. If a ** VFS is registered with a name that is NULL or an empty string, ** then the behavior is undefined. ** -** Unregister a VFS with the sqlite3_vfs_unregister() interface. -** If the default VFS is unregistered, another VFS is chosen as -** the default. The choice for the new VFS is arbitrary. -** -** Requirements: -** [H11203] [H11206] [H11209] [H11212] [H11215] [H11218] +** ^Unregister a VFS with the sqlite3_vfs_unregister() interface. +** ^(If the default VFS is unregistered, another VFS is chosen as +** the default. The choice for the new VFS is arbitrary.)^ */ sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); int sqlite3_vfs_unregister(sqlite3_vfs*); /* -** CAPI3REF: Mutexes {H17000} +** CAPI3REF: Mutexes ** ** The SQLite core uses these routines for thread ** synchronization. Though they are intended for internal @@ -4741,7 +4659,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. The following +** is selected automatically at compile-time. ^(The following ** implementations are available in the SQLite core: ** **
    @@ -4749,26 +4667,26 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_PTHREAD **
  • SQLITE_MUTEX_W32 **
  • SQLITE_MUTEX_NOOP -**
+** )^ ** -** The SQLITE_MUTEX_NOOP implementation is a set of routines +** ^The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. The SQLITE_MUTEX_OS2, +** a single-threaded application. ^The SQLITE_MUTEX_OS2, ** SQLITE_MUTEX_PTHREAD, and SQLITE_MUTEX_W32 implementations ** are appropriate for use on OS/2, Unix, and Windows. ** -** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize(). +** function that calls sqlite3_initialize().)^ ** -** {H17011} The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. {H17012} If it returns NULL -** that means that a mutex could not be allocated. {H17013} SQLite -** will unwind its stack and return an error. {H17014} The argument +** ^The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. ^If it returns NULL +** that means that a mutex could not be allocated. ^SQLite +** will unwind its stack and return an error. ^(The argument ** to sqlite3_mutex_alloc() is one of these integer constants: ** **
    @@ -4780,64 +4698,66 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_LRU2 -**
+** )^ ** -** {H17015} The first two constants cause sqlite3_mutex_alloc() to create -** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE -** is used but not necessarily so when SQLITE_MUTEX_FAST is used. {END} +** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) +** cause sqlite3_mutex_alloc() to create +** a new mutex. ^The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. {H17016} But SQLite will only request a recursive mutex in -** cases where it really needs one. {END} If a faster non-recursive mutex +** not want to. ^SQLite will only request a recursive mutex in +** cases where it really needs one. ^If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** -** {H17017} The other allowed parameters to sqlite3_mutex_alloc() each return -** a pointer to a static preexisting mutex. {END} Six static mutexes are +** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other +** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return +** a pointer to a static preexisting mutex. ^Six static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should ** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or ** SQLITE_MUTEX_RECURSIVE. ** -** {H17018} Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. {H17034} But for the static +** returns a different mutex on every call. ^But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** -** {H17019} The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. {H17020} SQLite is careful to deallocate every -** dynamic mutex that it allocates. {A17021} The dynamic mutexes must not be in -** use when they are deallocated. {A17022} Attempting to deallocate a static -** mutex results in undefined behavior. {H17023} SQLite never deallocates -** a static mutex. {END} +** ^The sqlite3_mutex_free() routine deallocates a previously +** allocated dynamic mutex. ^SQLite is careful to deallocate every +** dynamic mutex that it allocates. The dynamic mutexes must not be in +** use when they are deallocated. Attempting to deallocate a static +** mutex results in undefined behavior. ^SQLite never deallocates +** a static mutex. ** -** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt -** to enter a mutex. {H17024} If another thread is already within the mutex, +** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. ^If another thread is already within the mutex, ** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return -** SQLITE_BUSY. {H17025} The sqlite3_mutex_try() interface returns [SQLITE_OK] -** upon successful entry. {H17026} Mutexes created using +** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] +** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** {H17027} In such cases the, +** In such cases the, ** mutex must be exited an equal number of times before another thread -** can enter. {A17028} If the same thread tries to enter any other +** can enter.)^ ^(If the same thread tries to enter any other ** kind of mutex more than once, the behavior is undefined. -** {H17029} SQLite will never exhibit -** such behavior in its own use of mutexes. +** SQLite will never exhibit +** such behavior in its own use of mutexes.)^ ** -** Some systems (for example, Windows 95) do not support the operation +** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. {H17030} The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior. +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ ** -** {H17031} The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. {A17032} The behavior +** ^The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. ^(The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. {H17033} SQLite will -** never do either. {END} +** calling thread or is not currently allocated. SQLite will +** never do either.)^ ** -** If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines ** behave as no-ops. ** @@ -4850,7 +4770,7 @@ int sqlite3_mutex_try(sqlite3_mutex*); void sqlite3_mutex_leave(sqlite3_mutex*); /* -** CAPI3REF: Mutex Methods Object {H17120} +** CAPI3REF: Mutex Methods Object ** EXPERIMENTAL ** ** An instance of this structure defines the low-level routines @@ -4866,19 +4786,19 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** output variable when querying the system for the current mutex ** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. ** -** The xMutexInit method defined by this structure is invoked as +** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** {H17001} The xMutexInit routine shall be called by SQLite once for each +** ^The xMutexInit routine is calle by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** -** The xMutexEnd method defined by this structure is invoked as +** ^The xMutexEnd method defined by this structure is invoked as ** part of system shutdown by the sqlite3_shutdown() function. The ** implementation of this method is expected to release all outstanding ** resources obtained by the mutex methods implementation, especially -** those obtained by the xMutexInit method. {H17003} The xMutexEnd() -** interface shall be invoked once for each call to [sqlite3_shutdown()]. +** those obtained by the xMutexInit method. ^The xMutexEnd() +** interface is invoked exactly once for each call to [sqlite3_shutdown()]. ** -** The remaining seven methods defined by this structure (xMutexAlloc, +** ^(The remaining seven methods defined by this structure (xMutexAlloc, ** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and ** xMutexNotheld) implement the following interfaces (respectively): ** @@ -4890,7 +4810,7 @@ void sqlite3_mutex_leave(sqlite3_mutex*); **
  • [sqlite3_mutex_leave()]
    • **
  • [sqlite3_mutex_held()]
    • **
  • [sqlite3_mutex_notheld()]
    • -** +** )^ ** ** The only difference is that the public sqlite3_XXX functions enumerated ** above silently ignore any invocations that pass a NULL pointer instead @@ -4900,17 +4820,17 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. It must be harmless to +** The xMutexInit() method must be threadsafe. ^It must be harmless to ** invoke xMutexInit() mutiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). Similarly, xMutexAlloc() must not use SQLite memory -** allocation for a static mutex. However xMutexAlloc() may use SQLite +** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** -** SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is +** ^SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is ** called, but only if the prior call to xMutexInit returned SQLITE_OK. ** If xMutexInit fails in any way, it is expected to clean up after itself ** prior to returning. @@ -4929,39 +4849,41 @@ struct sqlite3_mutex_methods { }; /* -** CAPI3REF: Mutex Verification Routines {H17080} +** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. {H17081} The SQLite core +** are intended for use inside assert() statements. ^The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. {H17082} The core only +** are advised to follow the lead of the core. ^The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. {A17087} External mutex implementations +** with the SQLITE_DEBUG flag. ^External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** {H17083} These routines should return true if the mutex in their argument +** ^These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** {X17084} The implementation is not required to provided versions of these +** ^The implementation is not required to provided versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** {H17085} If the argument to sqlite3_mutex_held() is a NULL pointer then -** the routine should return 1. {END} This seems counter-intuitive since +** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But the ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. {H17086} The sqlite3_mutex_notheld() +** the appropriate thing to do. ^The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ +#ifndef NDEBUG int sqlite3_mutex_held(sqlite3_mutex*); int sqlite3_mutex_notheld(sqlite3_mutex*); +#endif /* -** CAPI3REF: Mutex Types {H17001} +** CAPI3REF: Mutex Types ** ** The [sqlite3_mutex_alloc()] interface takes a single argument ** which is one of these integer constants. @@ -4981,48 +4903,50 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU2 7 /* lru page list */ /* -** CAPI3REF: Retrieve the mutex for a database connection {H17002} +** CAPI3REF: Retrieve the mutex for a database connection ** -** This interface returns a pointer the [sqlite3_mutex] object that +** ^This interface returns a pointer the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument ** when the [threading mode] is Serialized. -** If the [threading mode] is Single-thread or Multi-thread then this +** ^If the [threading mode] is Single-thread or Multi-thread then this ** routine returns a NULL pointer. */ sqlite3_mutex *sqlite3_db_mutex(sqlite3*); /* -** CAPI3REF: Low-Level Control Of Database Files {H11300} +** CAPI3REF: Low-Level Control Of Database Files ** -** {H11301} The [sqlite3_file_control()] interface makes a direct call to the +** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated -** with a particular database identified by the second argument. {H11302} The -** name of the database is the name assigned to the database by the -** ATTACH SQL command that opened the -** database. {H11303} To control the main database file, use the name "main" -** or a NULL pointer. {H11304} The third and fourth parameters to this routine +** with a particular database identified by the second argument. ^The +** name of the database "main" for the main database or "temp" for the +** TEMP database, or the name that appears after the AS keyword for +** databases that are added using the [ATTACH] SQL command. +** ^A NULL pointer can be used in place of "main" to refer to the +** main database file. +** ^The third and fourth parameters to this routine ** are passed directly through to the second and third parameters of -** the xFileControl method. {H11305} The return value of the xFileControl +** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** -** {H11306} If the second parameter (zDbName) does not match the name of any -** open database file, then SQLITE_ERROR is returned. {H11307} This error +** ^If the second parameter (zDbName) does not match the name of any +** open database file, then SQLITE_ERROR is returned. ^This error ** code is not remembered and will not be recalled by [sqlite3_errcode()] -** or [sqlite3_errmsg()]. {A11308} The underlying xFileControl method might -** also return SQLITE_ERROR. {A11309} There is no way to distinguish between +** or [sqlite3_errmsg()]. The underlying xFileControl method might +** also return SQLITE_ERROR. There is no way to distinguish between ** an incorrect zDbName and an SQLITE_ERROR return from the underlying -** xFileControl method. {END} +** xFileControl method. ** ** See also: [SQLITE_FCNTL_LOCKSTATE] */ int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); /* -** CAPI3REF: Testing Interface {H11400} +** CAPI3REF: Testing Interface ** -** The sqlite3_test_control() interface is used to read out internal +** ^The sqlite3_test_control() interface is used to read out internal ** state of SQLite and to inject faults into SQLite for testing -** purposes. The first parameter is an operation code that determines +** purposes. ^The first parameter is an operation code that determines ** the number, meaning, and operation of all subsequent parameters. ** ** This interface is not for use by applications. It exists solely @@ -5037,7 +4961,7 @@ int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); int sqlite3_test_control(int op, ...); /* -** CAPI3REF: Testing Interface Operation Codes {H11410} +** CAPI3REF: Testing Interface Operation Codes ** ** These constants are the valid operation code parameters used ** as the first argument to [sqlite3_test_control()]. @@ -5047,6 +4971,7 @@ int sqlite3_test_control(int op, ...); ** Applications should not use any of these parameters or the ** [sqlite3_test_control()] interface. */ +#define SQLITE_TESTCTRL_FIRST 5 #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 #define SQLITE_TESTCTRL_PRNG_RESET 7 @@ -5057,27 +4982,30 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 +#define SQLITE_TESTCTRL_ISKEYWORD 16 +#define SQLITE_TESTCTRL_LAST 16 /* -** CAPI3REF: SQLite Runtime Status {H17200} +** CAPI3REF: SQLite Runtime Status ** EXPERIMENTAL ** -** This interface is used to retrieve runtime status information +** ^This interface is used to retrieve runtime status information ** about the preformance of SQLite, and optionally to reset various -** highwater marks. The first argument is an integer code for -** the specific parameter to measure. Recognized integer codes -** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...]. -** The current value of the parameter is returned into *pCurrent. -** The highest recorded value is returned in *pHighwater. If the +** highwater marks. ^The first argument is an integer code for +** the specific parameter to measure. ^(Recognized integer codes +** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ +** ^The current value of the parameter is returned into *pCurrent. +** ^The highest recorded value is returned in *pHighwater. ^If the ** resetFlag is true, then the highest record value is reset after -** *pHighwater is written. Some parameters do not record the highest +** *pHighwater is written. ^(Some parameters do not record the highest ** value. For those parameters -** nothing is written into *pHighwater and the resetFlag is ignored. -** Other parameters record only the highwater mark and not the current -** value. For these latter parameters nothing is written into *pCurrent. +** nothing is written into *pHighwater and the resetFlag is ignored.)^ +** ^(Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** This routine returns SQLITE_OK on success and a non-zero -** [error code] on failure. +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be ** called while other threads are running the same or different SQLite @@ -5092,14 +5020,14 @@ SQLITE_EXPERIMENTAL int sqlite3_status(int op, int *pCurrent, int *pHighwater, i /* -** CAPI3REF: Status Parameters {H17250} +** CAPI3REF: Status Parameters ** EXPERIMENTAL ** ** These integer constants designate various run-time status parameters ** that can be returned by [sqlite3_status()]. ** **
    -**
    SQLITE_STATUS_MEMORY_USED
    +** ^(
    SQLITE_STATUS_MEMORY_USED
    **
    This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application @@ -5107,45 +5035,45 @@ SQLITE_EXPERIMENTAL int sqlite3_status(int op, int *pCurrent, int *pHighwater, i ** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache ** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in ** this parameter. The amount returned is the sum of the allocation -** sizes as reported by the xSize method in [sqlite3_mem_methods].
    +** sizes as reported by the xSize method in [sqlite3_mem_methods].)^ ** -**
    SQLITE_STATUS_MALLOC_SIZE
    +** ^(
    SQLITE_STATUS_MALLOC_SIZE
    **
    This parameter records the largest memory allocation request ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. -** The value written into the *pCurrent parameter is undefined.
    +** The value written into the *pCurrent parameter is undefined.)^ ** -**
    SQLITE_STATUS_PAGECACHE_USED
    +** ^(
    SQLITE_STATUS_PAGECACHE_USED
    **
    This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The -** value returned is in pages, not in bytes.
    +** value returned is in pages, not in bytes.)^ ** -**
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    +** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache ** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because -** no space was left in the page cache.
    +** no space was left in the page cache.)^ ** -**
    SQLITE_STATUS_PAGECACHE_SIZE
    +** ^(
    SQLITE_STATUS_PAGECACHE_SIZE
    **
    This parameter records the largest memory allocation request ** handed to [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. -** The value written into the *pCurrent parameter is undefined.
    +** The value written into the *pCurrent parameter is undefined.)^ ** -**
    SQLITE_STATUS_SCRATCH_USED
    +** ^(
    SQLITE_STATUS_SCRATCH_USED
    **
    This parameter returns the number of allocations used out of the ** [scratch memory allocator] configured using ** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not ** in bytes. Since a single thread may only have one scratch allocation ** outstanding at time, this parameter also reports the number of threads -** using scratch memory at the same time.
    +** using scratch memory at the same time.)^ ** -**
    SQLITE_STATUS_SCRATCH_OVERFLOW
    +** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory ** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values @@ -5153,17 +5081,17 @@ SQLITE_EXPERIMENTAL int sqlite3_status(int op, int *pCurrent, int *pHighwater, i ** larger (that is, because the requested allocation was larger than the ** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer ** slots were available. -**
    +** )^ ** -**
    SQLITE_STATUS_SCRATCH_SIZE
    +** ^(
    SQLITE_STATUS_SCRATCH_SIZE
    **
    This parameter records the largest memory allocation request ** handed to [scratch memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. -** The value written into the *pCurrent parameter is undefined.
    +** The value written into the *pCurrent parameter is undefined.)^ ** -**
    SQLITE_STATUS_PARSER_STACK
    +** ^(
    SQLITE_STATUS_PARSER_STACK
    **
    This parameter records the deepest parser stack. It is only -** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    +** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].)^ **
    ** ** New status parameters may be added from time to time. @@ -5179,18 +5107,18 @@ SQLITE_EXPERIMENTAL int sqlite3_status(int op, int *pCurrent, int *pHighwater, i #define SQLITE_STATUS_SCRATCH_SIZE 8 /* -** CAPI3REF: Database Connection Status {H17500} +** CAPI3REF: Database Connection Status ** EXPERIMENTAL ** -** This interface is used to retrieve runtime status information -** about a single [database connection]. The first argument is the -** database connection object to be interrogated. The second argument -** is the parameter to interrogate. Currently, the only allowed value +** ^This interface is used to retrieve runtime status information +** about a single [database connection]. ^The first argument is the +** database connection object to be interrogated. ^The second argument +** is the parameter to interrogate. ^Currently, the only allowed value ** for the second parameter is [SQLITE_DBSTATUS_LOOKASIDE_USED]. ** Additional options will likely appear in future releases of SQLite. ** -** The current value of the requested parameter is written into *pCur -** and the highest instantaneous value is written into *pHiwtr. If +** ^The current value of the requested parameter is written into *pCur +** and the highest instantaneous value is written into *pHiwtr. ^If ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. ** @@ -5199,7 +5127,7 @@ SQLITE_EXPERIMENTAL int sqlite3_status(int op, int *pCurrent, int *pHighwater, i SQLITE_EXPERIMENTAL int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); /* -** CAPI3REF: Status Parameters for database connections {H17520} +** CAPI3REF: Status Parameters for database connections ** EXPERIMENTAL ** ** These constants are the available integer "verbs" that can be passed as @@ -5212,34 +5140,34 @@ SQLITE_EXPERIMENTAL int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiw ** if a discontinued or unsupported verb is invoked. ** **
    -**
    SQLITE_DBSTATUS_LOOKASIDE_USED
    +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    **
    This parameter returns the number of lookaside memory slots currently -** checked out.
    +** checked out.)^ **
    */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 /* -** CAPI3REF: Prepared Statement Status {H17550} +** CAPI3REF: Prepared Statement Status ** EXPERIMENTAL ** -** Each prepared statement maintains various +** ^(Each prepared statement maintains various ** [SQLITE_STMTSTATUS_SORT | counters] that measure the number -** of times it has performed specific operations. These counters can +** of times it has performed specific operations.)^ These counters can ** be used to monitor the performance characteristics of the prepared ** statements. For example, if the number of table steps greatly exceeds ** the number of table searches or result rows, that would tend to indicate ** that the prepared statement is using a full table scan rather than ** an index. ** -** This interface is used to retrieve and reset counter values from +** ^(This interface is used to retrieve and reset counter values from ** a [prepared statement]. The first argument is the prepared statement ** object to be interrogated. The second argument ** is an integer code for a specific [SQLITE_STMTSTATUS_SORT | counter] -** to be interrogated. -** The current value of the requested counter is returned. -** If the resetFlg is true, then the counter is reset to zero after this +** to be interrogated.)^ +** ^The current value of the requested counter is returned. +** ^If the resetFlg is true, then the counter is reset to zero after this ** interface call returns. ** ** See also: [sqlite3_status()] and [sqlite3_db_status()]. @@ -5247,7 +5175,7 @@ SQLITE_EXPERIMENTAL int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiw SQLITE_EXPERIMENTAL int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* -** CAPI3REF: Status Parameters for prepared statements {H17570} +** CAPI3REF: Status Parameters for prepared statements ** EXPERIMENTAL ** ** These preprocessor macros define integer codes that name counter @@ -5256,13 +5184,13 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** **
    **
    SQLITE_STMTSTATUS_FULLSCAN_STEP
    -**
    This is the number of times that SQLite has stepped forward in +**
    ^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter ** may indicate opportunities for performance improvement through ** careful use of indices.
    ** **
    SQLITE_STMTSTATUS_SORT
    -**
    This is the number of sort operations that have occurred. +**
    ^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to ** improvement performance through careful use of indices.
    ** @@ -5290,9 +5218,9 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** KEYWORDS: {page cache} ** EXPERIMENTAL ** -** The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure. The majority of the +** instance of the sqlite3_pcache_methods structure.)^ The majority of the ** heap memory used by SQLite is used by the page cache to cache data read ** from, or ready to be written to, the database file. By implementing a ** custom page cache using this API, an application can control more @@ -5301,69 +5229,69 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** determine exactly which parts of a database file are cached and for ** how long. ** -** The contents of the sqlite3_pcache_methods structure are copied to an +** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to -** [sqlite3_config()] returns. +** [sqlite3_config()] returns.)^ ** -** The xInit() method is called once for each call to [sqlite3_initialize()] -** (usually only once during the lifetime of the process). It is passed -** a copy of the sqlite3_pcache_methods.pArg value. It can be used to set -** up global structures and mutexes required by the custom page cache -** implementation. +** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** (usually only once during the lifetime of the process). ^(The xInit() +** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ +** ^The xInit() method can set up up global structures and/or any mutexes +** required by the custom page cache implementation. ** -** The xShutdown() method is called from within [sqlite3_shutdown()], +** ^The xShutdown() method is called from within [sqlite3_shutdown()], ** if the application invokes this API. It can be used to clean up ** any outstanding resources before process shutdown, if required. ** -** SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. The +** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes +** the xInit method, so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. ** -** SQLite will never invoke xInit() more than once without an intervening +** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** The xCreate() method is used to construct a new cache instance. SQLite +** ^The xCreate() method is used to construct a new cache instance. SQLite ** will typically create one cache instance for each open database file, -** though this is not guaranteed. The +** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. szPage will not be a power of two. szPage +** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. SQLite will use the +** increment (here called "R") of about 100 or 200. ^SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. -** R is constant for a particular build of SQLite. The second argument to +** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. The cache implementation +** false if it is used for an in-memory database. ^The cache implementation ** does not have to do anything special based with the value of bPurgeable; -** it is purely advisory. On a cache where bPurgeable is false, SQLite will +** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** In other words, a cache created with bPurgeable set to false will +** ^In other words, a cache created with bPurgeable set to false will ** never contain any unpinned pages. ** -** The xCachesize() method may be called at any time by SQLite to set the +** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command. As with the bPurgeable parameter, -** the implementation is not required to do anything with this +** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** The xPagecount() method should return the number of pages currently +** ^The xPagecount() method should return the number of pages currently ** stored in the cache. ** -** The xFetch() method is used to fetch a page and return a pointer to it. -** A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. The page to be fetched is determined by the key. The +** ^The xFetch() method is used to fetch a page and return a pointer to it. +** ^A 'page', in this context, is a buffer of szPage bytes aligned at an +** 8-byte boundary. ^The page to be fetched is determined by the key. ^The ** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** If the requested page is already in the page cache, then the page cache +** ^If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. If the requested page is not already in the cache, then the +** intact. ^(If the requested page is not already in the cache, then the ** behavior of the cache implementation is determined by the value of the ** createFlag parameter passed to xFetch, according to the following table: ** @@ -5374,7 +5302,7 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** Otherwise return NULL. ** 2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -** +** )^ ** ** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If ** a call to xFetch() with createFlag==1 returns NULL, then SQLite will @@ -5383,32 +5311,32 @@ typedef struct sqlite3_pcache sqlite3_pcache; ** attempting to unpin pages, the xFetch() method will be invoked again with ** a createFlag of 2. ** -** xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. If the third parameter, discard, is non-zero, +** ^xUnpin() is called by SQLite with a pointer to a currently pinned page +** as its second argument. ^(If the third parameter, discard, is non-zero, ** then the page should be evicted from the cache. In this case SQLite ** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed. If the discard parameter is -** zero, then the page is considered to be unpinned. The cache implementation +** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is +** zero, then the page is considered to be unpinned. ^The cache implementation ** may choose to evict unpinned pages at any time. ** -** The cache is not required to perform any reference counting. A single +** ^(The cache is not required to perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch(). +** to xFetch().)^ ** -** The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. If the cache +** ^The xRekey() method is used to change the key value associated with the +** page passed as the second argument from oldKey to newKey. ^If the cache ** previously contains an entry associated with newKey, it should be -** discarded. Any prior cache entry associated with newKey is guaranteed not +** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** When SQLite calls the xTruncate() method, the cache must discard all +** ^When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). If any +** to the value of the iLimit parameter passed to xTruncate(). ^If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** -** The xDestroy() method is used to delete a cache allocated by xCreate(). -** All resources associated with the specified cache should be freed. After +** ^The xDestroy() method is used to delete a cache allocated by xCreate(). +** All resources associated with the specified cache should be freed. ^After ** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] ** handle invalid, and will not use it with any other sqlite3_pcache_methods ** functions. @@ -5433,7 +5361,7 @@ struct sqlite3_pcache_methods { ** EXPERIMENTAL ** ** The sqlite3_backup object records state information about an ongoing -** online backup operation. The sqlite3_backup object is created by +** online backup operation. ^The sqlite3_backup object is created by ** a call to [sqlite3_backup_init()] and is destroyed by a call to ** [sqlite3_backup_finish()]. ** @@ -5445,20 +5373,20 @@ typedef struct sqlite3_backup sqlite3_backup; ** CAPI3REF: Online Backup API. ** EXPERIMENTAL ** -** This API is used to overwrite the contents of one database with that -** of another. It is useful either for creating backups of databases or +** The backup API copies the content of one database into another. +** It is useful either for creating backups of databases or ** for copying in-memory databases to or from persistent files. ** ** See Also: [Using the SQLite Online Backup API] ** -** Exclusive access is required to the destination database for the -** duration of the operation. However the source database is only -** read-locked while it is actually being read, it is not locked -** continuously for the entire operation. Thus, the backup may be -** performed on a live database without preventing other users from -** writing to the database for an extended period of time. +** ^Exclusive access is required to the destination database for the +** duration of the operation. ^However the source database is only +** read-locked while it is actually being read; it is not locked +** continuously for the entire backup operation. ^Thus, the backup may be +** performed on a live source database without preventing other users from +** reading or writing to the source database while the backup is underway. ** -** To perform a backup operation: +** ^(To perform a backup operation: **
      **
    1. sqlite3_backup_init() is called once to initialize the ** backup, @@ -5466,143 +5394,148 @@ typedef struct sqlite3_backup sqlite3_backup; ** the data between the two databases, and finally **
    2. sqlite3_backup_finish() is called to release all resources ** associated with the backup operation. -**
    +** )^ ** There should be exactly one call to sqlite3_backup_finish() for each ** successful call to sqlite3_backup_init(). ** ** sqlite3_backup_init() ** -** The first two arguments passed to [sqlite3_backup_init()] are the database -** handle associated with the destination database and the database name -** used to attach the destination database to the handle. The database name -** is "main" for the main database, "temp" for the temporary database, or -** the name specified as part of the [ATTACH] statement if the destination is -** an attached database. The third and fourth arguments passed to -** sqlite3_backup_init() identify the [database connection] -** and database name used -** to access the source database. The values passed for the source and -** destination [database connection] parameters must not be the same. +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database +** and the database name, respectively. +** ^The database name is "main" for the main database, "temp" for the +** temporary database, or the name specified after the AS keyword in +** an [ATTACH] statement for an attached database. +** ^The S and M arguments passed to +** sqlite3_backup_init(D,N,S,M) identify the [database connection] +** and database name of the source database, respectively. +** ^The source and destination [database connections] (parameters S and D) +** must be different or else sqlite3_backup_init(D,N,S,M) will file with +** an error. ** -** If an error occurs within sqlite3_backup_init(), then NULL is returned -** and an error code and error message written into the [database connection] -** passed as the first argument. They may be retrieved using the -** [sqlite3_errcode()], [sqlite3_errmsg()], and [sqlite3_errmsg16()] functions. -** Otherwise, if successful, a pointer to an [sqlite3_backup] object is -** returned. This pointer may be used with the sqlite3_backup_step() and +** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is +** returned and an error code and error message are store3d in the +** destination [database connection] D. +** ^The error code and message for the failed call to sqlite3_backup_init() +** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or +** [sqlite3_errmsg16()] functions. +** ^A successful call to sqlite3_backup_init() returns a pointer to an +** [sqlite3_backup] object. +** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and ** sqlite3_backup_finish() functions to perform the specified backup ** operation. ** ** sqlite3_backup_step() ** -** Function [sqlite3_backup_step()] is used to copy up to nPage pages between -** the source and destination databases, where nPage is the value of the -** second parameter passed to sqlite3_backup_step(). If nPage is a negative -** value, all remaining source pages are copied. If the required pages are -** succesfully copied, but there are still more pages to copy before the -** backup is complete, it returns [SQLITE_OK]. If no error occured and there -** are no more pages to copy, then [SQLITE_DONE] is returned. If an error -** occurs, then an SQLite error code is returned. As well as [SQLITE_OK] and +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** the source and destination databases specified by [sqlite3_backup] object B. +** ^If N is negative, all remaining source pages are copied. +** ^If sqlite3_backup_step(B,N) successfully copies N pages and there +** are still more pages to be copied, then the function resturns [SQLITE_OK]. +** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages +** from source to destination, then it returns [SQLITE_DONE]. +** ^If an error occurs while running sqlite3_backup_step(B,N), +** then an [error code] is returned. ^As well as [SQLITE_OK] and ** [SQLITE_DONE], a call to sqlite3_backup_step() may return [SQLITE_READONLY], ** [SQLITE_NOMEM], [SQLITE_BUSY], [SQLITE_LOCKED], or an ** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] extended error code. ** -** As well as the case where the destination database file was opened for -** read-only access, sqlite3_backup_step() may return [SQLITE_READONLY] if +** ^The sqlite3_backup_step() might return [SQLITE_READONLY] if the destination +** database was opened read-only or if ** the destination is an in-memory database with a different page size ** from the source database. ** -** If sqlite3_backup_step() cannot obtain a required file-system lock, then +** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then ** the [sqlite3_busy_handler | busy-handler function] -** is invoked (if one is specified). If the +** is invoked (if one is specified). ^If the ** busy-handler returns non-zero before the lock is available, then -** [SQLITE_BUSY] is returned to the caller. In this case the call to -** sqlite3_backup_step() can be retried later. If the source +** [SQLITE_BUSY] is returned to the caller. ^In this case the call to +** sqlite3_backup_step() can be retried later. ^If the source ** [database connection] ** is being used to write to the source database when sqlite3_backup_step() -** is called, then [SQLITE_LOCKED] is returned immediately. Again, in this -** case the call to sqlite3_backup_step() can be retried later on. If +** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this +** case the call to sqlite3_backup_step() can be retried later on. ^(If ** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or ** [SQLITE_READONLY] is returned, then ** there is no point in retrying the call to sqlite3_backup_step(). These -** errors are considered fatal. At this point the application must accept +** errors are considered fatal.)^ The application must accept ** that the backup operation has failed and pass the backup operation handle ** to the sqlite3_backup_finish() to release associated resources. ** -** Following the first call to sqlite3_backup_step(), an exclusive lock is -** obtained on the destination file. It is not released until either +** ^The first call to sqlite3_backup_step() obtains an exclusive lock +** on the destination file. ^The exclusive lock is not released until either ** sqlite3_backup_finish() is called or the backup operation is complete -** and sqlite3_backup_step() returns [SQLITE_DONE]. Additionally, each time -** a call to sqlite3_backup_step() is made a [shared lock] is obtained on -** the source database file. This lock is released before the -** sqlite3_backup_step() call returns. Because the source database is not -** locked between calls to sqlite3_backup_step(), it may be modified mid-way -** through the backup procedure. If the source database is modified by an +** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to +** sqlite3_backup_step() obtains a [shared lock] on the source database that +** lasts for the duration of the sqlite3_backup_step() call. +** ^Because the source database is not locked between calls to +** sqlite3_backup_step(), the source database may be modified mid-way +** through the backup process. ^If the source database is modified by an ** external process or via a database connection other than the one being -** used by the backup operation, then the backup will be transparently -** restarted by the next call to sqlite3_backup_step(). If the source +** used by the backup operation, then the backup will be automatically +** restarted by the next call to sqlite3_backup_step(). ^If the source ** database is modified by the using the same database connection as is used -** by the backup operation, then the backup database is transparently +** by the backup operation, then the backup database is automatically ** updated at the same time. ** ** sqlite3_backup_finish() ** -** Once sqlite3_backup_step() has returned [SQLITE_DONE], or when the -** application wishes to abandon the backup operation, the [sqlite3_backup] -** object should be passed to sqlite3_backup_finish(). This releases all -** resources associated with the backup operation. If sqlite3_backup_step() -** has not yet returned [SQLITE_DONE], then any active write-transaction on the -** destination database is rolled back. The [sqlite3_backup] object is invalid +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** application wishes to abandon the backup operation, the application +** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). +** ^The sqlite3_backup_finish() interfaces releases all +** resources associated with the [sqlite3_backup] object. +** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any +** active write-transaction on the destination database is rolled back. +** The [sqlite3_backup] object is invalid ** and may not be used following a call to sqlite3_backup_finish(). ** -** The value returned by sqlite3_backup_finish is [SQLITE_OK] if no error -** occurred, regardless or whether or not sqlite3_backup_step() was called -** a sufficient number of times to complete the backup operation. Or, if -** an out-of-memory condition or IO error occured during a call to -** sqlite3_backup_step() then [SQLITE_NOMEM] or an -** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] error code -** is returned. In this case the error code and an error message are -** written to the destination [database connection]. +** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no +** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() completed. +** ^If an out-of-memory condition or IO error occurred during any prior +** sqlite3_backup_step() call on the same [sqlite3_backup] object, then +** sqlite3_backup_finish() returns the corresponding [error code]. ** -** A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() is -** not a permanent error and does not affect the return value of +** ^A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() +** is not a permanent error and does not affect the return value of ** sqlite3_backup_finish(). ** ** sqlite3_backup_remaining(), sqlite3_backup_pagecount() ** -** Each call to sqlite3_backup_step() sets two values stored internally -** by an [sqlite3_backup] object. The number of pages still to be backed -** up, which may be queried by sqlite3_backup_remaining(), and the total -** number of pages in the source database file, which may be queried by -** sqlite3_backup_pagecount(). +** ^Each call to sqlite3_backup_step() sets two values inside +** the [sqlite3_backup] object: the number of pages still to be backed +** up and the total number of pages in the source databae file. +** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces +** retrieve these two values, respectively. ** -** The values returned by these functions are only updated by -** sqlite3_backup_step(). If the source database is modified during a backup +** ^The values returned by these functions are only updated by +** sqlite3_backup_step(). ^If the source database is modified during a backup ** operation, then the values are not updated to account for any extra ** pages that need to be updated or the size of the source database file ** changing. ** ** Concurrent Usage of Database Handles ** -** The source [database connection] may be used by the application for other +** ^The source [database connection] may be used by the application for other ** purposes while a backup operation is underway or being initialized. -** If SQLite is compiled and configured to support threadsafe database +** ^If SQLite is compiled and configured to support threadsafe database ** connections, then the source database connection may be used concurrently ** from within other threads. ** -** However, the application must guarantee that the destination database -** connection handle is not passed to any other API (by any thread) after +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after ** sqlite3_backup_init() is called and before the corresponding call to -** sqlite3_backup_finish(). Unfortunately SQLite does not currently check -** for this, if the application does use the destination [database connection] -** for some other purpose during a backup operation, things may appear to -** work correctly but in fact be subtly malfunctioning. Use of the -** destination database connection while a backup is in progress might -** also cause a mutex deadlock. +** sqlite3_backup_finish(). SQLite does not currently check to see +** if the application incorrectly accesses the destination [database connection] +** and so no error code is reported, but the operations may malfunction +** nevertheless. Use of the destination database connection while a +** backup is in progress might also also cause a mutex deadlock. ** -** Furthermore, if running in [shared cache mode], the application must +** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database ** is not accessed while the backup is running. In practice this means -** that the application must guarantee that the file-system file being +** that the application must guarantee that the disk file being ** backed up to is not accessed by any connection within the process, ** not just the specific connection that was passed to sqlite3_backup_init(). ** @@ -5628,48 +5561,48 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** CAPI3REF: Unlock Notification ** EXPERIMENTAL ** -** When running in shared-cache mode, a database operation may fail with +** ^When running in shared-cache mode, a database operation may fail with ** an [SQLITE_LOCKED] error if the required locks on the shared-cache or ** individual tables within the shared-cache cannot be obtained. See ** [SQLite Shared-Cache Mode] for a description of shared-cache locking. -** This API may be used to register a callback that SQLite will invoke +** ^This API may be used to register a callback that SQLite will invoke ** when the connection currently holding the required lock relinquishes it. -** This API is only available if the library was compiled with the +** ^This API is only available if the library was compiled with the ** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. ** ** See Also: [Using the SQLite Unlock Notification Feature]. ** -** Shared-cache locks are released when a database connection concludes +** ^Shared-cache locks are released when a database connection concludes ** its current transaction, either by committing it or rolling it back. ** -** When a connection (known as the blocked connection) fails to obtain a +** ^When a connection (known as the blocked connection) fails to obtain a ** shared-cache lock and SQLITE_LOCKED is returned to the caller, the ** identity of the database connection (the blocking connection) that -** has locked the required resource is stored internally. After an +** has locked the required resource is stored internally. ^After an ** application receives an SQLITE_LOCKED error, it may call the ** sqlite3_unlock_notify() method with the blocked connection handle as ** the first argument to register for a callback that will be invoked -** when the blocking connections current transaction is concluded. The +** when the blocking connections current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] ** call that concludes the blocking connections transaction. ** -** If sqlite3_unlock_notify() is called in a multi-threaded application, +** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, ** there is a chance that the blocking connection will have already ** concluded its transaction by the time sqlite3_unlock_notify() is invoked. ** If this happens, then the specified callback is invoked immediately, -** from within the call to sqlite3_unlock_notify(). +** from within the call to sqlite3_unlock_notify().)^ ** -** If the blocked connection is attempting to obtain a write-lock on a +** ^If the blocked connection is attempting to obtain a write-lock on a ** shared-cache table, and more than one other connection currently holds ** a read-lock on the same table, then SQLite arbitrarily selects one of ** the other connections to use as the blocking connection. ** -** There may be at most one unlock-notify callback registered by a +** ^(There may be at most one unlock-notify callback registered by a ** blocked connection. If sqlite3_unlock_notify() is called when the ** blocked connection already has a registered unlock-notify callback, -** then the new callback replaces the old. If sqlite3_unlock_notify() is +** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is cancelled. The blocked connections +** unlock-notify callback is cancelled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -5677,7 +5610,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** any sqlite3_xxx API functions from within an unlock-notify callback, a ** crash or deadlock may be the result. ** -** Unless deadlock is detected (see below), sqlite3_unlock_notify() always +** ^Unless deadlock is detected (see below), sqlite3_unlock_notify() always ** returns SQLITE_OK. ** ** Callback Invocation Details @@ -5691,7 +5624,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** ** When a blocking connections transaction is concluded, there may be ** more than one blocked connection that has registered for an unlock-notify -** callback. If two or more such blocked connections have specified the +** callback. ^If two or more such blocked connections have specified the ** same callback function, then instead of invoking the callback function ** multiple times, it is invoked once with the set of void* context pointers ** specified by the blocked connections bundled together into an array. @@ -5709,16 +5642,16 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** will proceed and the system may remain deadlocked indefinitely. ** ** To avoid this scenario, the sqlite3_unlock_notify() performs deadlock -** detection. If a given call to sqlite3_unlock_notify() would put the +** detection. ^If a given call to sqlite3_unlock_notify() would put the ** system in a deadlocked state, then SQLITE_LOCKED is returned and no ** unlock-notify callback is registered. The system is said to be in ** a deadlocked state if connection A has registered for an unlock-notify ** callback on the conclusion of connection B's transaction, and connection ** B has itself registered for an unlock-notify callback when connection -** A's transaction is concluded. Indirect deadlock is also detected, so +** A's transaction is concluded. ^Indirect deadlock is also detected, so ** the system is also considered to be deadlocked if connection B has ** registered for an unlock-notify callback on the conclusion of connection -** C's transaction, where connection C is waiting on connection A. Any +** C's transaction, where connection C is waiting on connection A. ^Any ** number of levels of indirection are allowed. ** ** The "DROP TABLE" Exception @@ -5734,10 +5667,10 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** or "DROP INDEX" query, an infinite loop might be the result. ** ** One way around this problem is to check the extended error code returned -** by an sqlite3_step() call. If there is a blocking connection, then the +** by an sqlite3_step() call. ^(If there is a blocking connection, then the ** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in ** the special "DROP TABLE/INDEX" case, the extended error code is just -** SQLITE_LOCKED. +** SQLITE_LOCKED.)^ */ int sqlite3_unlock_notify( sqlite3 *pBlocked, /* Waiting connection */ @@ -5750,13 +5683,37 @@ int sqlite3_unlock_notify( ** CAPI3REF: String Comparison ** EXPERIMENTAL ** -** The [sqlite3_strnicmp()] API allows applications and extensions to +** ^The [sqlite3_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a ** case-indendent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: Error Logging Interface +** EXPERIMENTAL +** +** ^The [sqlite3_log()] interface writes a message into the error log +** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. +** ^If logging is enabled, the zFormat string and subsequent arguments are +** passed through to [sqlite3_vmprintf()] to generate the final output string. +** +** The sqlite3_log() interface is intended for use by extensions such as +** virtual tables, collating functions, and SQL functions. While there is +** nothing to prevent an application from calling sqlite3_log(), doing so +** is considered bad form. +** +** The zFormat string must not be NULL. +** +** To avoid deadlocks and other threading problems, the sqlite3_log() routine +** will not use dynamically allocated memory. The log message is stored in +** a fixed-length buffer on the stack. If the log message is longer than +** a few hundred characters, it will be truncated to the length of the +** buffer. +*/ +void sqlite3_log(int iErrCode, const char *zFormat, ...); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1202971a..5720b9cf 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -76,41 +76,43 @@ #include #endif +/* +** The number of samples of an index that SQLite takes in order to +** construct a histogram of the table content when running ANALYZE +** and with SQLITE_ENABLE_STAT2 +*/ #define SQLITE_INDEX_SAMPLES 10 /* -** This macro is used to "hide" some ugliness in casting an int -** value to a ptr value under the MSVC 64-bit compiler. Casting -** non 64-bit values to ptr types results in a "hard" error with -** the MSVC 64-bit compiler which this attempts to avoid. +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. ** -** A simple compiler pragma or casting sequence could not be found -** to correct this in all situations, so this macro was introduced. -** -** It could be argued that the intptr_t type could be used in this -** case, but that type is not available on all compilers, or -** requires the #include of specific headers which differs between -** platforms. +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that very from one machine to the next. ** ** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on ** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). ** So we have to define the macros in different ways depending on the ** compiler. */ -#if defined(__GNUC__) -# if defined(HAVE_STDINT_H) -# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) -# else -# define SQLITE_INT_TO_PTR(X) ((void*)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(X)) -# endif -#else -# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) -# define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define SQLITE_INT_TO_PTR(X) ((void*)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(X)) #endif - /* ** The SQLITE_THREADSAFE macro must be defined as either 0 or 1. ** Older versions of SQLite used an optional THREADSAFE macro. @@ -140,23 +142,18 @@ ** ** SQLITE_SYSTEM_MALLOC // Use normal system malloc() ** SQLITE_MEMDEBUG // Debugging version of system malloc() -** SQLITE_MEMORY_SIZE // internal allocator #1 -** SQLITE_MMAP_HEAP_SIZE // internal mmap() allocator -** SQLITE_POW2_MEMORY_SIZE // internal power-of-two allocator +** +** (Historical note: There used to be several other options, but we've +** pared it down to just these two.) ** ** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as ** the default. */ -#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)+\ - defined(SQLITE_MEMORY_SIZE)+defined(SQLITE_MMAP_HEAP_SIZE)+\ - defined(SQLITE_POW2_MEMORY_SIZE)>1 +#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)>1 # error "At most one of the following compile-time configuration options\ - is allows: SQLITE_SYSTEM_MALLOC, SQLITE_MEMDEBUG, SQLITE_MEMORY_SIZE,\ - SQLITE_MMAP_HEAP_SIZE, SQLITE_POW2_MEMORY_SIZE" + is allows: SQLITE_SYSTEM_MALLOC, SQLITE_MEMDEBUG" #endif -#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)+\ - defined(SQLITE_MEMORY_SIZE)+defined(SQLITE_MMAP_HEAP_SIZE)+\ - defined(SQLITE_POW2_MEMORY_SIZE)==0 +#if defined(SQLITE_SYSTEM_MALLOC)+defined(SQLITE_MEMDEBUG)==0 # define SQLITE_SYSTEM_MALLOC 1 #endif @@ -328,20 +325,6 @@ #define OMIT_TEMPDB 0 #endif -/* -** If the following macro is set to 1, then NULL values are considered -** distinct when determining whether or not two entries are the same -** in a UNIQUE index. This is the way PostgreSQL, Oracle, DB2, MySQL, -** OCELOT, and Firebird all work. The SQL92 spec explicitly says this -** is the way things are suppose to work. -** -** If the following macro is set to 0, the NULLs are indistinct for -** a UNIQUE index. In this mode, you can only have a single NULL entry -** for a column declared UNIQUE. This is the way Informix and SQL Server -** work. -*/ -#define NULL_DISTINCT_FOR_UNIQUE 1 - /* ** The "file format" number is an integer that is incremented whenever ** the VDBE-level file format changes. The following macros define the @@ -353,6 +336,10 @@ # define SQLITE_DEFAULT_FILE_FORMAT 1 #endif +/* +** Determine whether triggers are recursive by default. This can be +** changed at run-time using a pragma. +*/ #ifndef SQLITE_DEFAULT_RECURSIVE_TRIGGERS # define SQLITE_DEFAULT_RECURSIVE_TRIGGERS 0 #endif @@ -597,7 +584,6 @@ typedef struct AggInfo AggInfo; typedef struct AuthContext AuthContext; typedef struct AutoincInfo AutoincInfo; typedef struct Bitvec Bitvec; -typedef struct RowSet RowSet; typedef struct CollSeq CollSeq; typedef struct Column Column; typedef struct Db Db; @@ -618,6 +604,7 @@ typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; typedef struct Parse Parse; +typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; typedef struct SrcList SrcList; @@ -625,9 +612,9 @@ typedef struct StrAccum StrAccum; typedef struct Table Table; typedef struct TableLock TableLock; typedef struct Token Token; +typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; -typedef struct Trigger Trigger; typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; typedef struct Walker Walker; @@ -693,7 +680,7 @@ struct Schema { /* ** These macros can be used to test, set, or clear bits in the -** Db.flags field. +** Db.pSchema->flags field. */ #define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))==(P)) #define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))!=0) @@ -701,7 +688,7 @@ struct Schema { #define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->flags&=~(P) /* -** Allowed values for the DB.flags field. +** Allowed values for the DB.pSchema->flags field. ** ** The DB_SchemaLoaded flag is set after the database schema has been ** read into internal hash tables. @@ -765,7 +752,7 @@ struct FuncDefHash { }; /* -** Each database is an instance of the following structure. +** Each database connection is an instance of the following structure. ** ** The sqlite.lastRowid records the last insert rowid generated by an ** insert statement. Inserts on views do not affect its value. Each @@ -804,6 +791,7 @@ struct sqlite3 { u8 dfltLockMode; /* Default locking-mode for attached dbs */ u8 dfltJournalMode; /* Default journal mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ + u8 suppressErr; /* Do not issue error messages if true */ int nextPagesize; /* Pagesize after VACUUM if >0 */ int nTable; /* Number of tables in the database */ CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ @@ -898,37 +886,43 @@ struct sqlite3 { #define ENC(db) ((db)->aDb[0].pSchema->enc) /* -** Possible values for the sqlite.flags and or Db.flags fields. -** -** On sqlite.flags, the SQLITE_InTrans value means that we have -** executed a BEGIN. On Db.flags, SQLITE_InTrans means a statement -** transaction is active on that particular database file. +** Possible values for the sqlite3.flags. */ -#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ -#define SQLITE_InTrans 0x00000008 /* True if in a transaction */ -#define SQLITE_InternChanges 0x00000010 /* Uncommitted Hash table changes */ -#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ -#define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ -#define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ +#define SQLITE_VdbeTrace 0x00000100 /* True to trace VDBE execution */ +#define SQLITE_InternChanges 0x00000200 /* Uncommitted Hash table changes */ +#define SQLITE_FullColNames 0x00000400 /* Show full column names on SELECT */ +#define SQLITE_ShortColNames 0x00000800 /* Show short columns names */ +#define SQLITE_CountRows 0x00001000 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ /* the count using a callback. */ -#define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ +#define SQLITE_NullCallback 0x00002000 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00000200 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00000400 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00000800 /* OK to update SQLITE_MASTER */ -#define SQLITE_NoReadlock 0x00001000 /* Readlocks are omitted when +#define SQLITE_SqlTrace 0x00004000 /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing 0x00008000 /* Debug listings of VDBE programs */ +#define SQLITE_WriteSchema 0x00010000 /* OK to update SQLITE_MASTER */ +#define SQLITE_NoReadlock 0x00020000 /* Readlocks are omitted when ** accessing read-only databases */ -#define SQLITE_IgnoreChecks 0x00002000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x00004000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00008000 /* Create new databases in format 1 */ -#define SQLITE_FullFSync 0x00010000 /* Use full fsync on the backend */ -#define SQLITE_LoadExtension 0x00020000 /* Enable load_extension */ +#define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ +#define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ +#define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ +#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ +#define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ +#define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ -#define SQLITE_RecoveryMode 0x00040000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x00100000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x00200000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x00400000 /* Enforce foreign key constraints */ +/* +** Bits of the sqlite3.flags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. +** These must be the low-order bits of the flags field. +*/ +#define SQLITE_QueryFlattener 0x01 /* Disable query flattening */ +#define SQLITE_ColumnCache 0x02 /* Disable the column cache */ +#define SQLITE_IndexSort 0x04 /* Disable indexes for sorting */ +#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ +#define SQLITE_IndexCover 0x10 /* Disable index covering table */ +#define SQLITE_OptMask 0x1f /* Mask of all disablable opts */ /* ** Possible values for the sqlite.magic field. @@ -1642,14 +1636,13 @@ struct Expr { #define EP_DblQuoted 0x0040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x0080 /* True for an infix function: LIKE, GLOB, etc */ #define EP_ExpCollate 0x0100 /* Collating sequence specified explicitly */ -#define EP_AnyAff 0x0200 /* Can take a cached column of any affinity */ -#define EP_FixedDest 0x0400 /* Result needed in a specific register */ -#define EP_IntValue 0x0800 /* Integer value contained in u.iValue */ -#define EP_xIsSelect 0x1000 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_FixedDest 0x0200 /* Result needed in a specific register */ +#define EP_IntValue 0x0400 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x0800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Reduced 0x2000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ -#define EP_TokenOnly 0x4000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ -#define EP_Static 0x8000 /* Held in memory not obtained from malloc() */ +#define EP_Reduced 0x1000 /* Expr struct is EXPR_REDUCEDSIZE bytes only */ +#define EP_TokenOnly 0x2000 /* Expr struct is EXPR_TOKENONLYSIZE bytes only */ +#define EP_Static 0x4000 /* Held in memory not obtained from malloc() */ /* ** The following are the meanings of bits in the Expr.flags2 field. @@ -1894,6 +1887,7 @@ struct WhereLevel { #define WHERE_OMIT_OPEN 0x0010 /* Table cursor are already open */ #define WHERE_OMIT_CLOSE 0x0020 /* Omit close of table & index cursors */ #define WHERE_FORCE_TABLE 0x0040 /* Do not use an index-only search */ +#define WHERE_ONETABLE_ONLY 0x0080 /* Only code the 1st table in pTabList */ /* ** The WHERE clause processing routine has two halves. The @@ -1906,6 +1900,7 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ SrcList *pTabList; /* List of tables in the join */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ @@ -2125,7 +2120,6 @@ struct Parse { struct yColCache { int iTable; /* Table cursor number */ int iColumn; /* Table column number */ - u8 affChange; /* True if this register has had an affinity change */ u8 tempReg; /* iReg is a temp register that needs to be freed */ int iLevel; /* Nesting level */ int iReg; /* Reg with value of this column. 0 means none. */ @@ -2374,6 +2368,8 @@ struct Sqlite3Config { int isPCacheInit; /* True after malloc is initialized */ sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ int nRefInitMutex; /* Number of users of pInitMutex */ + void (*xLog)(void*,int,const char*); /* Function for logging */ + void *pLogArg; /* First argument to xLog() */ }; /* @@ -2415,16 +2411,27 @@ int sqlite3WalkSelectFrom(Walker*, Select*); } /* -** The SQLITE_CORRUPT_BKPT macro can be either a constant (for production -** builds) or a function call (for debugging). If it is a function call, -** it allows the operator to set a breakpoint at the spot where database -** corruption is first detected. +** The SQLITE_*_BKPT macros are substitutes for the error codes with +** the same name but without the _BKPT suffix. These macros invoke +** routines that report the line-number on which the error originated +** using sqlite3_log(). The routines also provide a convenient place +** to set a debugger breakpoint. */ -#ifdef SQLITE_DEBUG - int sqlite3Corrupt(void); -# define SQLITE_CORRUPT_BKPT sqlite3Corrupt() -#else -# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT +int sqlite3CorruptError(int); +int sqlite3MisuseError(int); +int sqlite3CantopenError(int); +#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) +#define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) +#define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) + + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 #endif /* @@ -2526,7 +2533,11 @@ int sqlite3StatusValue(int); void sqlite3StatusAdd(int, int); void sqlite3StatusSet(int, int); -int sqlite3IsNaN(double); +#ifndef SQLITE_OMIT_FLOATING_POINT + int sqlite3IsNaN(double); +#else +# define sqlite3IsNaN(X) 0 +#endif void sqlite3VXPrintf(StrAccum*, int, const char*, va_list); #ifndef SQLITE_OMIT_TRACE @@ -2543,7 +2554,6 @@ char *sqlite3MAppendf(sqlite3*,char*,const char*,...); #endif void sqlite3SetString(char **, sqlite3*, const char*, ...); void sqlite3ErrorMsg(Parse*, const char*, ...); -void sqlite3ErrorClear(Parse*); int sqlite3Dequote(char*); int sqlite3KeywordCode(const unsigned char*, int); int sqlite3RunParser(Parse*, const char*, char **); @@ -2644,13 +2654,13 @@ void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16); void sqlite3WhereEnd(WhereInfo*); -int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, int); +int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCodeCopy(Parse*, int, int, int); void sqlite3ExprCacheStore(Parse*, int, int, int); void sqlite3ExprCachePush(Parse*); void sqlite3ExprCachePop(Parse*, int); -void sqlite3ExprCacheRemove(Parse*, int); +void sqlite3ExprCacheRemove(Parse*, int, int); void sqlite3ExprCacheClear(Parse*); void sqlite3ExprCacheAffinityChange(Parse*, int, int); void sqlite3ExprHardCopy(Parse*,int,int); @@ -2713,13 +2723,6 @@ FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,int); void sqlite3RegisterBuiltinFunctions(sqlite3*); void sqlite3RegisterDateTimeFunctions(void); void sqlite3RegisterGlobalFunctions(void); -#ifdef SQLITE_DEBUG - int sqlite3SafetyOn(sqlite3*); - int sqlite3SafetyOff(sqlite3*); -#else -# define sqlite3SafetyOn(A) 0 -# define sqlite3SafetyOff(A) 0 -#endif int sqlite3SafetyCheckOk(sqlite3*); int sqlite3SafetyCheckSickOrOk(sqlite3*); void sqlite3ChangeCookie(Parse*, int); @@ -2855,7 +2858,7 @@ void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); void sqlite3ValueFree(sqlite3_value*); sqlite3_value *sqlite3ValueNew(sqlite3 *); -char *sqlite3Utf16to8(sqlite3 *, const void*, int); +char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); #ifdef SQLITE_ENABLE_STAT2 char *sqlite3Utf8to16(sqlite3 *, u8, char *, int, int *); #endif diff --git a/src/status.c b/src/status.c index 58a7e68c..f4c77a91 100644 --- a/src/status.c +++ b/src/status.c @@ -83,7 +83,7 @@ void sqlite3StatusSet(int op, int X){ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ wsdStatInit; if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; diff --git a/src/test1.c b/src/test1.c index 5a2505a7..7c5d4971 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4909,7 +4909,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ extern int sqlite3_open_file_count; extern int sqlite3_sort_count; extern int sqlite3_current_time; -#if SQLITE_OS_UNIX && defined(__APPLE__) +#if SQLITE_OS_UNIX && defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE extern int sqlite3_hostid_num; #endif extern int sqlite3_max_blobsize; @@ -5140,7 +5140,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ (char*)&sqlite3_open_file_count, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite_current_time", (char*)&sqlite3_current_time, TCL_LINK_INT); -#if SQLITE_OS_UNIX && defined(__APPLE__) +#if SQLITE_OS_UNIX && defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE Tcl_LinkVar(interp, "sqlite_hostid_num", (char*)&sqlite3_hostid_num, TCL_LINK_INT); #endif diff --git a/src/test3.c b/src/test3.c index 06222f5f..68f5648a 100644 --- a/src/test3.c +++ b/src/test3.c @@ -219,7 +219,7 @@ static int btree_cursor( Btree *pBt; int iTable; BtCursor *pCur; - int rc; + int rc = SQLITE_OK; int wrFlag; char zBuf[30]; @@ -234,7 +234,9 @@ static int btree_cursor( pCur = (BtCursor *)ckalloc(sqlite3BtreeCursorSize()); memset(pCur, 0, sqlite3BtreeCursorSize()); sqlite3BtreeEnter(pBt); +#ifndef SQLITE_OMIT_SHARED_CACHE rc = sqlite3BtreeLockTable(pBt, iTable, wrFlag); +#endif if( rc==SQLITE_OK ){ rc = sqlite3BtreeCursor(pBt, iTable, wrFlag, 0, pCur); } diff --git a/src/test_config.c b/src/test_config.c index 1cb33e8a..73503926 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -183,7 +183,11 @@ static void set_options(Tcl_Interp *interp){ TCL_GLOBAL_ONLY); #endif - +#ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS + Tcl_SetVar2(interp, "sqlite_options", "compileoption_diags", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "compileoption_diags", "1", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_COMPLETE Tcl_SetVar2(interp, "sqlite_options", "complete", "0", TCL_GLOBAL_ONLY); diff --git a/src/test_func.c b/src/test_func.c index 00c45d03..c3277a06 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -341,6 +341,7 @@ static void testHexToBin(const char *zIn, char *zOut){ ** Convert the input string from HEX into binary. Then return the ** result using sqlite3_result_text16le(). */ +#ifndef SQLITE_OMIT_UTF16 static void testHexToUtf16be( sqlite3_context *pCtx, int nArg, @@ -360,6 +361,7 @@ static void testHexToUtf16be( sqlite3_result_text16be(pCtx, zOut, n/2, sqlite3_free); } } +#endif /* ** hex_to_utf8(HEX) @@ -393,6 +395,7 @@ static void testHexToUtf8( ** Convert the input string from HEX into binary. Then return the ** result using sqlite3_result_text16le(). */ +#ifndef SQLITE_OMIT_UTF16 static void testHexToUtf16le( sqlite3_context *pCtx, int nArg, @@ -412,6 +415,7 @@ static void testHexToUtf16le( sqlite3_result_text16le(pCtx, zOut, n/2, sqlite3_free); } } +#endif static int registerTestFunctions(sqlite3 *db){ static const struct { diff --git a/src/test_hexio.c b/src/test_hexio.c index db1aa44e..02bd60c8 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -330,12 +330,12 @@ static int getFts3Varint(const char *p, sqlite_int64 *v){ /* -** USAGE: read_varint BLOB VARNAME +** USAGE: read_fts3varint BLOB VARNAME ** ** Read a varint from the start of BLOB. Set variable VARNAME to contain ** the interpreted value. Return the number of bytes of BLOB consumed. */ -static int read_varint( +static int read_fts3varint( void * clientData, Tcl_Interp *interp, int objc, @@ -373,7 +373,7 @@ int Sqlitetest_hexio_Init(Tcl_Interp *interp){ { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, - { "read_varint", read_varint }, + { "read_fts3varint", read_fts3varint }, }; int i; for(i=0; izErrMsg ){ *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s", *pzErrMsg); pParse->zErrMsg = 0; nErr++; } diff --git a/src/trigger.c b/src/trigger.c index 51969ce2..f57d5600 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -126,7 +126,8 @@ void sqlite3BeginTrigger( goto trigger_cleanup; } pTab = sqlite3SrcListLookup(pParse, pTableName); - if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + if( db->init.busy==0 && pName2->n==0 && pTab + && pTab->pSchema==db->aDb[1].pSchema ){ iDb = 1; } @@ -254,12 +255,12 @@ void sqlite3FinishTrigger( TriggerStep *pStepList, /* The triggered program */ Token *pAll /* Token that describes the complete CREATE TRIGGER */ ){ - Trigger *pTrig = pParse->pNewTrigger; /* Trigger being finished */ - char *zName; /* Name of trigger */ - sqlite3 *db = pParse->db; /* The database */ - DbFixer sFix; - int iDb; /* Database containing the trigger */ - Token nameToken; /* Trigger name for error reporting */ + Trigger *pTrig = pParse->pNewTrigger; /* Trigger being finished */ + char *zName; /* Name of trigger */ + sqlite3 *db = pParse->db; /* The database */ + DbFixer sFix; /* Fixer object */ + int iDb; /* Database containing the trigger */ + Token nameToken; /* Trigger name for error reporting */ pTrig = pParse->pNewTrigger; pParse->pNewTrigger = 0; @@ -278,7 +279,7 @@ void sqlite3FinishTrigger( goto triggerfinish_cleanup; } - /* if we are not initializing, and this trigger is not on a TEMP table, + /* if we are not initializing, ** build the sqlite_master entry */ if( !db->init.busy ){ diff --git a/src/utf.c b/src/utf.c index 4ea63eed..8312cf93 100644 --- a/src/utf.c +++ b/src/utf.c @@ -437,11 +437,11 @@ int sqlite3Utf8To8(unsigned char *zIn){ ** ** NULL is returned if there is an allocation error. */ -char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte){ +char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){ Mem m; memset(&m, 0, sizeof(m)); m.db = db; - sqlite3VdbeMemSetStr(&m, z, nByte, SQLITE_UTF16NATIVE, SQLITE_STATIC); + sqlite3VdbeMemSetStr(&m, z, nByte, enc, SQLITE_STATIC); sqlite3VdbeChangeEncoding(&m, SQLITE_UTF8); if( db->mallocFailed ){ sqlite3VdbeMemRelease(&m); @@ -449,7 +449,9 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte){ } assert( (m.flags & MEM_Term)!=0 || db->mallocFailed ); assert( (m.flags & MEM_Str)!=0 || db->mallocFailed ); - return (m.flags & MEM_Dyn)!=0 ? m.z : sqlite3DbStrDup(db, m.z); + assert( (m.flags & MEM_Dyn)!=0 || db->mallocFailed ); + assert( m.z || db->mallocFailed ); + return m.z; } /* diff --git a/src/util.c b/src/util.c index 81e42b4a..ab31424e 100644 --- a/src/util.c +++ b/src/util.c @@ -31,6 +31,7 @@ void sqlite3Coverage(int x){ } #endif +#ifndef SQLITE_OMIT_FLOATING_POINT /* ** Return true if the floating point value is Not a Number (NaN). ** @@ -75,6 +76,7 @@ int sqlite3IsNaN(double x){ testcase( rc ); return rc; } +#endif /* SQLITE_OMIT_FLOATING_POINT */ /* ** Compute a string length that is limited to what can be stored in @@ -146,23 +148,20 @@ void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ ** (sqlite3_step() etc.). */ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ + char *zMsg; va_list ap; sqlite3 *db = pParse->db; - pParse->nErr++; - sqlite3DbFree(db, pParse->zErrMsg); va_start(ap, zFormat); - pParse->zErrMsg = sqlite3VMPrintf(db, zFormat, ap); + zMsg = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); - pParse->rc = SQLITE_ERROR; -} - -/* -** Clear the error message in pParse, if any -*/ -void sqlite3ErrorClear(Parse *pParse){ - sqlite3DbFree(pParse->db, pParse->zErrMsg); - pParse->zErrMsg = 0; - pParse->nErr = 0; + if( db->suppressErr ){ + sqlite3DbFree(db, zMsg); + }else{ + pParse->nErr++; + sqlite3DbFree(db, pParse->zErrMsg); + pParse->zErrMsg = zMsg; + pParse->rc = SQLITE_ERROR; + } } /* @@ -255,6 +254,7 @@ int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ z += incr; *realnum = 0; while( sqlite3Isdigit(*z) ){ z += incr; } +#ifndef SQLITE_OMIT_FLOATING_POINT if( *z=='.' ){ z += incr; if( !sqlite3Isdigit(*z) ) return 0; @@ -268,6 +268,7 @@ int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ while( sqlite3Isdigit(*z) ){ z += incr; } *realnum = 1; } +#endif return *z==0; } @@ -429,6 +430,9 @@ static int compare2pow63(const char *zNum){ c = memcmp(zNum,"922337203685477580",18)*10; if( c==0 ){ c = zNum[18] - '8'; + testcase( c==(-1) ); + testcase( c==0 ); + testcase( c==(+1) ); } return c; } @@ -465,6 +469,9 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum){ v = v*10 + c - '0'; } *pNum = neg ? -v : v; + testcase( i==18 ); + testcase( i==19 ); + testcase( i==20 ); if( c!=0 || (i==0 && zStart==zNum) || i>19 ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranting that it is too large) */ @@ -508,6 +515,9 @@ int sqlite3FitsIn64Bits(const char *zNum, int negFlag){ zNum++; /* Skip leading zeros. Ticket #2454 */ } for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); } + testcase( i==18 ); + testcase( i==19 ); + testcase( i==20 ); if( i<19 ){ /* Guaranteed to fit if less than 19 digits */ return 1; @@ -548,9 +558,11 @@ int sqlite3GetInt32(const char *zNum, int *pValue){ ** 1234567890 ** 2^31 -> 2147483648 */ + testcase( i==10 ); if( i>10 ){ return 0; } + testcase( v-neg==2147483647 ); if( v-neg>2147483647 ){ return 0; } @@ -638,6 +650,19 @@ int sqlite3PutVarint32(unsigned char *p, u32 v){ return sqlite3PutVarint(p, v); } +/* +** Bitmasks used by sqlite3GetVarint(). These precomputed constants +** are defined here rather than simply putting the constant expressions +** inline in order to work around bugs in the RVT compiler. +** +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f +** +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 +*/ +#define SLOT_2_0 0x001fc07f +#define SLOT_4_2_0 0xf01fc07f + + /* ** Read a 64-bit variable-length integer from memory starting at p[0]. ** Return the number of bytes read. The value is stored in *v. @@ -665,13 +690,17 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ return 2; } + /* Verify that constants are precomputed correctly */ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + p++; a = a<<14; a |= *p; /* a: p0<<14 | p2 (unmasked) */ if (!(a&0x80)) { - a &= (0x7f<<14)|(0x7f); + a &= SLOT_2_0; b &= 0x7f; b = b<<7; a |= b; @@ -680,14 +709,14 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ } /* CSE1 from below */ - a &= (0x7f<<14)|(0x7f); + a &= SLOT_2_0; p++; b = b<<14; b |= *p; /* b: p1<<14 | p3 (unmasked) */ if (!(b&0x80)) { - b &= (0x7f<<14)|(0x7f); + b &= SLOT_2_0; /* moved CSE1 up */ /* a &= (0x7f<<14)|(0x7f); */ a = a<<7; @@ -701,7 +730,7 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ /* moved CSE1 up */ /* a &= (0x7f<<14)|(0x7f); */ - b &= (0x7f<<14)|(0x7f); + b &= SLOT_2_0; s = a; /* s: p0<<14 | p2 (masked) */ @@ -734,7 +763,7 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ { /* we can skip this cause it was (effectively) done above in calc'ing s */ /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ - a &= (0x7f<<14)|(0x7f); + a &= SLOT_2_0; a = a<<7; a |= b; s = s>>18; @@ -748,8 +777,8 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ /* a: p2<<28 | p4<<14 | p6 (unmasked) */ if (!(a&0x80)) { - a &= (0x1f<<28)|(0x7f<<14)|(0x7f); - b &= (0x7f<<14)|(0x7f); + a &= SLOT_4_2_0; + b &= SLOT_2_0; b = b<<7; a |= b; s = s>>11; @@ -758,14 +787,14 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ } /* CSE2 from below */ - a &= (0x7f<<14)|(0x7f); + a &= SLOT_2_0; p++; b = b<<14; b |= *p; /* b: p3<<28 | p5<<14 | p7 (unmasked) */ if (!(b&0x80)) { - b &= (0x1f<<28)|(0x7f<<14)|(0x7f); + b &= SLOT_4_2_0; /* moved CSE2 up */ /* a &= (0x7f<<14)|(0x7f); */ a = a<<7; @@ -782,7 +811,7 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ /* moved CSE2 up */ /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ - b &= (0x7f<<14)|(0x7f); + b &= SLOT_2_0; b = b<<8; a |= b; @@ -902,9 +931,9 @@ u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ /* a: p0<<28 | p2<<14 | p4 (unmasked) */ if (!(a&0x80)) { - /* Walues between 268435456 and 34359738367 */ - a &= (0x1f<<28)|(0x7f<<14)|(0x7f); - b &= (0x1f<<28)|(0x7f<<14)|(0x7f); + /* Values between 268435456 and 34359738367 */ + a &= SLOT_4_2_0; + b &= SLOT_4_2_0; b = b<<7; *v = a | b; return 5; @@ -997,64 +1026,17 @@ void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ } #endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ - /* -** Change the sqlite.magic from SQLITE_MAGIC_OPEN to SQLITE_MAGIC_BUSY. -** Return an error (non-zero) if the magic was not SQLITE_MAGIC_OPEN -** when this routine is called. -** -** This routine is called when entering an SQLite API. The SQLITE_MAGIC_OPEN -** value indicates that the database connection passed into the API is -** open and is not being used by another thread. By changing the value -** to SQLITE_MAGIC_BUSY we indicate that the connection is in use. -** sqlite3SafetyOff() below will change the value back to SQLITE_MAGIC_OPEN -** when the API exits. -** -** This routine is a attempt to detect if two threads use the -** same sqlite* pointer at the same time. There is a race -** condition so it is possible that the error is not detected. -** But usually the problem will be seen. The result will be an -** error which can be used to debug the application that is -** using SQLite incorrectly. -** -** Ticket #202: If db->magic is not a valid open value, take care not -** to modify the db structure at all. It could be that db is a stale -** pointer. In other words, it could be that there has been a prior -** call to sqlite3_close(db) and db has been deallocated. And we do -** not want to write into deallocated memory. +** Log an error that is an API call on a connection pointer that should +** not have been used. The "type" of connection pointer is given as the +** argument. The zType is a word like "NULL" or "closed" or "invalid". */ -#ifdef SQLITE_DEBUG -int sqlite3SafetyOn(sqlite3 *db){ - if( db->magic==SQLITE_MAGIC_OPEN ){ - db->magic = SQLITE_MAGIC_BUSY; - assert( sqlite3_mutex_held(db->mutex) ); - return 0; - }else if( db->magic==SQLITE_MAGIC_BUSY ){ - db->magic = SQLITE_MAGIC_ERROR; - db->u1.isInterrupted = 1; - } - return 1; +static void logBadConnection(const char *zType){ + sqlite3_log(SQLITE_MISUSE, + "API call with %s database connection pointer", + zType + ); } -#endif - -/* -** Change the magic from SQLITE_MAGIC_BUSY to SQLITE_MAGIC_OPEN. -** Return an error (non-zero) if the magic was not SQLITE_MAGIC_BUSY -** when this routine is called. -*/ -#ifdef SQLITE_DEBUG -int sqlite3SafetyOff(sqlite3 *db){ - if( db->magic==SQLITE_MAGIC_BUSY ){ - db->magic = SQLITE_MAGIC_OPEN; - assert( sqlite3_mutex_held(db->mutex) ); - return 0; - }else{ - db->magic = SQLITE_MAGIC_ERROR; - db->u1.isInterrupted = 1; - return 1; - } -} -#endif /* ** Check to make sure we have a valid db pointer. This test is not @@ -1072,13 +1054,16 @@ int sqlite3SafetyOff(sqlite3 *db){ */ int sqlite3SafetyCheckOk(sqlite3 *db){ u32 magic; - if( db==0 ) return 0; + if( db==0 ){ + logBadConnection("NULL"); + return 0; + } magic = db->magic; - if( magic!=SQLITE_MAGIC_OPEN -#ifdef SQLITE_DEBUG - && magic!=SQLITE_MAGIC_BUSY -#endif - ){ + if( magic!=SQLITE_MAGIC_OPEN ){ + if( sqlite3SafetyCheckSickOrOk(db) ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + logBadConnection("unopened"); + } return 0; }else{ return 1; @@ -1089,6 +1074,11 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ magic = db->magic; if( magic!=SQLITE_MAGIC_SICK && magic!=SQLITE_MAGIC_OPEN && - magic!=SQLITE_MAGIC_BUSY ) return 0; - return 1; + magic!=SQLITE_MAGIC_BUSY ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + logBadConnection("invalid"); + return 0; + }else{ + return 1; + } } diff --git a/src/vacuum.c b/src/vacuum.c index 43545f3f..1d3b998c 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -18,28 +18,42 @@ #include "vdbeInt.h" #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) +/* +** Finalize a prepared statement. If there was an error, store the +** text of the error message in *pzErrMsg. Return the result code. +*/ +static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ + int rc; + rc = sqlite3VdbeFinalize((Vdbe*)pStmt); + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + return rc; +} + /* ** Execute zSql on database db. Return an error code. */ -static int execSql(sqlite3 *db, const char *zSql){ +static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ sqlite3_stmt *pStmt; VVA_ONLY( int rc; ) if( !zSql ){ return SQLITE_NOMEM; } if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); return sqlite3_errcode(db); } VVA_ONLY( rc = ) sqlite3_step(pStmt); assert( rc!=SQLITE_ROW ); - return sqlite3_finalize(pStmt); + return vacuumFinalize(db, pStmt, pzErrMsg); } /* ** Execute zSql on database db. The statement returns exactly ** one column. Execute this as SQL on the same database. */ -static int execExecSql(sqlite3 *db, const char *zSql){ +static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ sqlite3_stmt *pStmt; int rc; @@ -47,14 +61,14 @@ static int execExecSql(sqlite3 *db, const char *zSql){ if( rc!=SQLITE_OK ) return rc; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = execSql(db, (char*)sqlite3_column_text(pStmt, 0)); + rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); if( rc!=SQLITE_OK ){ - sqlite3_finalize(pStmt); + vacuumFinalize(db, pStmt, pzErrMsg); return rc; } } - return sqlite3_finalize(pStmt); + return vacuumFinalize(db, pStmt, pzErrMsg); } /* @@ -86,6 +100,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ int saved_flags; /* Saved value of the db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ + void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ int nRes; @@ -101,8 +116,10 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ saved_flags = db->flags; saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; + saved_xTrace = db->xTrace; db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; - db->flags &= ~SQLITE_ForeignKeys; + db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); + db->xTrace = 0; pMain = db->aDb[0].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); @@ -121,8 +138,12 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** time to parse and run the PRAGMA to turn journalling off than it does ** to write the journal header file. */ - zSql = "ATTACH '' AS vacuum_db;"; - rc = execSql(db, zSql); + if( sqlite3TempInMemory(db) ){ + zSql = "ATTACH ':memory:' AS vacuum_db;"; + }else{ + zSql = "ATTACH '' AS vacuum_db;"; + } + rc = execSql(db, pzErrMsg, zSql); if( rc!=SQLITE_OK ) goto end_of_vacuum; pDb = &db->aDb[db->nDb-1]; assert( strcmp(db->aDb[db->nDb-1].zName,"vacuum_db")==0 ); @@ -154,7 +175,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ rc = SQLITE_NOMEM; goto end_of_vacuum; } - rc = execSql(db, "PRAGMA vacuum_db.synchronous=OFF"); + rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); if( rc!=SQLITE_OK ){ goto end_of_vacuum; } @@ -165,23 +186,23 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ #endif /* Begin a transaction */ - rc = execSql(db, "BEGIN EXCLUSIVE;"); + rc = execSql(db, pzErrMsg, "BEGIN EXCLUSIVE;"); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ - rc = execExecSql(db, + rc = execExecSql(db, pzErrMsg, "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" " AND rootpage>0" ); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, + rc = execExecSql(db, pzErrMsg, "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, + rc = execExecSql(db, pzErrMsg, "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); if( rc!=SQLITE_OK ) goto end_of_vacuum; @@ -190,24 +211,23 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy ** the contents to the temporary database. */ - rc = execExecSql(db, + rc = execExecSql(db, pzErrMsg, "SELECT 'INSERT INTO vacuum_db.' || quote(name) " "|| ' SELECT * FROM main.' || quote(name) || ';'" "FROM main.sqlite_master " "WHERE type = 'table' AND name!='sqlite_sequence' " " AND rootpage>0" - ); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Copy over the sequence table */ - rc = execExecSql(db, + rc = execExecSql(db, pzErrMsg, "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " ); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, + rc = execExecSql(db, pzErrMsg, "SELECT 'INSERT INTO vacuum_db.' || quote(name) " "|| ' SELECT * FROM main.' || quote(name) || ';' " "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" @@ -220,7 +240,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ ** associated storage, so all we have to do is copy their entries ** from the SQLITE_MASTER table. */ - rc = execSql(db, + rc = execSql(db, pzErrMsg, "INSERT INTO vacuum_db.sqlite_master " " SELECT type, name, tbl_name, rootpage, sql" " FROM main.sqlite_master" @@ -283,6 +303,7 @@ end_of_vacuum: db->flags = saved_flags; db->nChange = saved_nChange; db->nTotalChange = saved_nTotalChange; + db->xTrace = saved_xTrace; /* Currently there is an SQL level transaction open on the vacuum ** database. No locks are held on any other files (since the main file diff --git a/src/vdbe.c b/src/vdbe.c index 10401a47..33dd2dac 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -239,17 +239,30 @@ static VdbeCursor *allocateCursor( static void applyNumericAffinity(Mem *pRec){ if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ int realnum; + u8 enc = pRec->enc; sqlite3VdbeMemNulTerminate(pRec); - if( (pRec->flags&MEM_Str) - && sqlite3IsNumber(pRec->z, &realnum, pRec->enc) ){ + if( (pRec->flags&MEM_Str) && sqlite3IsNumber(pRec->z, &realnum, enc) ){ i64 value; - sqlite3VdbeChangeEncoding(pRec, SQLITE_UTF8); - if( !realnum && sqlite3Atoi64(pRec->z, &value) ){ + char *zUtf8 = pRec->z; +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ){ + assert( pRec->db ); + zUtf8 = sqlite3Utf16to8(pRec->db, pRec->z, pRec->n, enc); + if( !zUtf8 ) return; + } +#endif + if( !realnum && sqlite3Atoi64(zUtf8, &value) ){ pRec->u.i = value; MemSetTypeFlag(pRec, MEM_Int); }else{ - sqlite3VdbeMemRealify(pRec); + sqlite3AtoF(zUtf8, &pRec->r); + MemSetTypeFlag(pRec, MEM_Real); } +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ){ + sqlite3DbFree(pRec->db, zUtf8); + } +#endif } } } @@ -538,7 +551,7 @@ static int checkSavepointCount(sqlite3 *db){ int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ - int pc; /* The program counter */ + int pc=0; /* The program counter */ Op *aOp = p->aOp; /* Copy of p->aOp */ Op *pOp; /* Current operation */ int rc = SQLITE_OK; /* Value to return */ @@ -563,7 +576,6 @@ int sqlite3VdbeExec( /*** INSERT STACK UNION HERE ***/ assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ - assert( db->magic==SQLITE_MAGIC_BUSY ); sqlite3VdbeMutexArrayEnter(p); if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or @@ -648,9 +660,7 @@ int sqlite3VdbeExec( if( checkProgress ){ if( db->nProgressOps==nProgressOps ){ int prc; - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; - prc =db->xProgress(db->pProgressArg); - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; + prc = db->xProgress(db->pProgressArg); if( prc!=0 ){ rc = SQLITE_INTERRUPT; goto vdbe_error_halt; @@ -850,7 +860,13 @@ case OP_Halt: { p->errorAction = (u8)pOp->p2; p->pc = pc; if( pOp->p4.z ){ + assert( p->rc!=SQLITE_OK ); sqlite3SetString(&p->zErrMsg, db, "%s", pOp->p4.z); + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pc, p->zSql, pOp->p4.z); + }else if( p->rc ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(pOp->p1, "constraint failed at %d in [%s]", pc, p->zSql); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -884,6 +900,7 @@ case OP_Int64: { /* out2-prerelease */ break; } +#ifndef SQLITE_OMIT_FLOATING_POINT /* Opcode: Real * P2 * P4 * ** ** P4 is a pointer to a 64-bit floating point value. @@ -895,6 +912,7 @@ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ pOut->r = *pOp->p4.pReal; break; } +#endif /* Opcode: String8 * P2 * P4 * ** @@ -1295,6 +1313,10 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ break; } } +#ifdef SQLITE_OMIT_FLOATING_POINT + pOut->u.i = rB; + MemSetTypeFlag(pOut, MEM_Int); +#else if( sqlite3IsNaN(rB) ){ goto arithmetic_result_is_null; } @@ -1303,6 +1325,7 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ if( (flags & MEM_Real)==0 ){ sqlite3VdbeIntegerAffinity(pOut); } +#endif } break; @@ -1393,21 +1416,12 @@ case OP_Function: { assert( pOp[-1].opcode==OP_CollSeq ); ctx.pColl = pOp[-1].p4.pColl; } - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; (*ctx.pFunc->xFunc)(&ctx, n, apVal); - if( sqlite3SafetyOn(db) ){ - sqlite3VdbeMemRelease(&ctx.s); - goto abort_due_to_misuse; - } if( db->mallocFailed ){ /* Even though a malloc() has failed, the implementation of the ** user function may have called an sqlite3_result_XXX() function ** to return a value. The following call releases any resources ** associated with such a value. - ** - ** Note: Maybe MemRelease() should be called if sqlite3SafetyOn() - ** fails also (the if(...) statement above). But if people are - ** misusing sqlite, they have bigger problems than a leaked value. */ sqlite3VdbeMemRelease(&ctx.s); goto no_mem; @@ -1530,6 +1544,7 @@ case OP_MustBeInt: { /* jump, in1 */ break; } +#ifndef SQLITE_OMIT_FLOATING_POINT /* Opcode: RealAffinity P1 * * * * ** ** If register P1 holds an integer convert it to a real value. @@ -1546,6 +1561,7 @@ case OP_RealAffinity: { /* in1 */ } break; } +#endif #ifndef SQLITE_OMIT_CAST /* Opcode: ToText P1 * * * * @@ -1629,7 +1645,7 @@ case OP_ToInt: { /* same as TK_TO_INT, in1 */ break; } -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) /* Opcode: ToReal P1 * * * * ** ** Force the value in register P1 to be a floating point number. @@ -1646,7 +1662,7 @@ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ } break; } -#endif /* SQLITE_OMIT_CAST */ +#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ /* Opcode: Lt P1 P2 P3 P4 P5 ** @@ -1728,9 +1744,13 @@ case OP_Gt: /* same as TK_GT, jump, in1, in3 */ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ int res; /* Result of the comparison of pIn1 against pIn3 */ char affinity; /* Affinity to use for comparison */ + u16 flags1; /* Copy of initial value of pIn1->flags */ + u16 flags3; /* Copy of initial value of pIn3->flags */ pIn1 = &aMem[pOp->p1]; pIn3 = &aMem[pOp->p3]; + flags1 = pIn1->flags; + flags3 = pIn3->flags; if( (pIn1->flags | pIn3->flags)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ @@ -1785,6 +1805,10 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ }else if( res ){ pc = pOp->p2-1; } + + /* Undo any changes made by applyAffinity() to the input registers. */ + pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (flags1&MEM_TypeMask); + pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (flags3&MEM_TypeMask); break; } @@ -2059,7 +2083,7 @@ case OP_Column: { u8 *zIdx; /* Index into header */ u8 *zEndHdr; /* Pointer to first byte after the header */ u32 offset; /* Offset into the data */ - u64 offset64; /* 64-bit offset. 64 bits needed to catch overflow */ + u32 szField; /* Number of bytes in the content of a field */ int szHdr; /* Size of the header size field at start of record */ int avail; /* Number of bytes of available data */ Mem *pReg; /* PseudoTable input register */ @@ -2234,12 +2258,16 @@ case OP_Column: { ** column and aOffset[i] will contain the offset from the beginning ** of the record to the start of the data for the i-th column */ - offset64 = offset; for(i=0; i zEndHdr)|| (offset64 > payloadSize) - || (zIdx==zEndHdr && offset64!=(u64)payloadSize) ){ + if( (zIdx > zEndHdr) || (offset > payloadSize) + || (zIdx==zEndHdr && offset!=payloadSize) ){ rc = SQLITE_CORRUPT_BKPT; goto op_column_out; } @@ -3106,7 +3134,7 @@ case OP_OpenEphemeral: { ** register P2. In other words, cursor P1 becomes an alias for the ** MEM_Blob content contained in register P2. ** -** A pseudo-table created by this opcode is used to hold the a single +** A pseudo-table created by this opcode is used to hold a single ** row output from the sorter so that the row can be decomposed into ** individual columns using the OP_Column opcode. The OP_Column opcode ** is the only cursor opcode that works with a pseudo-table. @@ -3663,7 +3691,7 @@ case OP_NewRowid: { /* out2-prerelease */ goto abort_due_to_error; } if( res ){ - v = 1; + v = 1; /* IMP: R-61914-48074 */ }else{ assert( sqlite3BtreeCursorIsValid(pC->pCursor) ); rc = sqlite3BtreeKeySize(pC->pCursor, &v); @@ -3671,7 +3699,7 @@ case OP_NewRowid: { /* out2-prerelease */ if( v==MAX_ROWID ){ pC->useRandomRowid = 1; }else{ - v++; + v++; /* IMP: R-29538-34987 */ } } } @@ -3695,7 +3723,7 @@ case OP_NewRowid: { /* out2-prerelease */ sqlite3VdbeMemIntegerify(pMem); assert( (pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){ - rc = SQLITE_FULL; + rc = SQLITE_FULL; /* IMP: R-12275-61338 */ goto abort_due_to_error; } if( vu.i+1 ){ @@ -3708,6 +3736,11 @@ case OP_NewRowid: { /* out2-prerelease */ sqlite3BtreeSetCachedRowid(pC->pCursor, vuseRandomRowid ){ + /* IMPLEMENTATION-OF: R-48598-02938 If the largest ROWID is equal to the + ** largest possible integer (9223372036854775807) then the database + ** engine starts picking candidate ROWIDs at random until it finds one + ** that is not previously used. + */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ v = db->lastRowid; @@ -3723,7 +3756,7 @@ case OP_NewRowid: { /* out2-prerelease */ cnt++; }while( cnt<100 && rc==SQLITE_OK && res==0 ); if( rc==SQLITE_OK && res==0 ){ - rc = SQLITE_FULL; + rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } } @@ -4031,12 +4064,10 @@ case OP_Rowid: { /* out2-prerelease */ pVtab = pC->pVtabCursor->pVtab; pModule = pVtab->pModule; assert( pModule->xRowid ); - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; rc = pModule->xRowid(pC->pVtabCursor, &v); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = pVtab->zErrMsg; pVtab->zErrMsg = 0; - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ assert( pC->pCursor!=0 ); @@ -4567,12 +4598,11 @@ case OP_ParseSchema: { initData.iDb = pOp->p1; initData.pzErrMsg = &p->zErrMsg; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s", + "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", db->aDb[iDb].zName, zMaster, pOp->p4.z); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ - (void)sqlite3SafetyOff(db); assert( db->init.busy==0 ); db->init.busy = 1; initData.rc = SQLITE_OK; @@ -4581,7 +4611,6 @@ case OP_ParseSchema: { if( rc==SQLITE_OK ) rc = initData.rc; sqlite3DbFree(db, zSql); db->init.busy = 0; - (void)sqlite3SafetyOn(db); } } sqlite3BtreeLeaveAll(db); @@ -5151,9 +5180,7 @@ case OP_AggFinal: { ** a transaction. */ case OP_Vacuum: { - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; rc = sqlite3RunVacuum(&p->zErrMsg, db); - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; break; } #endif @@ -5297,12 +5324,10 @@ case OP_VOpen: { pVtab = pOp->p4.pVtab->pVtab; pModule = (sqlite3_module *)pVtab->pModule; assert(pVtab && pModule); - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; rc = pModule->xOpen(pVtab, &pVtabCursor); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = pVtab->zErrMsg; pVtab->zErrMsg = 0; - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; if( SQLITE_OK==rc ){ /* Initialize sqlite3_vtab_cursor base class */ pVtabCursor->pVtab = pVtab; @@ -5376,7 +5401,6 @@ case OP_VFilter: { /* jump */ sqlite3VdbeMemStoreType(apArg[i]); } - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; p->inVtabMethod = 1; rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg); p->inVtabMethod = 0; @@ -5386,7 +5410,6 @@ case OP_VFilter: { /* jump */ if( rc==SQLITE_OK ){ res = pModule->xEof(pVtabCursor); } - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; if( res ){ pc = pOp->p2 - 1; @@ -5432,7 +5455,6 @@ case OP_VColumn: { sqlite3VdbeMemMove(&sContext.s, pDest); MemSetTypeFlag(&sContext.s, MEM_Null); - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = pVtab->zErrMsg; @@ -5450,9 +5472,6 @@ case OP_VColumn: { REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); - if( sqlite3SafetyOn(db) ){ - goto abort_due_to_misuse; - } if( sqlite3VdbeMemTooBig(pDest) ){ goto too_big; } @@ -5489,7 +5508,6 @@ case OP_VNext: { /* jump */ ** data is available) and the error code returned when xColumn or ** some other method is next invoked on the save virtual table cursor. */ - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; p->inVtabMethod = 1; rc = pModule->xNext(pCur->pVtabCursor); p->inVtabMethod = 0; @@ -5499,7 +5517,6 @@ case OP_VNext: { /* jump */ if( rc==SQLITE_OK ){ res = pModule->xEof(pCur->pVtabCursor); } - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; if( !res ){ /* If there is data, jump to P2 */ @@ -5525,12 +5542,10 @@ case OP_VRename: { assert( pVtab->pModule->xRename ); REGISTER_TRACE(pOp->p1, pName); assert( pName->flags & MEM_Str ); - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; rc = pVtab->pModule->xRename(pVtab, pName->z); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = pVtab->zErrMsg; pVtab->zErrMsg = 0; - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; break; } @@ -5581,12 +5596,10 @@ case OP_VUpdate: { apArg[i] = pX; pX++; } - if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse; rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = pVtab->zErrMsg; pVtab->zErrMsg = 0; - if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse; if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); db->lastRowid = rowid; @@ -5659,6 +5672,7 @@ case OP_Trace: { ** the same as a no-op. This opcodesnever appears in a real VM program. */ default: { /* This is really OP_Noop and OP_Explain */ + assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); break; } @@ -5710,6 +5724,9 @@ default: { /* This is really OP_Noop and OP_Explain */ vdbe_error_halt: assert( rc ); p->rc = rc; + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(rc, "statement aborts at %d: [%s] %s", + pc, p->zSql, p->zErrMsg); sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; rc = SQLITE_ERROR; @@ -5738,12 +5755,6 @@ no_mem: rc = SQLITE_NOMEM; goto vdbe_error_halt; - /* Jump to here for an SQLITE_MISUSE error. - */ -abort_due_to_misuse: - rc = SQLITE_MISUSE; - /* Fall thru into abort_due_to_error */ - /* Jump to here for any other kind of fatal error. The "rc" variable ** should hold the error number. */ diff --git a/src/vdbe.h b/src/vdbe.h index f67e903b..c234d51a 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -182,6 +182,7 @@ void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); void sqlite3VdbeUsesBtree(Vdbe*, int); VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); int sqlite3VdbeMakeLabel(Vdbe*); +void sqlite3VdbeRunOnlyOnce(Vdbe*); void sqlite3VdbeDelete(Vdbe*); void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); int sqlite3VdbeFinalize(Vdbe*); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 93550e9d..489e8da6 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -301,6 +301,7 @@ struct Vdbe { u8 explain; /* True if EXPLAIN present on SQL command */ u8 changeCntOn; /* True to update the change-counter */ u8 expired; /* True if the VM needs to be recompiled */ + u8 runOnlyOnce; /* Automatically expire on reset */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ u8 inVtabMethod; /* See comments above */ u8 usesStmtJournal; /* True if uses a statement journal */ @@ -362,7 +363,11 @@ void sqlite3VdbeMemMove(Mem*, Mem*); int sqlite3VdbeMemNulTerminate(Mem*); int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); void sqlite3VdbeMemSetInt64(Mem*, i64); -void sqlite3VdbeMemSetDouble(Mem*, double); +#ifdef SQLITE_OMIT_FLOATING_POINT +# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 +#else + void sqlite3VdbeMemSetDouble(Mem*, double); +#endif void sqlite3VdbeMemSetNull(Mem*); void sqlite3VdbeMemSetZeroBlob(Mem*,int); void sqlite3VdbeMemSetRowSet(Mem*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index a8667080..125f325d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -31,6 +31,28 @@ int sqlite3_expired(sqlite3_stmt *pStmt){ } #endif +/* +** Check on a Vdbe to make sure it has not been finalized. Log +** an error and return true if it has been finalized (or is otherwise +** invalid). Return false if it is ok. +*/ +static int vdbeSafety(Vdbe *p){ + if( p->db==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with finalized prepared statement"); + return 1; + }else{ + return 0; + } +} +static int vdbeSafetyNotNull(Vdbe *p){ + if( p==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with NULL prepared statement"); + return 1; + }else{ + return vdbeSafety(p); + } +} + /* ** The following routine destroys a virtual machine that is created by ** the sqlite3_compile() routine. The integer returned is an SQLITE_ @@ -48,7 +70,11 @@ int sqlite3_finalize(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; sqlite3 *db = v->db; #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = v->db->mutex; + sqlite3_mutex *mutex; +#endif + if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; +#if SQLITE_THREADSAFE + mutex = v->db->mutex; #endif sqlite3_mutex_enter(mutex); rc = sqlite3VdbeFinalize(v); @@ -295,26 +321,23 @@ static int sqlite3Step(Vdbe *p){ assert(p); if( p->magic!=VDBE_MAGIC_RUN ){ - return SQLITE_MISUSE; + sqlite3_log(SQLITE_MISUSE, + "attempt to step a halted statement: [%s]", p->zSql); + return SQLITE_MISUSE_BKPT; } - /* Assert that malloc() has not failed */ + /* Check that malloc() has not failed. If it has, return early. */ db = p->db; if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; return SQLITE_NOMEM; } if( p->pc<=0 && p->expired ){ - if( ALWAYS(p->rc==SQLITE_OK || p->rc==SQLITE_SCHEMA) ){ - p->rc = SQLITE_SCHEMA; - } + p->rc = SQLITE_SCHEMA; rc = SQLITE_ERROR; goto end_of_step; } - if( sqlite3SafetyOn(db) ){ - p->rc = SQLITE_MISUSE; - return SQLITE_MISUSE; - } if( p->pc<0 ){ /* If there are no other statements currently running, then ** reset the interrupt flag. This prevents a call to sqlite3_interrupt @@ -347,10 +370,6 @@ static int sqlite3Step(Vdbe *p){ rc = sqlite3VdbeExec(p); } - if( sqlite3SafetyOff(db) ){ - rc = SQLITE_MISUSE; - } - #ifndef SQLITE_OMIT_TRACE /* Invoke the profile callback if there is one */ @@ -397,39 +416,44 @@ end_of_step: ** call sqlite3Reprepare() and try again. */ int sqlite3_step(sqlite3_stmt *pStmt){ - int rc = SQLITE_MISUSE; - if( pStmt ){ - int cnt = 0; - Vdbe *v = (Vdbe*)pStmt; - sqlite3 *db = v->db; - sqlite3_mutex_enter(db->mutex); - while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < 5 - && (rc = sqlite3Reprepare(v))==SQLITE_OK ){ - sqlite3_reset(pStmt); - v->expired = 0; - } - if( rc==SQLITE_SCHEMA && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ - /* This case occurs after failing to recompile an sql statement. - ** The error message from the SQL compiler has already been loaded - ** into the database handle. This block copies the error message - ** from the database handle into the statement and sets the statement - ** program counter to 0 to ensure that when the statement is - ** finalized or reset the parser error message is available via - ** sqlite3_errmsg() and sqlite3_errcode(). - */ - const char *zErr = (const char *)sqlite3_value_text(db->pErr); - sqlite3DbFree(db, v->zErrMsg); - if( !db->mallocFailed ){ - v->zErrMsg = sqlite3DbStrDup(db, zErr); - } else { - v->zErrMsg = 0; - v->rc = SQLITE_NOMEM; - } - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); + int rc = SQLITE_OK; /* Result from sqlite3Step() */ + int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ + Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ + int cnt = 0; /* Counter to prevent infinite loop of reprepares */ + sqlite3 *db; /* The database connection */ + + if( vdbeSafetyNotNull(v) ){ + return SQLITE_MISUSE_BKPT; } + db = v->db; + sqlite3_mutex_enter(db->mutex); + while( (rc = sqlite3Step(v))==SQLITE_SCHEMA + && cnt++ < 5 + && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ + sqlite3_reset(pStmt); + v->expired = 0; + } + if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ + /* This case occurs after failing to recompile an sql statement. + ** The error message from the SQL compiler has already been loaded + ** into the database handle. This block copies the error message + ** from the database handle into the statement and sets the statement + ** program counter to 0 to ensure that when the statement is + ** finalized or reset the parser error message is available via + ** sqlite3_errmsg() and sqlite3_errcode(). + */ + const char *zErr = (const char *)sqlite3_value_text(db->pErr); + sqlite3DbFree(db, v->zErrMsg); + if( !db->mallocFailed ){ + v->zErrMsg = sqlite3DbStrDup(db, zErr); + v->rc = rc2; + } else { + v->zErrMsg = 0; + v->rc = rc = SQLITE_NOMEM; + } + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); return rc; } @@ -483,8 +507,9 @@ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ assert( p && p->pFunc && p->pFunc->xStep ); assert( sqlite3_mutex_held(p->s.db->mutex) ); pMem = p->pMem; + testcase( nByte<0 ); if( (pMem->flags & MEM_Agg)==0 ){ - if( nByte==0 ){ + if( nByte<=0 ){ sqlite3VdbeMemReleaseExternal(pMem); pMem->flags = MEM_Null; pMem->z = 0; @@ -898,12 +923,16 @@ const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ */ static int vdbeUnbind(Vdbe *p, int i){ Mem *pVar; - if( p==0 ) return SQLITE_MISUSE; + if( vdbeSafetyNotNull(p) ){ + return SQLITE_MISUSE_BKPT; + } sqlite3_mutex_enter(p->db->mutex); if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ sqlite3Error(p->db, SQLITE_MISUSE, 0); sqlite3_mutex_leave(p->db->mutex); - return SQLITE_MISUSE; + sqlite3_log(SQLITE_MISUSE, + "bind on a busy prepared statement: [%s]", p->zSql); + return SQLITE_MISUSE_BKPT; } if( i<1 || i>p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE, 0); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 06350860..40a46ff6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -66,7 +66,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ */ const char *sqlite3_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return (p->isPrepareV2 ? p->zSql : 0); + return (p && p->isPrepareV2) ? p->zSql : 0; } /* @@ -255,6 +255,13 @@ void sqlite3VdbeResolveLabel(Vdbe *p, int x){ } } +/* +** Mark the VDBE as one that can only be run one time. +*/ +void sqlite3VdbeRunOnlyOnce(Vdbe *p){ + p->runOnlyOnce = 1; +} + #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* @@ -1041,22 +1048,24 @@ void sqlite3VdbeFrameDelete(VdbeFrame *p){ ** p->explain==2, only OP_Explain instructions are listed and these ** are shown in a different format. p->explain==2 is used to implement ** EXPLAIN QUERY PLAN. +** +** When p->explain==1, first the main program is listed, then each of +** the trigger subprograms are listed one by one. */ int sqlite3VdbeList( Vdbe *p /* The VDBE */ ){ - int nRow; /* Total number of rows to return */ + int nRow; /* Stop when row count reaches this */ int nSub = 0; /* Number of sub-vdbes seen so far */ SubProgram **apSub = 0; /* Array of sub-vdbes */ - Mem *pSub = 0; - sqlite3 *db = p->db; - int i; - int rc = SQLITE_OK; - Mem *pMem = p->pResultSet = &p->aMem[1]; + Mem *pSub = 0; /* Memory cell hold array of subprogs */ + sqlite3 *db = p->db; /* The database connection */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + Mem *pMem = p->pResultSet = &p->aMem[1]; /* First Mem of result set */ assert( p->explain ); assert( p->magic==VDBE_MAGIC_RUN ); - assert( db->magic==SQLITE_MAGIC_BUSY ); assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); /* Even though this opcode does not use dynamic strings for @@ -1072,12 +1081,24 @@ int sqlite3VdbeList( return SQLITE_ERROR; } - /* Figure out total number of rows that will be returned by this - ** EXPLAIN program. */ + /* When the number of output rows reaches nRow, that means the + ** listing has finished and sqlite3_step() should return SQLITE_DONE. + ** nRow is the sum of the number of rows in the main program, plus + ** the sum of the number of rows in all trigger subprograms encountered + ** so far. The nRow value will increase as new trigger subprograms are + ** encountered, but p->pc will eventually catch up to nRow. + */ nRow = p->nOp; if( p->explain==1 ){ + /* The first 8 memory cells are used for the result set. So we will + ** commandeer the 9th cell to use as storage for an array of pointers + ** to trigger subprograms. The VDBE is guaranteed to have at least 9 + ** cells. */ + assert( p->nMem>9 ); pSub = &p->aMem[9]; if( pSub->flags&MEM_Blob ){ + /* On the first call to sqlite3_step(), pSub will hold a NULL. It is + ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ nSub = pSub->n/sizeof(Vdbe*); apSub = (SubProgram **)pSub->z; } @@ -1100,8 +1121,12 @@ int sqlite3VdbeList( char *z; Op *pOp; if( inOp ){ + /* The output line number is small enough that we are still in the + ** main program. */ pOp = &p->aOp[i]; }else{ + /* We are currently listing subprograms. Figure out which one and + ** pick up the appropriate opcode. */ int j; i -= p->nOp; for(j=0; i>=apSub[j]->nOp; j++){ @@ -1123,6 +1148,11 @@ int sqlite3VdbeList( pMem->enc = SQLITE_UTF8; pMem++; + /* When an OP_Program opcode is encounter (the only opcode that has + ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms + ** kept in p->aMem[9].z to hold the new program - assuming this subprogram + ** has not already been seen. + */ if( pOp->p4type==P4_SUBPROGRAM ){ int nByte = (nSub+1)*sizeof(SubProgram*); int j; @@ -1254,38 +1284,43 @@ void sqlite3VdbeIOTraceSql(Vdbe *p){ #endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ /* -** Allocate space from a fixed size buffer. Make *pp point to the -** allocated space. (Note: pp is a char* rather than a void** to -** work around the pointer aliasing rules of C.) *pp should initially -** be zero. If *pp is not zero, that means that the space has already -** been allocated and this routine is a noop. +** Allocate space from a fixed size buffer and return a pointer to +** that space. If insufficient space is available, return NULL. +** +** The pBuf parameter is the initial value of a pointer which will +** receive the new memory. pBuf is normally NULL. If pBuf is not +** NULL, it means that memory space has already been allocated and that +** this routine should not allocate any new memory. When pBuf is not +** NULL simply return pBuf. Only allocate new memory space when pBuf +** is NULL. ** ** nByte is the number of bytes of space needed. ** -** *ppFrom point to available space and pEnd points to the end of the -** available space. +** *ppFrom points to available space and pEnd points to the end of the +** available space. When space is allocated, *ppFrom is advanced past +** the end of the allocated space. ** ** *pnByte is a counter of the number of bytes of space that have failed ** to allocate. If there is insufficient space in *ppFrom to satisfy the ** request, then increment *pnByte by the amount of the request. */ -static void allocSpace( - char *pp, /* IN/OUT: Set *pp to point to allocated buffer */ +static void *allocSpace( + void *pBuf, /* Where return pointer will be stored */ int nByte, /* Number of bytes to allocate */ u8 **ppFrom, /* IN/OUT: Allocate from *ppFrom */ u8 *pEnd, /* Pointer to 1 byte past the end of *ppFrom buffer */ int *pnByte /* If allocation cannot be made, increment *pnByte */ ){ assert( EIGHT_BYTE_ALIGNMENT(*ppFrom) ); - if( (*(void**)pp)==0 ){ - nByte = ROUND8(nByte); - if( &(*ppFrom)[nByte] <= pEnd ){ - *(void**)pp = (void *)*ppFrom; - *ppFrom += nByte; - }else{ - *pnByte += nByte; - } + if( pBuf ) return pBuf; + nByte = ROUND8(nByte); + if( &(*ppFrom)[nByte] <= pEnd ){ + pBuf = (void*)*ppFrom; + *ppFrom += nByte; + }else{ + *pnByte += nByte; } + return pBuf; } /* @@ -1344,9 +1379,10 @@ void sqlite3VdbeMakeReady( ** being called from sqlite3_reset() to reset the virtual machine. */ if( nVar>=0 && ALWAYS(db->mallocFailed==0) ){ - u8 *zCsr = (u8 *)&p->aOp[p->nOp]; - u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; - int nByte; + u8 *zCsr = (u8 *)&p->aOp[p->nOp]; /* Memory avaliable for alloation */ + u8 *zEnd = (u8 *)&p->aOp[p->nOpAlloc]; /* First byte past available mem */ + int nByte; /* How much extra memory needed */ + resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)usesStmtJournal; if( isExplain && nMem<10 ){ @@ -1356,15 +1392,24 @@ void sqlite3VdbeMakeReady( zCsr += (zCsr - (u8*)0)&7; assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + /* Memory for registers, parameters, cursor, etc, is allocated in two + ** passes. On the first pass, we try to reuse unused space at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the rest using a fresh allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover space at the end of the opcode array can significantly + ** reduce the amount of memory held by a prepared statement. + */ do { nByte = 0; - allocSpace((char*)&p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); - allocSpace((char*)&p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); - allocSpace((char*)&p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); - allocSpace((char*)&p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); - allocSpace((char*)&p->apCsr, - nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte - ); + p->aMem = allocSpace(p->aMem, nMem*sizeof(Mem), &zCsr, zEnd, &nByte); + p->aVar = allocSpace(p->aVar, nVar*sizeof(Mem), &zCsr, zEnd, &nByte); + p->apArg = allocSpace(p->apArg, nArg*sizeof(Mem*), &zCsr, zEnd, &nByte); + p->azVar = allocSpace(p->azVar, nVar*sizeof(char*), &zCsr, zEnd, &nByte); + p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), + &zCsr, zEnd, &nByte); if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1435,9 +1480,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; const sqlite3_module *pModule = pCx->pModule; p->inVtabMethod = 1; - (void)sqlite3SafetyOff(p->db); pModule->xClose(pVtabCursor); - (void)sqlite3SafetyOn(p->db); p->inVtabMethod = 0; } #endif @@ -1618,9 +1661,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* If there are any write-transactions at all, invoke the commit hook */ if( needXcommit && db->xCommitCallback ){ - (void)sqlite3SafetyOff(db); rc = db->xCommitCallback(db->pCommitArg); - (void)sqlite3SafetyOn(db); if( rc ){ return SQLITE_CONSTRAINT; } @@ -1706,10 +1747,11 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ */ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( i==1 ) continue; /* Ignore the TEMP database */ if( sqlite3BtreeIsInTrans(pBt) ){ char const *zFile = sqlite3BtreeGetJournalname(pBt); - if( zFile[0]==0 ) continue; /* Ignore :memory: databases */ + if( zFile==0 || zFile[0]==0 ){ + continue; /* Ignore TEMP and :memory: databases */ + } if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ needSync = 1; } @@ -2085,12 +2127,17 @@ int sqlite3VdbeHalt(Vdbe *p){ /* If eStatementOp is non-zero, then a statement transaction needs to ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to ** do so. If this operation returns an error, and the current statement - ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then set the error - ** code to the new value. + ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the + ** current statement error code. + ** + ** Note that sqlite3VdbeCloseStatement() can only fail if eStatementOp + ** is SAVEPOINT_ROLLBACK. But if p->rc==SQLITE_OK then eStatementOp + ** must be SAVEPOINT_RELEASE. Hence the NEVER(p->rc==SQLITE_OK) in + ** the following code. */ if( eStatementOp ){ rc = sqlite3VdbeCloseStatement(p, eStatementOp); - if( rc && (p->rc==SQLITE_OK || p->rc==SQLITE_CONSTRAINT) ){ + if( rc && (NEVER(p->rc==SQLITE_OK) || p->rc==SQLITE_CONSTRAINT) ){ p->rc = rc; sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; @@ -2173,9 +2220,7 @@ int sqlite3VdbeReset(Vdbe *p){ ** error, then it might not have been halted properly. So halt ** it now. */ - (void)sqlite3SafetyOn(db); sqlite3VdbeHalt(p); - (void)sqlite3SafetyOff(db); /* If the VDBE has be run even partially, then transfer the error code ** and error message from the VDBE into the main database structure. But @@ -2195,6 +2240,7 @@ int sqlite3VdbeReset(Vdbe *p){ }else{ sqlite3Error(db, SQLITE_OK, 0); } + if( p->runOnlyOnce ) p->expired = 1; }else if( p->rc && p->expired ){ /* The expired flag was set on the VDBE before the first call ** to sqlite3_step(). For consistency (since sqlite3_step() was @@ -2296,6 +2342,7 @@ void sqlite3VdbeDelete(Vdbe *p){ sqlite3DbFree(db, p->zSql); p->magic = VDBE_MAGIC_DEAD; sqlite3DbFree(db, p->pFree); + p->db = 0; sqlite3DbFree(db, p); } @@ -2976,7 +3023,7 @@ int sqlite3VdbeIdxKeyCompare( ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ if( nCellKey<=0 || nCellKey>0x7fffffff ){ *res = 0; - return SQLITE_CORRUPT; + return SQLITE_CORRUPT_BKPT; } memset(&m, 0, sizeof(m)); rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (int)nCellKey, 1, &m); diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 6a6c6a71..829b6de6 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -95,13 +95,6 @@ int sqlite3_blob_open( memset(pParse, 0, sizeof(Parse)); pParse->db = db; - if( sqlite3SafetyOn(db) ){ - sqlite3DbFree(db, zErr); - sqlite3StackFree(db, pParse); - sqlite3_mutex_leave(db->mutex); - return SQLITE_MISUSE; - } - sqlite3BtreeEnterAll(db); pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); if( pTab && IsVirtual(pTab) ){ @@ -121,7 +114,6 @@ int sqlite3_blob_open( pParse->zErrMsg = 0; } rc = SQLITE_ERROR; - (void)sqlite3SafetyOff(db); sqlite3BtreeLeaveAll(db); goto blob_open_out; } @@ -136,7 +128,6 @@ int sqlite3_blob_open( sqlite3DbFree(db, zErr); zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); rc = SQLITE_ERROR; - (void)sqlite3SafetyOff(db); sqlite3BtreeLeaveAll(db); goto blob_open_out; } @@ -177,7 +168,6 @@ int sqlite3_blob_open( sqlite3DbFree(db, zErr); zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); rc = SQLITE_ERROR; - (void)sqlite3SafetyOff(db); sqlite3BtreeLeaveAll(db); goto blob_open_out; } @@ -227,8 +217,7 @@ int sqlite3_blob_open( } sqlite3BtreeLeaveAll(db); - rc = sqlite3SafetyOff(db); - if( NEVER(rc!=SQLITE_OK) || db->mallocFailed ){ + if( db->mallocFailed ){ goto blob_open_out; } @@ -329,7 +318,7 @@ static int blobReadWrite( Vdbe *v; sqlite3 *db; - if( p==0 ) return SQLITE_MISUSE; + if( p==0 ) return SQLITE_MISUSE_BKPT; db = p->db; sqlite3_mutex_enter(db->mutex); v = (Vdbe*)p->pStmt; diff --git a/src/vdbemem.c b/src/vdbemem.c index efcf948f..8217c4d2 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -315,6 +315,10 @@ void sqlite3VdbeMemRelease(Mem *p){ ** before attempting the conversion. */ static i64 doubleToInt64(double r){ +#ifdef SQLITE_OMIT_FLOATING_POINT + /* When floating-point is omitted, double and int64 are the same thing */ + return r; +#else /* ** Many compilers we encounter do not define constants for the ** minimum and maximum 64-bit integers, or they define them @@ -336,6 +340,7 @@ static i64 doubleToInt64(double r){ }else{ return (i64)r; } +#endif } /* @@ -463,21 +468,26 @@ int sqlite3VdbeMemRealify(Mem *pMem){ /* ** Convert pMem so that it has types MEM_Real or MEM_Int or both. ** Invalidate any prior representations. +** +** Every effort is made to force the conversion, even if the input +** is a string that does not look completely like a number. Convert +** as much of the string as we can and ignore the rest. */ int sqlite3VdbeMemNumerify(Mem *pMem){ - double r1, r2; - i64 i; + int rc; assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ); assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - r1 = sqlite3VdbeRealValue(pMem); - i = doubleToInt64(r1); - r2 = (double)i; - if( r1==r2 ){ - sqlite3VdbeMemIntegerify(pMem); + rc = sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8); + if( rc ) return rc; + rc = sqlite3VdbeMemNulTerminate(pMem); + if( rc ) return rc; + if( sqlite3Atoi64(pMem->z, &pMem->u.i) ){ + MemSetTypeFlag(pMem, MEM_Int); }else{ - pMem->r = r1; + pMem->r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); + sqlite3VdbeIntegerAffinity(pMem); } return SQLITE_OK; } @@ -529,6 +539,7 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ pMem->type = SQLITE_INTEGER; } +#ifndef SQLITE_OMIT_FLOATING_POINT /* ** Delete any previous value and set the value stored in *pMem to val, ** manifest type REAL. @@ -543,6 +554,7 @@ void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ pMem->type = SQLITE_FLOAT; } } +#endif /* ** Delete any previous value and set the value of pMem to be an @@ -597,7 +609,7 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ sqlite3VdbeMemReleaseExternal(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->xDel = 0; - if( (pFrom->flags&MEM_Dyn)!=0 || pFrom->z==pFrom->zMalloc ){ + if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); assert( srcType==MEM_Ephem || srcType==MEM_Static ); pTo->flags |= srcType; diff --git a/src/vtab.c b/src/vtab.c index f5269e51..cbb75235 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -123,16 +123,7 @@ void sqlite3VtabUnlock(VTable *pVTab){ if( pVTab->nRef==0 ){ sqlite3_vtab *p = pVTab->pVtab; if( p ){ -#ifdef SQLITE_DEBUG - if( pVTab->db->magic==SQLITE_MAGIC_BUSY ){ - (void)sqlite3SafetyOff(db); - p->pModule->xDisconnect(p); - (void)sqlite3SafetyOn(db); - } else -#endif - { - p->pModule->xDisconnect(p); - } + p->pModule->xDisconnect(p); } sqlite3DbFree(db, pVTab); } @@ -468,9 +459,7 @@ static int vtabCallConstructor( db->pVTab = pTab; /* Invoke the virtual table constructor */ - (void)sqlite3SafetyOff(db); rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); - (void)sqlite3SafetyOn(db); if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; if( SQLITE_OK!=rc ){ @@ -658,7 +647,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ if( !pTab ){ sqlite3Error(db, SQLITE_MISUSE, 0); sqlite3_mutex_leave(db->mutex); - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } assert( (pTab->tabFlags & TF_Virtual)!=0 ); @@ -669,11 +658,11 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ pParse->declareVtab = 1; pParse->db = db; - if( - SQLITE_OK == sqlite3RunParser(pParse, zCreateTable, &zErr) && - pParse->pNewTable && - !pParse->pNewTable->pSelect && - (pParse->pNewTable->tabFlags & TF_Virtual)==0 + if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) + && pParse->pNewTable + && !db->mallocFailed + && !pParse->pNewTable->pSelect + && (pParse->pNewTable->tabFlags & TF_Virtual)==0 ){ if( !pTab->aCol ){ pTab->aCol = pParse->pNewTable->aCol; @@ -682,7 +671,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ pParse->pNewTable->aCol = 0; } db->pVTab = 0; - } else { + }else{ sqlite3Error(db, SQLITE_ERROR, zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; @@ -717,10 +706,8 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ VTable *p = vtabDisconnectAll(db, pTab); - rc = sqlite3SafetyOff(db); assert( rc==SQLITE_OK ); rc = p->pMod->pModule->xDestroy(p->pVtab); - (void)sqlite3SafetyOn(db); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ @@ -772,10 +759,8 @@ static void callFinaliser(sqlite3 *db, int offset){ int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ int i; int rc = SQLITE_OK; - int rcsafety; VTable **aVTrans = db->aVTrans; - rc = sqlite3SafetyOff(db); db->aVTrans = 0; for(i=0; rc==SQLITE_OK && inVTrans; i++){ int (*x)(sqlite3_vtab *); @@ -788,11 +773,6 @@ int sqlite3VtabSync(sqlite3 *db, char **pzErrmsg){ } } db->aVTrans = aVTrans; - rcsafety = sqlite3SafetyOn(db); - - if( rc==SQLITE_OK ){ - rc = rcsafety; - } return rc; } diff --git a/src/where.c b/src/where.c index 067ec407..5f81ce07 100644 --- a/src/where.c +++ b/src/where.c @@ -653,7 +653,7 @@ static int isLikeOrGlob( } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ pColl = sqlite3ExprCollSeq(pParse, pLeft); - assert( pColl!=0 ); /* Every non-IPK column has a collating sequence */ + if( pColl==0 ) return 0; /* Happens when LHS has an undefined collation */ if( (pColl->type!=SQLITE_COLL_BINARY || *pnoCase) && (pColl->type!=SQLITE_COLL_NOCASE || !*pnoCase) ){ /* IMP: R-09003-32046 For the GLOB operator, the column must use the @@ -1096,7 +1096,7 @@ static void exprAnalyze( Expr *pExpr; /* The expression to be analyzed */ Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ Bitmask prereqAll; /* Prerequesites of pExpr */ - Bitmask extraRight = 0; /* */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ int noCase = 0; /* LIKE/GLOB distinguishes case */ @@ -1168,7 +1168,8 @@ static void exprAnalyze( pLeft = pDup->pLeft; pNew->leftCursor = pLeft->iTable; pNew->u.leftColumn = pLeft->iColumn; - pNew->prereqRight = prereqLeft; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; pNew->eOperator = operatorMask(pDup->op); } @@ -1758,12 +1759,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ int i; int rc; - (void)sqlite3SafetyOff(pParse->db); WHERETRACE(("xBestIndex for %s\n", pTab->zName)); TRACE_IDX_INPUTS(p); rc = pVtab->pModule->xBestIndex(pVtab, p); TRACE_IDX_OUTPUTS(p); - (void)sqlite3SafetyOn(pParse->db); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ){ @@ -3160,7 +3159,7 @@ static Bitmask codeOneLoopStart( nConstraint = nEq; if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq); + sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); sqlite3ExprCode(pParse, pRight, regBase+nEq); sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt); if( zAff ){ @@ -3265,13 +3264,14 @@ static Bitmask codeOneLoopStart( */ WhereClause *pOrWc; /* The OR-clause broken out into subterms */ WhereTerm *pFinal; /* Final subterm within the OR-clause. */ - SrcList oneTab; /* Shortened table list */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ int regRowset = 0; /* Register for RowSet object */ int regRowid = 0; /* Register holding rowid */ int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ int ii; pTerm = pLevel->plan.u.pTerm; @@ -3280,11 +3280,30 @@ static Bitmask codeOneLoopStart( assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); pOrWc = &pTerm->u.pOrInfo->wc; pFinal = &pOrWc->a[pOrWc->nTerm-1]; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; - /* Set up a SrcList containing just the table being scanned by this loop. */ - oneTab.nSrc = 1; - oneTab.nAlloc = 1; - oneTab.a[0] = *pTabItem; + /* Set up a new SrcList ni pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + struct SrcList_item *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(pParse->db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (i16)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; + } /* Initialize the rowset register to contain NULL. An SQL NULL is ** equivalent to an empty rowset. @@ -3309,32 +3328,38 @@ static Bitmask codeOneLoopStart( if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, &oneTab, pOrTerm->pExpr, 0, - WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, + WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, - regRowid, 0); + regRowid); sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, sqlite3VdbeCurrentAddr(v)+2, r, iSet); } sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + /* Finish the loop through table entries that match term pOrTerm. */ sqlite3WhereEnd(pSubWInfo); } } } sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); - /* sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); */ sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); - pLevel->op = OP_Return; - pLevel->p1 = regReturn; - disableTerm(pLevel, pTerm); + if( pWInfo->nLevel>1 ) sqlite3StackFree(pParse->db, pOrTab); + if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -3362,7 +3387,12 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ) continue; + if( (pTerm->prereqAll & notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } pE = pTerm->pExpr; assert( pE!=0 ); if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ @@ -3385,7 +3415,10 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ) continue; + if( (pTerm->prereqAll & notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; + } assert( pTerm->pExpr ); sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); pTerm->wtFlags |= TERM_CODED; @@ -3528,6 +3561,7 @@ WhereInfo *sqlite3WhereBegin( ){ int i; /* Loop counter */ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ + int nTabList; /* Number of elements in pTabList */ WhereInfo *pWInfo; /* Will become the return value of this function */ Vdbe *v = pParse->pVdbe; /* The virtual database engine */ Bitmask notReady; /* Cursors that are not yet positioned */ @@ -3547,6 +3581,13 @@ WhereInfo *sqlite3WhereBegin( return 0; } + /* This function normally generates a nested loop for all tables in + ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should + ** only generate code for the first table in pTabList and assume that + ** any cursors associated with subsequent tables are uninitialized. + */ + nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; + /* Allocate and initialize the WhereInfo structure that will become the ** return value. A single allocation is used to store the WhereInfo ** struct, the contents of WhereInfo.a[], the WhereClause structure @@ -3555,7 +3596,7 @@ WhereInfo *sqlite3WhereBegin( ** some architectures. Hence the ROUND8() below. */ db = pParse->db; - nByteWInfo = ROUND8(sizeof(WhereInfo)+(pTabList->nSrc-1)*sizeof(WhereLevel)); + nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereClause) + @@ -3564,7 +3605,7 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ){ goto whereBeginError; } - pWInfo->nLevel = pTabList->nSrc; + pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->iBreak = sqlite3VdbeMakeLabel(v); @@ -3583,7 +3624,7 @@ WhereInfo *sqlite3WhereBegin( /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. */ - if( pWhere && (pTabList->nSrc==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ + if( pWhere && (nTabList==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ sqlite3ExprIfFalse(pParse, pWhere, pWInfo->iBreak, SQLITE_JUMPIFNULL); pWhere = 0; } @@ -3603,6 +3644,11 @@ WhereInfo *sqlite3WhereBegin( ** to virtual table cursors are set. This is used to selectively disable ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful ** with virtual tables. + ** + ** Note that bitmasks are created for all pTabList->nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_ONETABLE_ONLY flag is set. */ assert( pWC->vmask==0 && pMaskSet->n==0 ); for(i=0; inSrc; i++){ @@ -3654,7 +3700,7 @@ WhereInfo *sqlite3WhereBegin( pLevel = pWInfo->a; andFlags = ~0; WHERETRACE(("*** Optimizer Start ***\n")); - for(i=iFrom=0, pLevel=pWInfo->a; inSrc; i++, pLevel++){ + for(i=iFrom=0, pLevel=pWInfo->a; i=0 && bestJ<0; isOptimal--){ Bitmask mask = (isOptimal ? 0 : notReady); - assert( (pTabList->nSrc-iFrom)>1 || isOptimal ); - for(j=iFrom, pTabItem=&pTabList->a[j]; jnSrc; j++, pTabItem++){ + assert( (nTabList-iFrom)>1 || isOptimal ); + for(j=iFrom, pTabItem=&pTabList->a[j]; ja; inSrc; i++, pLevel++){ + for(i=0, pLevel=pWInfo->a; inSrc; i++){ + for(i=0; iiContinue = pWInfo->a[i].addrCont; } @@ -3888,7 +3934,7 @@ WhereInfo *sqlite3WhereBegin( ** the index is listed as "{}". If the primary key is used the ** index name is '*'. */ - for(i=0; inSrc; i++){ + for(i=0; ia[i]; @@ -3956,7 +4002,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ /* Generate loop termination code. */ sqlite3ExprCacheClear(pParse); - for(i=pTabList->nSrc-1; i>=0; i--){ + for(i=pWInfo->nLevel-1; i>=0; i--){ pLevel = &pWInfo->a[i]; sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ @@ -4002,7 +4048,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ /* Close all of the cursors that were opened by sqlite3WhereBegin. */ - for(i=0, pLevel=pWInfo->a; inSrc; i++, pLevel++){ + assert( pWInfo->nLevel==1 || pWInfo->nLevel==pTabList->nSrc ); + for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; Table *pTab = pTabItem->pTab; assert( pTab!=0 ); diff --git a/test/analyze3.test b/test/analyze3.test index f623f987..438ecd7b 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -481,10 +481,10 @@ do_test analyze3-4.1.2 { sqlite3_bind_text $S 2 "abc" 3 execsql { DROP TABLE t1 } sqlite3_step $S -} {SQLITE_SCHEMA} +} {SQLITE_ERROR} do_test analyze3-4.1.3 { sqlite3_finalize $S -} {SQLITE_SCHEMA} +} {SQLITE_ERROR} # Check an authorization error. # @@ -511,10 +511,10 @@ do_test analyze3-4.2.2 { sqlite3_reset $S sqlite3_bind_text $S 2 "abc" 3 sqlite3_step $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} do_test analyze3-4.2.4 { sqlite3_finalize $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} # Check the effect of an authorization error that occurs in a re-prepare # performed by sqlite3_step() is the same as one that occurs within @@ -526,10 +526,10 @@ do_test analyze3-4.3.1 { execsql { CREATE TABLE t2(d, e, f) } db auth auth sqlite3_step $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} do_test analyze3-4.3.2 { sqlite3_finalize $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} db auth {} #------------------------------------------------------------------------- diff --git a/test/attach.test b/test/attach.test index 56d6445f..61e1272e 100644 --- a/test/attach.test +++ b/test/attach.test @@ -814,5 +814,25 @@ do_test attach-9.3 { } } {1 2 3 4} - +# Ticket [abe728bbc311d81334dae9762f0db87c07a98f79]. +# Multi-database commit on an attached TEMP database. +# +do_test attach-10.1 { + execsql { + ATTACH '' AS noname; + ATTACH ':memory:' AS inmem; + BEGIN; + CREATE TABLE noname.noname(x); + CREATE TABLE inmem.inmem(y); + CREATE TABLE main.main(z); + COMMIT; + SELECT name FROM noname.sqlite_master; + SELECT name FROM inmem.sqlite_master; + } +} {noname inmem} +do_test attach-10.2 { + lrange [execsql { + PRAGMA database_list; + }] 9 end +} {4 noname {} 5 inmem {}} finish_test diff --git a/test/attachmalloc.test b/test/attachmalloc.test index 30841cea..cc506bee 100644 --- a/test/attachmalloc.test +++ b/test/attachmalloc.test @@ -60,4 +60,17 @@ do_malloc_test attachmalloc-2 -tclprep { ATTACH 'test2.db' AS db1; } +set enable_shared_cache [sqlite3_enable_shared_cache 1] +sqlite3 dbaux test2.db +dbaux eval {SELECT * FROM sqlite_master} +do_malloc_test attachmalloc-3 -sqlbody { + SELECT * FROM sqlite_master; + ATTACH 'test2.db' AS two; +} -cleanup { + db eval { DETACH two } +} +dbaux close +sqlite3_enable_shared_cache $enable_shared_cache + + finish_test diff --git a/test/auth2.test b/test/auth2.test index 98270ae7..f5dba148 100644 --- a/test/auth2.test +++ b/test/auth2.test @@ -102,6 +102,7 @@ SQLITE_READ sqlite_master name main {} SQLITE_READ sqlite_master rootpage main {} SQLITE_READ sqlite_master sql main {} SQLITE_READ sqlite_master tbl_name main {} +SQLITE_READ sqlite_master ROWID main {} } do_test auth2-2.2 { set ::authargs {} @@ -121,6 +122,7 @@ SQLITE_READ sqlite_master name main {} SQLITE_READ sqlite_master rootpage main {} SQLITE_READ sqlite_master sql main {} SQLITE_READ sqlite_master tbl_name main {} +SQLITE_READ sqlite_master ROWID main {} } do_test auth2-2.3 { set ::authargs {} diff --git a/test/backup.test b/test/backup.test index ae155959..ec30adb3 100644 --- a/test/backup.test +++ b/test/backup.test @@ -904,4 +904,63 @@ ifcapable memorymanage { db3 close } + +# Test that if the database is written to via the same database handle being +# used as the source by a backup operation: +# +# 10.1.*: If the db is in-memory, the backup is restarted. +# 10.2.*: If the db is a file, the backup is not restarted. +# +db close +file delete -force test.db test.db-journal +foreach {tn file rc} { + 1 test.db SQLITE_DONE + 2 :memory: SQLITE_OK +} { + do_test backup-10.$tn.1 { + sqlite3 db $file + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + BEGIN; + INSERT INTO t1 VALUES(NULL, randomblob(200)); + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + INSERT INTO t1 SELECT NULL, randomblob(200) FROM t1; + COMMIT; + SELECT count(*) FROM t1; + } + } {256} + + do_test backup-10.$tn.2 { + set pgs [execsql {pragma page_count}] + expr {$pgs > 50 && $pgs < 75} + } {1} + + do_test backup-10.$tn.3 { + file delete -force bak.db bak.db-journal + sqlite3 db2 bak.db + sqlite3_backup B db2 main db main + B step 50 + } {SQLITE_OK} + + do_test backup-10.$tn.4 { + execsql { UPDATE t1 SET b = randomblob(200) WHERE a IN (1, 250) } + } {} + + do_test backup-10.$tn.5 { + B step 50 + } $rc + + do_test backup-10.$tn.6 { + B finish + } {SQLITE_OK} + + db2 close +} + finish_test diff --git a/test/capi3c.test b/test/capi3c.test index 1bcc4373..55f4667e 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -1176,7 +1176,7 @@ do_test capi3c-19.4 { sqlite3_reset $STMT db eval {DROP TABLE t3} sqlite3_step $STMT -} SQLITE_SCHEMA +} SQLITE_ERROR do_test capi3c-19.4.1 { sqlite3_errmsg $DB } {no such table: t3} diff --git a/test/check.test b/test/check.test index 51ab1f31..d2867a09 100644 --- a/test/check.test +++ b/test/check.test @@ -239,43 +239,43 @@ do_test check-4.3 { SELECT * FROM t4 } } {4 3} -do_test check-4.3 { +do_test check-4.4 { execsql { UPDATE t4 SET x=12, y=2; SELECT * FROM t4 } } {12 2} -do_test check-4.4 { +do_test check-4.5 { execsql { UPDATE t4 SET x=12, y=-22; SELECT * FROM t4 } } {12 -22} -do_test check-4.5 { +do_test check-4.6 { catchsql { UPDATE t4 SET x=0, y=1; } } {1 {constraint failed}} -do_test check-4.6 { +do_test check-4.7 { execsql { SELECT * FROM t4; } } {12 -22} -do_test check-4.7 { +do_test check-4.8 { execsql { PRAGMA ignore_check_constraints=ON; UPDATE t4 SET x=0, y=1; SELECT * FROM t4; } } {0 1} -do_test check-4.8 { +do_test check-4.9 { catchsql { PRAGMA ignore_check_constraints=OFF; UPDATE t4 SET x=0, y=2; } } {1 {constraint failed}} ifcapable vacuum { - do_test check_4.9 { + do_test check_4.10 { catchsql { VACUUM } diff --git a/test/corrupt.test b/test/corrupt.test index be6236ea..fc84033f 100644 --- a/test/corrupt.test +++ b/test/corrupt.test @@ -1,4 +1,4 @@ -# 2004 August 30 +# 2004 August 30 {} # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -311,4 +311,41 @@ ifcapable oversize_cell_check { } {1 {database disk image is malformed}} } +db close +file delete -force test.db test.db-journal +do_test corrupt-8.1 { + sqlite3 db test.db + execsql { + PRAGMA page_size = 1024; + PRAGMA secure_delete = on; + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(5, randomblob(1900)); + } + + hexio_write test.db 2044 [hexio_render_int32 2] + hexio_write test.db 24 [hexio_render_int32 45] + + catchsql { INSERT OR REPLACE INTO t1 VALUES(5, randomblob(1900)) } +} {1 {database disk image is malformed}} + +db close +file delete -force test.db test.db-journal +do_test corrupt-8.2 { + sqlite3 db test.db + execsql { + PRAGMA page_size = 1024; + PRAGMA secure_delete = on; + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(5, randomblob(900)); + INSERT INTO t1 VALUES(6, randomblob(900)); + } + + hexio_write test.db 2047 FF + hexio_write test.db 24 [hexio_render_int32 45] + + catchsql { INSERT INTO t1 VALUES(4, randomblob(1900)) } +} {1 {database disk image is malformed}} + finish_test diff --git a/test/corrupt7.test b/test/corrupt7.test index 8f8d437a..39aa6201 100644 --- a/test/corrupt7.test +++ b/test/corrupt7.test @@ -89,7 +89,7 @@ Corruption detected in cell 15 on page 2}} sqlite3 db test.db db eval {PRAGMA integrity_check(1)} } {{*** in database main *** -Corruption detected in cell 15 on page 2}} +On tree page 2 cell 15: Rowid 0 out of order (previous was 15)}} } # The code path that was causing the buffer overrun that this test diff --git a/test/corruptE.test b/test/corruptE.test new file mode 100644 index 00000000..35fa545b --- /dev/null +++ b/test/corruptE.test @@ -0,0 +1,171 @@ +# 2010 February 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to make sure SQLite does not crash or +# segfault if it sees a corrupt database file. It specifcally +# focuses on rowid order corruption. +# +# $Id: corruptE.test,v 1.14 2009/07/11 06:55:34 danielk1977 Exp $ + +catch {file delete -force test.db test.db-journal test.bu} + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Construct a compact, dense database for testing. +# +do_test corruptE-1.1 { + execsql { + PRAGMA auto_vacuum = 0; + PRAGMA legacy_file_format=1; + BEGIN; + CREATE TABLE t1(x,y); + INSERT INTO t1 VALUES(1,1); + INSERT OR IGNORE INTO t1 SELECT x*2,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*3,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*5,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*7,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*11,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*13,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*17,y FROM t1; + INSERT OR IGNORE INTO t1 SELECT x*19,y FROM t1; + CREATE INDEX t1i1 ON t1(x); + CREATE TABLE t2 AS SELECT x,2 as y FROM t1 WHERE rowid%5!=0; + COMMIT; + } +} {} + +ifcapable {integrityck} { + integrity_check corruptE-1.2 +} + +# Copy file $from into $to +# +proc copy_file {from to} { + file copy -force $from $to +} + +# Setup for the tests. Make a backup copy of the good database in test.bu. +# +db close +copy_file test.db test.bu +sqlite3 db test.db +set fsize [file size test.db] + + +do_test corruptE-2.1 { + db close + copy_file test.bu test.db + + # insert corrupt byte(s) + hexio_write test.db 2041 [format %02x 0x2e] + + sqlite3 db test.db + + set res [ catchsql {PRAGMA integrity_check} ] + set ans [lindex $res 1] + + list [regexp {out of order.*previous was} $ans] \ + [regexp {out of order.*max larger than parent max} $ans] +} {1 1} + +do_test corruptE-2.2 { + db close + copy_file test.bu test.db + + # insert corrupt byte(s) + hexio_write test.db 2047 [format %02x 0x84] + + sqlite3 db test.db + + set res [ catchsql {PRAGMA integrity_check} ] + set ans [lindex $res 1] + + list [regexp {out of order.*previous was} $ans] \ + [regexp {out of order.*min less than parent min} $ans] +} {1 1} + +do_test corruptE-2.3 { + db close + copy_file test.bu test.db + + # insert corrupt byte(s) + hexio_write test.db 7420 [format %02x 0xa8] + hexio_write test.db 10459 [format %02x 0x8d] + + sqlite3 db test.db + + set res [ catchsql {PRAGMA integrity_check} ] + set ans [lindex $res 1] + + list [regexp {out of order.*max larger than parent min} $ans] +} {1} + +do_test corruptE-2.4 { + db close + copy_file test.bu test.db + + # insert corrupt byte(s) + hexio_write test.db 10233 [format %02x 0xd0] + + sqlite3 db test.db + + set res [ catchsql {PRAGMA integrity_check} ] + set ans [lindex $res 1] + + list [regexp {out of order.*min less than parent max} $ans] +} {1} + + +set tests [list {10233 0xd0} \ + {941 0x42} \ + {1028 0x53} \ + {2041 0xd0} \ + {2042 0x1f} \ + {2047 0xaa} \ + {2263 0x29} \ + {2274 0x75} \ + {3267 0xf2} \ + {4104 0x2c} \ + {5113 0x36} \ + {10233 0x84} \ + {10234 0x74} \ + {10239 0x41} \ + {10453 0x11} \ + {11273 0x28} \ + {11455 0x11} \ + {11461 0xe6} \ + {12281 0x99} \ + {12296 0x9e} \ + {12297 0xd7} \ + {13303 0x53} ] + +set tc 1 +foreach test $tests { + do_test corruptE-3.$tc { + db close + copy_file test.bu test.db + + # insert corrupt byte(s) + hexio_write test.db [lindex $test 0] [format %02x [lindex $test 1]] + + sqlite3 db test.db + + set res [ catchsql {PRAGMA integrity_check} ] + set ans [lindex $res 1] + + list [regexp {out of order} $ans] + } {1} + incr tc 1 +} + +finish_test diff --git a/test/crash8.test b/test/crash8.test index 9e7f814e..b4e89f4c 100644 --- a/test/crash8.test +++ b/test/crash8.test @@ -340,4 +340,62 @@ ifcapable pragma { } {jkl} } +for {set i 1} {$i < 10} {incr i} { + catch { db close } + file delete -force test.db test.db-journal + sqlite3 db test.db + do_test crash8-5.$i.1 { + execsql { + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 rows */ + } + crashsql -file test.db -delay [expr ($::i%2) + 1] { + PRAGMA cache_size = 10; + BEGIN; + UPDATE t1 SET x = randomblob(900); + ROLLBACK; + INSERT INTO t1 VALUES(randomblob(900)); + } + execsql { PRAGMA integrity_check } + } {ok} + + catch { db close } + file delete -force test.db test.db-journal + sqlite3 db test.db + do_test crash8-5.$i.2 { + execsql { + PRAGMA cache_size = 10; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 rows */ + BEGIN; + UPDATE t1 SET x = randomblob(900); + } + file delete -force testX.db testX.db-journal + copy_file test.db testX.db + copy_file test.db-journal testX.db-journal + db close + + crashsql -file test.db -delay [expr ($::i%2) + 1] { + SELECT * FROM sqlite_master; + INSERT INTO t1 VALUES(randomblob(900)); + } + + sqlite3 db2 testX.db + execsql { PRAGMA integrity_check } db2 + } {ok} +} +catch {db2 close} + finish_test diff --git a/test/ctime.test b/test/ctime.test new file mode 100644 index 00000000..bb184b8b --- /dev/null +++ b/test/ctime.test @@ -0,0 +1,224 @@ +# 2009 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for the compile time diagnostic +# functions. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Test organization: +# +# ctime-1.*: Test pragma support. +# ctime-2.*: Test function support. +# + +ifcapable !pragma||!compileoption_diags { + finish_test + return +} + +##################### +# ctime-1.*: Test pragma support. + +do_test ctime-1.1.1 { + catchsql { + PRAGMA compile_options(); + } +} {1 {near ")": syntax error}} +do_test ctime-1.1.2 { + catchsql { + PRAGMA compile_options(NULL); + } +} {1 {near "NULL": syntax error}} +do_test ctime-1.1.3 { + catchsql { + PRAGMA compile_options *; + } +} {1 {near "*": syntax error}} + +do_test ctime-1.2.1 { + set ans [ catchsql { + PRAGMA compile_options; + } ] + list [ lindex $ans 0 ] +} {0} +# the results should be in sorted order already +do_test ctime-1.2.2 { + set ans [ catchsql { + PRAGMA compile_options; + } ] + list [ lindex $ans 0 ] [ expr { [lsort [lindex $ans 1]]==[lindex $ans 1] } ] +} {0 1} + +# SQLITE_THREADSAFE should pretty much always be defined +# one way or the other, and it must have a value of 0 or 1. +do_test ctime-1.4.1 { + catchsql { + SELECT sqlite_compileoption_used('SQLITE_THREADSAFE'); + } +} {0 1} +do_test ctime-1.4.2 { + catchsql { + SELECT sqlite_compileoption_used('THREADSAFE'); + } +} {0 1} +do_test ctime-1.4.3 { + catchsql { + SELECT sqlite_compileoption_used("THREADSAFE"); + } +} {0 1} + +do_test ctime-1.5 { + set ans1 [ catchsql { + SELECT sqlite_compileoption_used('THREADSAFE=0'); + } ] + set ans2 [ catchsql { + SELECT sqlite_compileoption_used('THREADSAFE=1'); + } ] + lsort [ list $ans1 $ans2 ] +} {{0 0} {0 1}} + +do_test ctime-1.6 { + execsql { + SELECT sqlite_compileoption_used('THREADSAFE='); + } +} {0} + +do_test ctime-1.7.1 { + execsql { + SELECT sqlite_compileoption_used('SQLITE_OMIT_COMPILEOPTION_DIAGS'); + } +} {0} +do_test ctime-1.7.2 { + execsql { + SELECT sqlite_compileoption_used('OMIT_COMPILEOPTION_DIAGS'); + } +} {0} + +##################### +# ctime-2.*: Test function support. + +do_test ctime-2.1.1 { + catchsql { + SELECT sqlite_compileoption_used(); + } +} {1 {wrong number of arguments to function sqlite_compileoption_used()}} +do_test ctime-2.1.2 { + catchsql { + SELECT sqlite_compileoption_used(NULL); + } +} {0 {{}}} +do_test ctime-2.1.3 { + catchsql { + SELECT sqlite_compileoption_used(""); + } +} {0 0} +do_test ctime-2.1.4 { + catchsql { + SELECT sqlite_compileoption_used(''); + } +} {0 0} +do_test ctime-2.1.5 { + catchsql { + SELECT sqlite_compileoption_used(foo); + } +} {1 {no such column: foo}} +do_test ctime-2.1.6 { + catchsql { + SELECT sqlite_compileoption_used('THREADSAFE', 0); + } +} {1 {wrong number of arguments to function sqlite_compileoption_used()}} +do_test ctime-2.1.7 { + catchsql { + SELECT sqlite_compileoption_used(0); + } +} {0 0} +do_test ctime-2.1.8 { + catchsql { + SELECT sqlite_compileoption_used('0'); + } +} {0 0} +do_test ctime-2.1.9 { + catchsql { + SELECT sqlite_compileoption_used(1.0); + } +} {0 0} + +do_test ctime-2.2.1 { + catchsql { + SELECT sqlite_compileoption_get(); + } +} {1 {wrong number of arguments to function sqlite_compileoption_get()}} +do_test ctime-2.2.2 { + catchsql { + SELECT sqlite_compileoption_get(0, 0); + } +} {1 {wrong number of arguments to function sqlite_compileoption_get()}} + +# This assumes there is at least 1 compile time option +# (see SQLITE_THREADSAFE above). +do_test ctime-2.3 { + catchsql { + SELECT sqlite_compileoption_used(sqlite_compileoption_get(0)); + } +} {0 1} + +# This assumes there is at least 1 compile time option +# (see SQLITE_THREADSAFE above). +do_test ctime-2.4 { + set ans [ catchsql { + SELECT sqlite_compileoption_get(0); + } ] + list [lindex $ans 0] +} {0} + +# Get the list of defines using the pragma, +# then try querying each one with the functions. +set ans [ catchsql { + PRAGMA compile_options; +} ] +set opts [ lindex $ans 1 ] +set tc 1 +foreach opt $opts { + do_test ctime-2.5.$tc { + set N [ expr {$tc-1} ] + set ans1 [ catchsql { + SELECT sqlite_compileoption_get($N); + } ] + set ans2 [ catchsql { + SELECT sqlite_compileoption_used($opt); + } ] + list [ lindex $ans1 0 ] [ expr { [lindex $ans1 1]==$opt } ] \ + [ expr { $ans2 } ] + } {0 1 {0 1}} + incr tc 1 +} +# test 1 past array bounds +do_test ctime-2.5.$tc { + set N [ expr {$tc-1} ] + set ans [ catchsql { + SELECT sqlite_compileoption_get($N); + } ] +} {0 {{}}} +incr tc 1 +# test 1 before array bounds (N=-1) +do_test ctime-2.5.$tc { + set N -1 + set ans [ catchsql { + SELECT sqlite_compileoption_get($N); + } ] +} {0 {{}}} + + +finish_test diff --git a/test/e_fkey.test b/test/e_fkey.test index 76095d15..cc87f359 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -31,10 +31,9 @@ proc eqp {sql {db db}} { uplevel execsql [list "EXPLAIN QUERY PLAN $sql"] $db } ########################################################################### #------------------------------------------------------------------------- -# /* EV: R-33710-56344 */ -# -# Test builds neither OMIT_FOREIGN_KEY or OMIT_TRIGGER defined have -# foreign key functionality. +# EVIDENCE-OF: R-33710-56344 In order to use foreign key constraints in +# SQLite, the library must be compiled with neither +# SQLITE_OMIT_FOREIGN_KEY or SQLITE_OMIT_TRIGGER defined. # ifcapable trigger&&foreignkey { do_test e_fkey-1 { @@ -51,16 +50,26 @@ ifcapable trigger&&foreignkey { } #------------------------------------------------------------------------- -# /* EV: R-44697-61543 */ -# # Test the effects of defining OMIT_TRIGGER but not OMIT_FOREIGN_KEY. # -# /* EV: R-22567-44039 */ -# /* EV: R-41784-13339 */ +# EVIDENCE-OF: R-44697-61543 If SQLITE_OMIT_TRIGGER is defined but +# SQLITE_OMIT_FOREIGN_KEY is not, then SQLite behaves as it did prior to +# version 3.6.19 - foreign key definitions are parsed and may be queried +# using PRAGMA foreign_key_list, but foreign key constraints are not +# enforced. # # Specifically, test that "PRAGMA foreign_keys" is a no-op in this case. # When using the pragma to query the current setting, 0 rows are returned. # +# EVIDENCE-OF: R-22567-44039 The PRAGMA foreign_keys command is a no-op +# in this configuration. +# +# EVIDENCE-OF: R-41784-13339 Tip: If the command "PRAGMA foreign_keys" +# returns no data instead of a single row containing "0" or "1", then +# the version of SQLite you are using does not support foreign keys +# (either because it is older than 3.6.19 or because it was compiled +# with SQLITE_OMIT_FOREIGN_KEY or SQLITE_OMIT_TRIGGER defined). +# reset_db ifcapable !trigger&&foreignkey { do_test e_fkey-2.1 { @@ -84,11 +93,11 @@ ifcapable !trigger&&foreignkey { #------------------------------------------------------------------------- -# /* EV: R-58428-36660 */ -# # Test the effects of defining OMIT_FOREIGN_KEY. # -# /* EV: R-58428-36660 */ +# EVIDENCE-OF: R-58428-36660 If OMIT_FOREIGN_KEY is defined, then +# foreign key definitions cannot even be parsed (attempting to specify a +# foreign key definition is a syntax error). # # Specifically, test that foreign key constraints cannot even be parsed # in such a build. @@ -120,15 +129,16 @@ reset_db #------------------------------------------------------------------------- -# /* EV: R-07280-60510 */ -# -# Test that even if foreign keys are supported by the build, they must -# be enabled using "PRAGMA foreign_keys = ON" (or similar). -# -# /* EV: R-59578-04990 */ +# EVIDENCE-OF: R-07280-60510 Assuming the library is compiled with +# foreign key constraints enabled, it must still be enabled by the +# application at runtime, using the PRAGMA foreign_keys command. # # This also tests that foreign key constraints are disabled by default. # +# EVIDENCE-OF: R-59578-04990 Foreign key constraints are disabled by +# default (for backwards compatibility), so must be enabled separately +# for each database connection separately. +# drop_all_tables do_test e_fkey-4.1 { execsql { @@ -153,13 +163,14 @@ do_test e_fkey-4.2 { } {world} #------------------------------------------------------------------------- -# /* EV: R-15278-54456 */ -# /* EV: R-11255-19907 */ +# EVIDENCE-OF: R-15278-54456 The application can can also use a PRAGMA +# foreign_keys statement to determine if foreign keys are currently +# enabled. # -# Test that the application can use "PRAGMA foreign_keys" to query for -# whether or not foreign keys are currently enabled. This also tests -# the example code in section 2 of foreignkeys.in. +# This also tests the example code in section 2 of foreignkeys.in. # +# EVIDENCE-OF: R-11255-19907 +# reset_db do_test e_fkey-5.1 { execsql { PRAGMA foreign_keys } @@ -178,11 +189,14 @@ do_test e_fkey-5.3 { } {0} #------------------------------------------------------------------------- -# /* EV: R-46649-58537 */ -# # Test that it is not possible to enable or disable foreign key support # while not in auto-commit mode. # +# EVIDENCE-OF: R-46649-58537 It is not possible to enable or disable +# foreign key constraints in the middle of a multi-statement transaction +# (when SQLite is not in autocommit mode). Attempting to do so does not +# return an error; it simply has no effect. +# reset_db do_test e_fkey-6.1 { execsql { @@ -221,10 +235,14 @@ do_test e_fkey-6.4 { execsql "PRAGMA foreign_keys = ON" #------------------------------------------------------------------------- -# /* EV: R-04042-24825 */ -# # Verify that the syntax in the first example in section 1 is valid. # +# EVIDENCE-OF: R-04042-24825 To do so, a foreign key definition may be +# added by modifying the declaration of the track table to the +# following: CREATE TABLE track( trackid INTEGER, trackname TEXT, +# trackartist INTEGER, FOREIGN KEY(trackartist) REFERENCES +# artist(artistid) ); +# do_test e_fkey-7.1 { execsql { CREATE TABLE artist( @@ -241,10 +259,9 @@ do_test e_fkey-7.1 { } {} #------------------------------------------------------------------------- -# /* EV: R-61362-32087 */ -# -# Attempting to insert a row into the 'track' table that corresponds -# to no row in the 'artist' table fails. +# EVIDENCE-OF: R-61362-32087 Attempting to insert a row into the track +# table that does not correspond to any row in the artist table will +# fail, # do_test e_fkey-8.1 { catchsql { INSERT INTO track VALUES(1, 'track 1', 1) } @@ -258,11 +275,12 @@ do_test e_fkey-8.2 { } {} #------------------------------------------------------------------------- -# /* EV: R-24401-52400 */ -# # Attempting to delete a row from the 'artist' table while there are # dependent rows in the track table also fails. # +# EVIDENCE-OF: R-24401-52400 as will attempting to delete a row from the +# artist table when there exist dependent rows in the track table +# do_test e_fkey-9.1 { catchsql { DELETE FROM artist WHERE artistid = 2 } } {1 {foreign key constraint failed}} @@ -274,11 +292,13 @@ do_test e_fkey-9.2 { } {} #------------------------------------------------------------------------- -# /* EV: R-23980-48859 */ -# # If the foreign key column (trackartist) in table 'track' is set to NULL, # there is no requirement for a matching row in the 'artist' table. # +# EVIDENCE-OF: R-23980-48859 There is one exception: if the foreign key +# column in the track table is NULL, then no corresponding entry in the +# artist table is required. +# do_test e_fkey-10.1 { execsql { INSERT INTO track VALUES(1, 'track 1', NULL); @@ -307,13 +327,15 @@ do_test e_fkey-10.5 { } {} #------------------------------------------------------------------------- -# /* EV: R-52486-21352 */ -# # Test that the following is true fo all rows in the track table: # # trackartist IS NULL OR # EXISTS(SELECT 1 FROM artist WHERE artistid=trackartist) # +# EVIDENCE-OF: R-52486-21352 Expressed in SQL, this means that for every +# row in the track table, the following expression evaluates to true: +# trackartist IS NULL OR EXISTS(SELECT 1 FROM artist WHERE +# artistid=trackartist) # This procedure executes a test case to check that statement # R-52486-21352 is true after executing the SQL statement passed. @@ -362,11 +384,14 @@ for {set i 0} {$i < 500} {incr i} { } #------------------------------------------------------------------------- -# /* EV: R-42412-59321 */ -# # Check that a NOT NULL constraint can be added to the example schema # to prohibit NULL child keys from being inserted. # +# EVIDENCE-OF: R-42412-59321 Tip: If the application requires a stricter +# relationship between artist and track, where NULL values are not +# permitted in the trackartist column, simply add the appropriate "NOT +# NULL" constraint to the schema. +# drop_all_tables do_test e_fkey-12.1 { execsql { @@ -387,7 +412,7 @@ do_test e_fkey-12.2 { } {1 {track.trackartist may not be NULL}} #------------------------------------------------------------------------- -# /* EV: R-17902-59250 */ +# EVIDENCE-OF: R-16127-35442 # # Test an example from foreignkeys.html. # @@ -431,7 +456,7 @@ do_test e_fkey-13.5 { } {} #------------------------------------------------------------------------- -# /* EV: R-15034-64331 */ +# EVIDENCE-OF: R-15958-50233 # # Test the second example from the first section of foreignkeys.html. # @@ -460,18 +485,18 @@ do_test e_fkey-14.4 { #------------------------------------------------------------------------- -# /* EV: R-56032-24923 */ +# EVIDENCE-OF: R-56032-24923 The foreign key constraint is satisfied if +# for each row in the child table either one or more of the child key +# columns are NULL, or there exists a row in the parent table for which +# each parent key column contains a value equal to the value in its +# associated child key column. # -# Test that a foreign key constraint is satisifed if "for each row in the child -# table either one or more of the child key columns are NULL, or there exists a -# row in the parent table for which each parent key column contains a value -# equal to the value in its associated child key column". -# -# /* EV: R-57765-12380 */ -# -# Test also that the comparison rules are used when testing if there +# Test also that the usual comparison rules are used when testing if there # is a matching row in the parent table of a foreign key constraint. # +# EVIDENCE-OF: R-57765-12380 In the above paragraph, the term "equal" +# means equal when values are compared using the rules specified here. +# drop_all_tables do_test e_fkey-15.1 { execsql { @@ -508,11 +533,12 @@ test_efkey_45 8 0 "INSERT INTO chi VALUES(X'31')" test_efkey_45 9 1 "INSERT INTO chi VALUES(X'32')" #------------------------------------------------------------------------- -# /* EV: R-15796-47513 */ -# # Specifically, test that when comparing child and parent key values the # default collation sequence of the parent key column is used. # +# EVIDENCE-OF: R-15796-47513 When comparing text values, the collating +# sequence associated with the parent key column is always used. +# drop_all_tables do_test e_fkey-16.1 { execsql { @@ -537,12 +563,14 @@ do_test e_fkey-16.4 { } {1 {foreign key constraint failed}} #------------------------------------------------------------------------- -# /* EV: R-04240-13860 */ -# # Specifically, test that when comparing child and parent key values the # affinity of the parent key column is applied to the child key value # before the comparison takes place. # +# EVIDENCE-OF: R-04240-13860 When comparing values, if the parent key +# column has an affinity, then that affinity is applied to the child key +# value before the comparison is performed. +# drop_all_tables do_test e_fkey-17.1 { execsql { @@ -571,18 +599,23 @@ do_test e_fkey-17.4 { ########################################################################### #------------------------------------------------------------------------- -# /* EV: R-13435-26311 */ -# # A parent key must be either a PRIMARY KEY, subject to a UNIQUE # constraint, or have a UNIQUE index created on it. -# -# /* EV: R-00376-39212 */ # +# EVIDENCE-OF: R-13435-26311 Usually, the parent key of a foreign key +# constraint is the primary key of the parent table. If they are not the +# primary key, then the parent key columns must be collectively subject +# to a UNIQUE constraint or have a UNIQUE index. +# # Also test that if a parent key is not subject to a PRIMARY KEY or UNIQUE # constraint, but does have a UNIQUE index created on it, then the UNIQUE index # must use the default collation sequences associated with the parent key # columns. # +# EVIDENCE-OF: R-00376-39212 If the parent key columns have a UNIQUE +# index, then that index must use the collation sequences that are +# specified in the CREATE TABLE statement for the parent table. +# drop_all_tables do_test e_fkey-18.1 { execsql { @@ -616,21 +649,27 @@ test_efkey_57 9 1 { # This block tests an example in foreignkeys.html. Several testable # statements refer to this example, as follows # -# /* EV: R-27484-01467 */ +# EVIDENCE-OF: R-27484-01467 # # FK Constraints on child1, child2 and child3 are Ok. # -# /* EV: R-51039-44840 */ +# Problem with FK on child4: # -# Problem with FK on child4. +# EVIDENCE-OF: R-51039-44840 The foreign key declared as part of table +# child4 is an error because even though the parent key column is +# indexed, the index is not UNIQUE. # -# /* EV: R-01060-48788 */ +# Problem with FK on child5: # -# Problem with FK on child5. +# EVIDENCE-OF: R-01060-48788 The foreign key for table child5 is an +# error because even though the parent key column has a unique index, +# the index uses a different collating sequence. # -# /* EV: R-63088-37469 */ +# Problem with FK on child6 and child7: # -# Problem with FK on child6 and child7. +# EVIDENCE-OF: R-63088-37469 Tables child6 and child7 are incorrect +# because while both have UNIQUE indices on their parent keys, the keys +# are not an exact match to the columns of a single UNIQUE index. # drop_all_tables do_test e_fkey-19.1 { @@ -671,15 +710,33 @@ do_test e_fkey-19.5 { } {1 {foreign key mismatch}} #------------------------------------------------------------------------- -# /* EV: R-45488-08504 */ -# /* EV: R-48391-38472 */ -# /* EV: R-03108-63659 */ -# /* EV: R-60781-26576 */ -# # Test errors in the database schema that are detected while preparing # DML statements. The error text for these messages always matches # either "foreign key mismatch" or "no such table*" (using [string match]). # +# EVIDENCE-OF: R-45488-08504 If the database schema contains foreign key +# errors that require looking at more than one table definition to +# identify, then those errors are not detected when the tables are +# created. +# +# EVIDENCE-OF: R-48391-38472 Instead, such errors prevent the +# application from preparing SQL statements that modify the content of +# the child or parent tables in ways that use the foreign keys. +# +# EVIDENCE-OF: R-03108-63659 The English language error message for +# foreign key DML errors is usually "foreign key mismatch" but can also +# be "no such table" if the parent table does not exist. +# +# EVIDENCE-OF: R-60781-26576 Foreign key DML errors are may be reported +# if: The parent table does not exist, or The parent key columns named +# in the foreign key constraint do not exist, or The parent key columns +# named in the foreign key constraint are not the primary key of the +# parent table and are not subject to a unique constraint using +# collating sequence specified in the CREATE TABLE, or The child table +# references the primary key of the parent without specifying the +# primary key columns and the number of primary key columns in the +# parent do not match the number of child key columns. +# do_test e_fkey-20.1 { execsql { CREATE TABLE c1(c REFERENCES nosuchtable, d); @@ -739,7 +796,7 @@ foreach {tn tbl ptbl err} { } #------------------------------------------------------------------------- -# /* EV: R-19353-43643 */ +# EVIDENCE-OF: R-19353-43643 # # Test the example of foreign key mismatch errors caused by implicitly # mapping a child key to the primary key of the parent table when the @@ -781,18 +838,23 @@ do_test e_fkey-21.8 { } {1 {foreign key mismatch}} #------------------------------------------------------------------------- -# /* EV: R-23682-59820 */ -# # Test errors that are reported when creating the child table. # Specifically: # # * different number of child and parent key columns, and # * child columns that do not exist. # -# /* EV: R-33883-28833 */ +# EVIDENCE-OF: R-23682-59820 By contrast, if foreign key errors can be +# recognized simply by looking at the definition of the child table and +# without having to consult the parent table definition, then the CREATE +# TABLE statement for the child table fails. # # These errors are reported whether or not FK support is enabled. # +# EVIDENCE-OF: R-33883-28833 Foreign key DDL errors are reported +# regardless of whether or not foreign key constraints are enabled when +# the table is created. +# drop_all_tables foreach fk [list OFF ON] { execsql "PRAGMA foreign_keys = $fk" @@ -814,10 +876,13 @@ foreach fk [list OFF ON] { } #------------------------------------------------------------------------- -# /* EV: R-47109-40581 */ -# # Test that a REFERENCING clause that does not specify parent key columns # implicitly maps to the primary key of the parent table. +# +# EVIDENCE-OF: R-43879-08025 Attaching a "REFERENCES " +# clause to a column definition creates a foreign +# key constraint that maps the column to the primary key of +# . # do_test e_fkey-23.1 { execsql { @@ -841,16 +906,18 @@ test_efkey_60 6 0 "INSERT INTO p2 VALUES(239, 231)" test_efkey_60 7 0 "INSERT INTO c2 VALUES(239, 231)" #------------------------------------------------------------------------- -# /* EV: R-15417-28014 */ -# # Test that an index on on the child key columns of an FK constraint # is optional. # -# /* EV: R-15741-50893 */ +# EVIDENCE-OF: R-15417-28014 Indices are not required for child key +# columns # # Also test that if an index is created on the child key columns, it does # not make a difference whether or not it is a UNIQUE index. # +# EVIDENCE-OF: R-15741-50893 The child key index does not have to be +# (and usually will not be) a UNIQUE index. +# drop_all_tables do_test e_fkey-24.1 { execsql { @@ -876,17 +943,16 @@ foreach {tn c} [list 2 c1 3 c2 4 c3] { } #------------------------------------------------------------------------- -# /* EV: R-00279-52283 */ +# EVIDENCE-OF: R-00279-52283 # # Test an example showing that when a row is deleted from the parent # table, the child table is queried for orphaned rows as follows: # # SELECT rowid FROM track WHERE trackartist = ? # -# /* EV: R-23302-30956 */ -# -# Also test that if the SELECT above would return any rows, a foreign -# key constraint is violated. +# EVIDENCE-OF: R-23302-30956 If this SELECT returns any rows at all, +# then SQLite concludes that deleting the row from the parent table +# would violate the foreign key constraint and returns an error. # do_test e_fkey-25.1 { execsql { @@ -944,7 +1010,8 @@ do_test e_fkey-25.7 { } {2 1 {foreign key constraint failed}} #------------------------------------------------------------------------- -# /* EV: R-54172-55848 */ +# EVIDENCE-OF: R-47936-10044 Or, more generally: +# SELECT rowid FROM WHERE = :parent_key_value # # Test that when a row is deleted from the parent table of an FK # constraint, the child table is queried for orphaned rows. The @@ -952,8 +1019,6 @@ do_test e_fkey-25.7 { # # SELECT rowid FROM WHERE = :parent_key_value # -# /* EV: R-61616-46700 */ -# # Also test that when a row is inserted into the parent table, or when the # parent key values of an existing row are modified, a query equivalent # to the following is planned. In some cases it is not executed, but it @@ -961,6 +1026,10 @@ do_test e_fkey-25.7 { # # SELECT rowid FROM WHERE = :parent_key_value # +# EVIDENCE-OF: R-61616-46700 Similar queries may be run if the content +# of the parent key is modified or a new row is inserted into the parent +# table. +# # drop_all_tables do_test e_fkey-26.1 { @@ -1000,7 +1069,7 @@ foreach {tn sql} { } #------------------------------------------------------------------------- -# /* EV: R-14553-34013 */ +# EVIDENCE-OF: R-14553-34013 # # Test the example schema at the end of section 3. Also test that is # is "efficient". In this case "efficient" means that foreign key @@ -1044,10 +1113,11 @@ do_test e_fkey-27.4 { ########################################################################### #------------------------------------------------------------------------- -# /* EV: R-41062-34431 */ -# # Check that parent and child keys must have the same number of columns. # +# EVIDENCE-OF: R-41062-34431 Parent and child keys must have the same +# cardinality. +# foreach {tn sql err} { 1 "CREATE TABLE c(jj REFERENCES p(x, y))" {foreign key on jj should reference only one column of table p} @@ -1091,7 +1161,7 @@ do_test e_fkey-28.9 { #------------------------------------------------------------------------- -# /* EV: R-24676-09859 */ +# EVIDENCE-OF: R-24676-09859 # # Test the example schema in the "Composite Foreign Key Constraints" # section. @@ -1130,10 +1200,9 @@ do_test e_fkey-29.3 { #------------------------------------------------------------------------- -# /* EV: R-33626-48418 */ -# -# Check that if any of the child key columns in the above schema are NULL, -# there is no requirement for a corresponding parent key. +# EVIDENCE-OF: R-33626-48418 In SQLite, if any of the child key columns +# (in this case songartist and songalbum) are NULL, then there is no +# requirement for a corresponding row in the parent table. # do_test e_fkey-30.1 { execsql { @@ -1147,18 +1216,16 @@ do_test e_fkey-30.1 { ########################################################################### #------------------------------------------------------------------------- -# Note: R-35290-16460 is tested below. -# -# TODO: R-30323-21917 - -#------------------------------------------------------------------------- -# /* EV: R-09323-30470 */ -# # Test that if a statement violates an immediate FK constraint, and the # database does not satisfy the FK constraint once all effects of the # statement have been applied, an error is reported and the effects of # the statement rolled back. # +# EVIDENCE-OF: R-09323-30470 If a statement modifies the contents of the +# database so that an immediate foreign key constraint is in violation +# at the conclusion the statement, an exception is thrown and the +# effects of the statement are reverted. +# drop_all_tables do_test e_fkey-31.1 { execsql { @@ -1204,16 +1271,25 @@ do_test e_fkey-31.5 { } {1 {} 2 {}} #------------------------------------------------------------------------- -# /* EV: R-49178-21358 */ -# /* EV: R-39692-12488 */ -# /* EV: R-55147-47664 */ -# /* EV: R-29604-30395 */ -# # Test that if a deferred constraint is violated within a transaction, # nothing happens immediately and the database is allowed to persist # in a state that does not satisfy the FK constraint. However attempts # to COMMIT the transaction fail until the FK constraint is satisfied. # +# EVIDENCE-OF: R-49178-21358 By contrast, if a statement modifies the +# contents of the database such that a deferred foreign key constraint +# is violated, the violation is not reported immediately. +# +# EVIDENCE-OF: R-39692-12488 Deferred foreign key constraints are not +# checked until the transaction tries to COMMIT. +# +# EVIDENCE-OF: R-55147-47664 For as long as the user has an open +# transaction, the database is allowed to exist in a state that violates +# any number of deferred foreign key constraints. +# +# EVIDENCE-OF: R-29604-30395 However, COMMIT will fail as long as +# foreign key constraints remain in violation. +# proc test_efkey_34 {tn isError sql} { do_test e_fkey-32.$tn " catchsql {$sql} @@ -1235,11 +1311,15 @@ test_efkey_34 8 0 "INSERT INTO ll VALUES(5)" test_efkey_34 9 0 "COMMIT" #------------------------------------------------------------------------- -# /* EV: R-56844-61705 */ -# # When not running inside a transaction, a deferred constraint is similar # to an immediate constraint (violations are reported immediately). # +# EVIDENCE-OF: R-56844-61705 If the current statement is not inside an +# explicit transaction (a BEGIN/COMMIT/ROLLBACK block), then an implicit +# transaction is committed as soon as the statement has finished +# executing. In this case deferred constraints behave the same as +# immediate constraints. +# drop_all_tables proc test_efkey_35 {tn isError sql} { do_test e_fkey-33.$tn " @@ -1261,14 +1341,14 @@ test_efkey_35 4 0 "INSERT INTO child VALUES('x', 'y')" #------------------------------------------------------------------------- -# /* EV: R-12782-61841 */ +# EVIDENCE-OF: R-12782-61841 # # Test that an FK constraint is made deferred by adding the following # to the definition: # # DEFERRABLE INITIALLY DEFERRED # -# /* EV: R-09005-28791 */ +# EVIDENCE-OF: R-09005-28791 # # Also test that adding any of the following to a foreign key definition # makes the constraint IMMEDIATE: @@ -1279,12 +1359,14 @@ test_efkey_35 4 0 "INSERT INTO child VALUES('x', 'y')" # DEFERRABLE INITIALLY IMMEDIATE # DEFERRABLE # -# /* EV: R-35290-16460 */ -# # Foreign keys are IMMEDIATE by default (if there is no DEFERRABLE or NOT # DEFERRABLE clause). # -# /* EV: R-30323-21917 */ FKs are either IMMEDIATE or DEFERRED. +# EVIDENCE-OF: R-35290-16460 Foreign key constraints are immediate by +# default. +# +# EVIDENCE-OF: R-30323-21917 Each foreign key constraint in SQLite is +# classified as either immediate or deferred. # drop_all_tables do_test e_fkey-34.1 { @@ -1381,7 +1463,7 @@ test_efkey_29 32 "COMMIT" 1 test_efkey_29 33 "ROLLBACK" 0 #------------------------------------------------------------------------- -# /* EV: R-35043-01546 */ +# EVIDENCE-OF: R-24499-57071 # # Test an example from foreignkeys.html dealing with a deferred foreign # key constraint. @@ -1415,11 +1497,13 @@ do_test e_fkey-35.3 { } {} #------------------------------------------------------------------------- -# /* EV: R-07223-48323 */ -# # Verify that a nested savepoint may be released without satisfying # deferred foreign key constraints. # +# EVIDENCE-OF: R-07223-48323 A nested savepoint transaction may be +# RELEASEd while the database is in a state that does not satisfy a +# deferred foreign key constraint. +# drop_all_tables do_test e_fkey-36.1 { execsql { @@ -1451,12 +1535,16 @@ do_test e_fkey-36.4 { #------------------------------------------------------------------------- -# /* EV: R-44295-13823 */ -# # Check that a transaction savepoint (an outermost savepoint opened when # the database was in auto-commit mode) cannot be released without # satisfying deferred foreign key constraints. It may be rolled back. # +# EVIDENCE-OF: R-44295-13823 A transaction savepoint (a non-nested +# savepoint that was opened while there was not currently an open +# transaction), on the other hand, is subject to the same restrictions +# as a COMMIT - attempting to RELEASE it while the database is in such a +# state will fail. +# do_test e_fkey-37.1 { execsql { SAVEPOINT one; @@ -1490,11 +1578,14 @@ do_test e_fkey-37.6 { } {} #------------------------------------------------------------------------- -# /* EV: R-37736-42616 */ -# # Test that if a COMMIT operation fails due to deferred foreign key # constraints, any nested savepoints remain open. # +# EVIDENCE-OF: R-37736-42616 If a COMMIT statement (or the RELEASE of a +# transaction SAVEPOINT) fails because the database is currently in a +# state that violates a deferred foreign key constraint and there are +# currently nested savepoints, the nested savepoints remain open. +# do_test e_fkey-38.1 { execsql { DELETE FROM t1 WHERE a>3; @@ -1551,16 +1642,20 @@ do_test e_fkey-38.8 { ########################################################################### #------------------------------------------------------------------------- -# /* EV: R-48270-44282 */ -# # Test that configured ON DELETE and ON UPDATE actions take place when # deleting or modifying rows of the parent table, respectively. # -# /* EV: R-48124-63225 */ +# EVIDENCE-OF: R-48270-44282 Foreign key ON DELETE and ON UPDATE clauses +# are used to configure actions that take place when deleting rows from +# the parent table (ON DELETE), or modifying the parent key values of +# existing rows (ON UPDATE). # # Test that a single FK constraint may have different actions configured # for ON DELETE and ON UPDATE. # +# EVIDENCE-OF: R-48124-63225 A single foreign key constraint may have +# different actions configured for ON DELETE and ON UPDATE. +# do_test e_fkey-39.1 { execsql { CREATE TABLE p(a, b PRIMARY KEY, c); @@ -1600,14 +1695,17 @@ do_test e_fkey-39.4 { } {1 xx k0 2 xx {} 3 xx {}} #------------------------------------------------------------------------- -# /* EV: R-33326-45252 */ -# # Each foreign key in the system has an ON UPDATE and ON DELETE action, # either "NO ACTION", "RESTRICT", "SET NULL", "SET DEFAULT" or "CASCADE". # -# /* EV: R-19803-45884 */ +# EVIDENCE-OF: R-33326-45252 The ON DELETE and ON UPDATE action +# associated with each foreign key in an SQLite database is one of "NO +# ACTION", "RESTRICT", "SET NULL", "SET DEFAULT" or "CASCADE". # # If none is specified explicitly, "NO ACTION" is the default. +# +# EVIDENCE-OF: R-19803-45884 If an action is not explicitly specified, +# it defaults to "NO ACTION". # drop_all_tables do_test e_fkey-40.1 { @@ -1648,11 +1746,13 @@ foreach {tn zTab lRes} { } #------------------------------------------------------------------------- -# /* EV: R-19971-54976 */ -# # Test that "NO ACTION" means that nothing happens to a child row when # it's parent row is updated or deleted. # +# EVIDENCE-OF: R-19971-54976 Configuring "NO ACTION" means just that: +# when a parent key is modified or deleted from the database, no special +# action is taken. +# drop_all_tables do_test e_fkey-41.1 { execsql { @@ -1685,12 +1785,15 @@ do_test e_fkey-41.4 { } {} #------------------------------------------------------------------------- -# /* EV: R-04272-38653 */ -# # Test that "RESTRICT" means the application is prohibited from deleting # or updating a parent table row when there exists one or more child keys # mapped to it. # +# EVIDENCE-OF: R-04272-38653 The "RESTRICT" action means that the +# application is prohibited from deleting (for ON DELETE RESTRICT) or +# modifying (for ON UPDATE RESTRICT) a parent key when there exists one +# or more child keys mapped to it. +# drop_all_tables do_test e_fkey-41.1 { execsql { @@ -1720,12 +1823,17 @@ do_test e_fkey-41.4 { } {1 {foreign key constraint failed}} #------------------------------------------------------------------------- -# /* EV: R-37997-42187 */ -# # Test that RESTRICT is slightly different from NO ACTION for IMMEDIATE # constraints, in that it is enforced immediately, not at the end of the # statement. # +# EVIDENCE-OF: R-37997-42187 The difference between the effect of a +# RESTRICT action and normal foreign key constraint enforcement is that +# the RESTRICT action processing happens as soon as the field is updated +# - not at the end of the current statement as it would with an +# immediate constraint, or at the end of the current transaction as it +# would with a deferred constraint. +# drop_all_tables do_test e_fkey-42.1 { execsql { @@ -1806,10 +1914,13 @@ do_test e_fkey-42.9 { } {key2} #------------------------------------------------------------------------- -# /* EV: R-24179-60523 */ -# # Test that RESTRICT is enforced immediately, even for a DEFERRED constraint. # +# EVIDENCE-OF: R-24179-60523 Even if the foreign key constraint it is +# attached to is deferred, configuring a RESTRICT action causes SQLite +# to return an error immediately if a parent key with dependent child +# keys is deleted or modified. +# drop_all_tables do_test e_fkey-43.1 { execsql { @@ -1879,10 +1990,14 @@ do_test e_fkey-43.10 { } {} #------------------------------------------------------------------------- -# /* EV: R-03353-05327 */ -# # Test SET NULL actions. # +# EVIDENCE-OF: R-03353-05327 If the configured action is "SET NULL", +# then when a parent key is deleted (for ON DELETE SET NULL) or modified +# (for ON UPDATE SET NULL), the child key columns of all rows in the +# child table that mapped to the parent key are set to contain SQL NULL +# values. +# drop_all_tables do_test e_fkey-44.1 { execsql { @@ -1918,10 +2033,12 @@ do_test e_fkey-44.5 { } {NULL} #------------------------------------------------------------------------- -# /* EV: R-43054-54832 */ -# # Test SET DEFAULT actions. # +# EVIDENCE-OF: R-43054-54832 The "SET DEFAULT" actions are similar to +# "SET NULL", except that each of the child key columns is set to +# contain the columns default value instead of NULL. +# drop_all_tables do_test e_fkey-45.1 { execsql { @@ -1958,11 +2075,15 @@ do_test e_fkey-45.5 { } {X'9999'} #------------------------------------------------------------------------- -# /* EV: R-61376-57267 */ -# /* EV: R-61809-62207 */ -# # Test ON DELETE CASCADE actions. # +# EVIDENCE-OF: R-61376-57267 A "CASCADE" action propagates the delete or +# update operation on the parent key to each dependent child key. +# +# EVIDENCE-OF: R-61809-62207 For an "ON DELETE CASCADE" action, this +# means that each row in the child table that was associated with the +# deleted parent row is also deleted. +# drop_all_tables do_test e_fkey-46.1 { execsql { @@ -1995,11 +2116,15 @@ do_test e_fkey-46.4 { #------------------------------------------------------------------------- -# /* EV: R-61376-57267 */ -# /* EV: R-13877-64542 */ -# # Test ON UPDATE CASCADE actions. # +# EVIDENCE-OF: R-13877-64542 For an "ON UPDATE CASCADE" action, it means +# that the values stored in each dependent child key are modified to +# match the new parent key values. +# +# EVIDENCE-OF: R-61376-57267 A "CASCADE" action propagates the delete or +# update operation on the parent key to each dependent child key. +# drop_all_tables do_test e_fkey-47.1 { execsql { @@ -2037,7 +2162,7 @@ do_test e_fkey-46.5 { } {{} 6 4 11 5 10} #------------------------------------------------------------------------- -# /* EV: R-51329-33438 */ +# EVIDENCE-OF: R-65058-57158 # # Test an example from the "ON DELETE and ON UPDATE Actions" section # of foreignkeys.html. @@ -2076,11 +2201,13 @@ do_test e_fkey-48.4 { #------------------------------------------------------------------------- -# /* EV: R-53968-51642 */ -# # Verify that adding an FK action does not absolve the user of the # requirement not to violate the foreign key constraint. # +# EVIDENCE-OF: R-53968-51642 Configuring an ON UPDATE or ON DELETE +# action does not mean that the foreign key constraint does not need to +# be satisfied. +# drop_all_tables do_test e_fkey-49.1 { execsql { @@ -2114,14 +2241,19 @@ do_test e_fkey-49.4 { #------------------------------------------------------------------------- -# /* EV: R-07065-59588 */ -# /* EV: R-28220-46694 */ +# EVIDENCE-OF: R-11856-19836 # # Test an example from the "ON DELETE and ON UPDATE Actions" section # of foreignkeys.html. This example shows that adding an "ON DELETE DEFAULT" # clause does not abrogate the need to satisfy the foreign key constraint # (R-28220-46694). # +# EVIDENCE-OF: R-28220-46694 For example, if an "ON DELETE SET DEFAULT" +# action is configured, but there is no row in the parent table that +# corresponds to the default values of the child key columns, deleting a +# parent key while dependent child keys exist still causes a foreign key +# violation. +# drop_all_tables do_test e_fkey-50.1 { execsql { @@ -2155,7 +2287,7 @@ do_test e_fkey-50.5 { } {14 {Mr. Bojangles} 0} #------------------------------------------------------------------------- -# /* EV: R-09564-22170 */ +# EVIDENCE-OF: R-09564-22170 # # Check that the order of steps in an UPDATE or DELETE on a parent # table is as follows: @@ -2207,13 +2339,15 @@ do_test e_fkey-51.3 { #------------------------------------------------------------------------- -# /* EV: R-27383-10246 */ -# # Verify that ON UPDATE actions only actually take place if the parent key # is set to a new value that is distinct from the old value. The default # collation sequence and affinity are used to determine if the new value # is 'distinct' from the old or not. # +# EVIDENCE-OF: R-27383-10246 An ON UPDATE action is only taken if the +# values of the parent key are modified so that the new parent key +# values are not equal to the old. +# drop_all_tables do_test e_fkey-52.1 { execsql { @@ -2261,7 +2395,7 @@ do_test e_fkey-52.6 { } {integer 1 null {}} #------------------------------------------------------------------------- -# /* EV: R-58589-50781 */ +# EVIDENCE-OF: R-35129-58141 # # Test an example from the "ON DELETE and ON UPDATE Actions" section # of foreignkeys.html. This example demonstrates that ON UPDATE actions @@ -2295,21 +2429,27 @@ do_test e_fkey-53.3 { ########################################################################### #------------------------------------------------------------------------- -# /* EV: R-36018-21755 */ -# /* EV: R-25384-39337 */ -# # Test that parent keys are not checked when tables are created. # +# EVIDENCE-OF: R-36018-21755 The parent key definitions of foreign key +# constraints are not checked when a table is created. +# +# EVIDENCE-OF: R-25384-39337 There is nothing stopping the user from +# creating a foreign key definition that refers to a parent table that +# does not exist, or to parent key columns that do not exist or are not +# collectively bound by a PRIMARY KEY or UNIQUE constraint. +# # Child keys are checked to ensure all component columns exist. If parent # key columns are explicitly specified, SQLite checks to make sure there # are the same number of columns in the child and parent keys. (TODO: This # is tested but does not correspond to any testable statement.) # -# /* EV: R-08908-23439 */ -# # Also test that the above statements are true regardless of whether or not # foreign keys are enabled: "A CREATE TABLE command operates the same whether # or not foreign key constraints are enabled." +# +# EVIDENCE-OF: R-08908-23439 A CREATE TABLE command operates the same +# whether or not foreign key constraints are enabled. # foreach {tn zCreateTbl lRes} { 1 "CREATE TABLE t1(a, b REFERENCES t1)" {0 {}} @@ -2338,7 +2478,10 @@ foreach {tn zCreateTbl lRes} { } #------------------------------------------------------------------------- -# /* EV: R-47952-62498 */ +# EVIDENCE-OF: R-47952-62498 It is not possible to use the "ALTER TABLE +# ... ADD COLUMN" syntax to add a column that includes a REFERENCES +# clause, unless the default value of the new column is NULL. Attempting +# to do so returns an error. # proc test_efkey_6 {tn zAlter isError} { drop_all_tables @@ -2355,15 +2498,20 @@ test_efkey_6 2 "ALTER TABLE tbl ADD COLUMN c DEFAULT NULL REFERENCES xx" 0 test_efkey_6 3 "ALTER TABLE tbl ADD COLUMN c DEFAULT 0 REFERENCES xx" 1 #------------------------------------------------------------------------- -# /* EV: R-47080-02069 */ -# # Test that ALTER TABLE adjusts REFERENCES clauses when the parent table # is RENAMED. # -# /* EV: R-63827-54774 */ +# EVIDENCE-OF: R-47080-02069 If an "ALTER TABLE ... RENAME TO" command +# is used to rename a table that is the parent table of one or more +# foreign key constraints, the definitions of the foreign key +# constraints are modified to refer to the parent table by its new name # # Test that these adjustments are visible in the sqlite_master table. # +# EVIDENCE-OF: R-63827-54774 The text of the child CREATE TABLE +# statement or statements stored in the sqlite_master table are modified +# to reflect the new parent table name. +# do_test e_fkey-56.1 { drop_all_tables execsql { @@ -2403,12 +2551,17 @@ do_test e_fkey-56.4 { ] #------------------------------------------------------------------------- -# /* EV: R-14208-23986 */ -# /* EV: R-11078-03945 */ -# # Check that a DROP TABLE does an implicit DELETE FROM. Which does not # cause any triggers to fire, but does fire foreign key actions. # +# EVIDENCE-OF: R-14208-23986 If foreign key constraints are enabled when +# it is prepared, the DROP TABLE command performs an implicit DELETE to +# remove all rows from the table before dropping it. +# +# EVIDENCE-OF: R-11078-03945 The implicit DELETE does not cause any SQL +# triggers to fire, but may invoke foreign key actions or constraint +# violations. +# do_test e_fkey-57.1 { drop_all_tables execsql { @@ -2469,11 +2622,12 @@ do_test e_fkey-57.7 { } {{delete 1}} #------------------------------------------------------------------------- -# /* EV: R-32768-47925 */ -# # If an IMMEDIATE foreign key fails as a result of a DROP TABLE, the # DROP TABLE command fails. # +# EVIDENCE-OF: R-32768-47925 If an immediate foreign key constraint is +# violated, the DROP TABLE statement fails and the table is not dropped. +# do_test e_fkey-58.1 { execsql { DELETE FROM c1; @@ -2501,11 +2655,14 @@ do_test e_fkey-58.4 { } {a b a b} #------------------------------------------------------------------------- -# /* EV: R-05903-08460 */ -# # If a DEFERRED foreign key fails as a result of a DROP TABLE, attempting # to commit the transaction fails unless the violation is fixed. # +# EVIDENCE-OF: R-05903-08460 If a deferred foreign key constraint is +# violated, then an error is reported when the user attempts to commit +# the transaction if the foreign key constraint violations still exist +# at that point. +# do_test e_fkey-59.1 { execsql { DELETE FROM c1 ; DELETE FROM c2 ; DELETE FROM c3 ; @@ -2533,11 +2690,12 @@ do_test e_fkey-59.5 { } {} #------------------------------------------------------------------------- -# /* EV: R-57242-37005 */ -# # Any "foreign key mismatch" errors encountered while running an implicit # "DELETE FROM tbl" are ignored. # +# EVIDENCE-OF: R-57242-37005 Any "foreign key mismatch" errors +# encountered as part of an implicit DELETE are ignored. +# drop_all_tables do_test e_fkey-60.1 { execsql { @@ -2580,8 +2738,6 @@ do_test e_fkey-60.6 { } {} #------------------------------------------------------------------------- -# /* EV: R-54142-41346 */ -# # Test that the special behaviours of ALTER and DROP TABLE are only # activated when foreign keys are enabled. Special behaviours are: # @@ -2590,6 +2746,9 @@ do_test e_fkey-60.6 { # 2. Modifying foreign key definitions when a parent table is RENAMEd. # 3. Running an implicit DELETE FROM command as part of DROP TABLE. # +# EVIDENCE-OF: R-54142-41346 The properties of the DROP TABLE and ALTER +# TABLE commands described above only apply if foreign keys are enabled. +# do_test e_fkey-61.1.1 { drop_all_tables execsql { CREATE TABLE t1(a, b) } @@ -2655,12 +2814,15 @@ do_test e_fkey-61.3.3 { ########################################################################### #------------------------------------------------------------------------- -# /* EV: R-24728-13230 */ -# /* EV: R-24450-46174 */ -# # Test that MATCH clauses are parsed, but SQLite treats every foreign key # constraint as if it were "MATCH SIMPLE". # +# EVIDENCE-OF: R-24728-13230 SQLite parses MATCH clauses (i.e. does not +# report a syntax error if you specify one), but does not enforce them. +# +# EVIDENCE-OF: R-24450-46174 All foreign key constraints in SQLite are +# handled as if MATCH SIMPLE were specified. +# foreach zMatch [list SIMPLE PARTIAL FULL Simple parTIAL FuLL ] { drop_all_tables do_test e_fkey-62.$zMatch.1 { @@ -2688,11 +2850,12 @@ foreach zMatch [list SIMPLE PARTIAL FULL Simple parTIAL FuLL ] { } #------------------------------------------------------------------------- -# /* EV: R-21599-16038 */ -# # Test that SQLite does not support the SET CONSTRAINT statement. And # that it is possible to create both immediate and deferred constraints. # +# EVIDENCE-OF: R-21599-16038 In SQLite, a foreign key constraint is +# permanently marked as deferred or immediate when it is created. +# drop_all_tables do_test e_fkey-62.1 { catchsql { SET CONSTRAINTS ALL IMMEDIATE } @@ -2728,12 +2891,15 @@ do_test e_fkey-62.7 { } {} #------------------------------------------------------------------------- -# /* EV: R-42264-30503 */ -# # Test that the maximum recursion depth of foreign key action programs is # governed by the SQLITE_MAX_TRIGGER_DEPTH and SQLITE_LIMIT_TRIGGER_DEPTH # settings. # +# EVIDENCE-OF: R-42264-30503 The SQLITE_MAX_TRIGGER_DEPTH and +# SQLITE_LIMIT_TRIGGER_DEPTH settings determine the maximum allowable +# depth of trigger program recursion. For the purposes of these limits, +# foreign key actions are considered trigger programs. +# proc test_on_delete_recursion {limit} { drop_all_tables execsql { @@ -2811,11 +2977,12 @@ do_test e_fkey-63.2.5 { } {5} #------------------------------------------------------------------------- -# /* EV: R-51769-32730 */ -# # The setting of the recursive_triggers pragma does not affect foreign # key actions. # +# EVIDENCE-OF: R-51769-32730 The PRAGMA recursive_triggers setting does +# not not affect the operation of foreign key actions. +# foreach recursive_triggers_setting [list 0 1 ON OFF] { drop_all_tables execsql "PRAGMA recursive_triggers = $recursive_triggers_setting" diff --git a/test/e_fts3.test b/test/e_fts3.test index 1c7afc26..6a14d50d 100644 --- a/test/e_fts3.test +++ b/test/e_fts3.test @@ -20,6 +20,7 @@ source $testdir/tester.tcl # ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl # Procs used to make the tests in this file easier to read. # @@ -46,7 +47,14 @@ proc error_test {tn sql result} { # DO_MALLOC_TEST=1: Run tests with transient OOM errors. # DO_MALLOC_TEST=2: Run tests with persistent OOM errors. # -foreach DO_MALLOC_TEST [lrange {0 1 2} 0 end] { +foreach {DO_MALLOC_TEST enc} { + 0 utf-8 + 1 utf-8 + 2 utf-8 + 1 utf-16 +} { + +#if {$DO_MALLOC_TEST} break # Reset the database and database connection. If this iteration of the # [foreach] loop is testing with OOM errors, disable the lookaside buffer. @@ -55,6 +63,15 @@ db close file delete -force test.db test.db-journal sqlite3 db test.db if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 } +db eval "PRAGMA encoding = '$enc'" + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit ########################################################################## # Test the example CREATE VIRTUAL TABLE statements in section 1.1 @@ -175,7 +192,7 @@ write_test 1.2.2.4 docs_content { } read_test 1.2.2.5 { SELECT count(*) FROM docs_segdir } {3} write_test 1.2.2.6 docs_segdir { - SELECT * FROM (SELECT optimize(docs) FROM docs LIMIT 1) WHERE 0; + INSERT INTO docs(docs) VALUES('optimize'); } read_test 1.2.2.7 { SELECT count(*) FROM docs_segdir } {1} ddl_test 1.2.2.8 { DROP TABLE docs } @@ -402,7 +419,7 @@ read_test 1.7.1.6 { ddl_test 1.7.2.1 { CREATE VIRTUAL TABLE text USING fts3() } -write_test 3.2.2 text_content { +write_test 1.7.2.2 text_content { INSERT INTO text VALUES(' During 30 Nov-1 Dec, 2-3oC drops. Cool in the upper portion, minimum temperature 14-16oC and cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6-12oC. Northeasterly winds 15-30 km/hr. After that, temperature increases. Northeasterly winds 15-30 km/hr. '); @@ -410,11 +427,28 @@ write_test 3.2.2 text_content { read_test 1.7.2.3 { SELECT snippet(text) FROM text WHERE text MATCH 'cold' -} {{... elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum ...}} +} {{...cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6...}} read_test 1.7.2.4 { SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"' -} {{... 2-3oC drops. Cool in the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] ...}} +} {{...the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] [temperature] 17-20oC. Cold...}} + +ddl_test 1.7.3.1 { DROP TABLE IF EXISTS t1 } +ddl_test 1.7.3.2 { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } +write_test 1.7.3.3 t1_content { + INSERT INTO t1 VALUES( + 'transaction default models default', 'Non transaction reads'); +} +write_test 1.7.3.4 t1_content { + INSERT INTO t1 VALUES('the default transaction', 'these semantics present'); +} +write_test 1.7.3.5 t1_content { + INSERT INTO t1 VALUES('single request', 'default data'); +} +read_test 1.7.3.6 { + SELECT mit(matchinfo(t1)) FROM t1 + WHERE t1 MATCH 'default transaction "these semantics"'; +} {{3 2 1 3 2 0 1 1 1 2 2 0 1 1 0 0 0 1 1 1}} ########################################################################## # Test the example in section 5 (custom tokenizers). @@ -443,6 +477,7 @@ read_test 1.8.2.4 { # functions are handled correctly. # set DO_MALLOC_TEST 0 +ddl_test 2.1.0 { DROP TABLE IF EXISTS t1 } ddl_test 2.1.1 { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } write_test 2.1.2 t1_content { INSERT INTO t1 VALUES('one two three', x'A1B2C3D4E5F6'); @@ -463,7 +498,208 @@ error_test 2.1.7 { SELECT snippet() FROM t1 WHERE a MATCH 'one' } {unable to use function snippet in the requested context} error_test 2.1.8 { - SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one' + SELECT snippet(a, b, 'A', 'B', 'C', 'D', 'E') FROM t1 WHERE a MATCH 'one' } {wrong number of arguments to function snippet()} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test the effect of an OOM error while installing the FTS3 module (i.e. +# opening a database handle). This case was not tested by the OOM testing +# of the document examples above. +# +do_malloc_test e_fts3-3 -tclbody { + if {[catch {sqlite3 db test.db}]} { error "out of memory" } +} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Verify the return values of the optimize() function. If no error occurs, +# the returned value should be "Index optimized" if the data structure +# was modified, or "Index already optimal" if it were not. +# +set DO_MALLOC_TEST 0 +ddl_test 4.1 { CREATE VIRTUAL TABLE t4 USING fts3(a, b) } +write_test 4.2 t4_content { + INSERT INTO t4 VALUES('In Xanadu', 'did Kubla Khan'); +} +write_test 4.3 t4_content { + INSERT INTO t4 VALUES('a stately pleasure', 'dome decree'); +} +do_test e_fts3-4.4 { + execsql { SELECT optimize(t4) FROM t4 LIMIT 1 } +} {{Index optimized}} +do_test e_fts3-4.5 { + execsql { SELECT optimize(t4) FROM t4 LIMIT 1 } +} {{Index already optimal}} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test that the snippet function appears to work correctly with 1, 2, 3 +# or 4 arguments passed to it. +# +set DO_MALLOC_TEST 0 +ddl_test 5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) } +write_test 5.2 t5_content { + INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea. So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.'); +} +read_test 5.3 { + SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles' +} {{...to a sunless sea. So twice five miles of fertile ground With walls and towers...}} +read_test 5.4 { + SELECT snippet(t5, '') FROM t5 WHERE t5 MATCH 'miles' +} {{...to a sunless sea. So twice five miles of fertile ground With walls and towers...}} +read_test 5.5 { + SELECT snippet(t5, '', '') FROM t5 WHERE t5 MATCH 'miles' +} {{...to a sunless sea. So twice five miles of fertile ground With walls and towers...}} +read_test 5.6 { + SELECT snippet(t5, '', '', 'XXX') FROM t5 WHERE t5 MATCH 'miles' +} {{XXXto a sunless sea. So twice five miles of fertile ground With walls and towersXXX}} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test that an empty MATCH expression returns an empty result set. As +# does passing a NULL value as a MATCH expression. +# +set DO_MALLOC_TEST 0 +ddl_test 6.1 { CREATE VIRTUAL TABLE t6 USING fts3(x) } +write_test 6.2 t5_content { INSERT INTO t6 VALUES('a'); } +write_test 6.3 t5_content { INSERT INTO t6 VALUES('b'); } +write_test 6.4 t5_content { INSERT INTO t6 VALUES('c'); } +read_test 6.5 { SELECT * FROM t6 WHERE t6 MATCH '' } {} +read_test 6.6 { SELECT * FROM t6 WHERE x MATCH '' } {} +read_test 6.7 { SELECT * FROM t6 WHERE t6 MATCH NULL } {} +read_test 6.8 { SELECT * FROM t6 WHERE x MATCH NULL } {} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test a few facets of the FTS3 xFilter() callback implementation: +# +# 1. That the sqlite3_index_constraint.usable flag is respected. +# +# 2. That it is an error to use the "docid" or "rowid" column of +# an FTS3 table as the LHS of a MATCH operator. +# +# 3. That it is an error to AND together two MATCH expressions in +# that refer to a single FTS3 table in a WHERE clause. +# +# +set DO_MALLOC_TEST 0 +ddl_test 7.1.1 { CREATE VIRTUAL TABLE t7 USING fts3(a) } +ddl_test 7.1.2 { CREATE VIRTUAL TABLE t8 USING fts3(b) } +write_test 7.1.3 t7_content { INSERT INTO t7(docid, a) VALUES(4,'number four') } +write_test 7.1.4 t7_content { INSERT INTO t7(docid, a) VALUES(5,'number five') } +write_test 7.1.5 t8_content { INSERT INTO t8(docid, b) VALUES(4,'letter D') } +write_test 7.1.6 t8_content { INSERT INTO t8(docid, b) VALUES(5,'letter E') } +read_test 7.1.7 { + SELECT a || ':' || b FROM t7 JOIN t8 USING(docid) +} {{number four:letter D} {number five:letter E}} + +error_test 7.2.1 { + SELECT * FROM t7 WHERE docid MATCH 'number' +} {unable to use function MATCH in the requested context} +error_test 7.2.2 { + SELECT * FROM t7 WHERE rowid MATCH 'number' +} {unable to use function MATCH in the requested context} + +error_test 7.3.1 { + SELECT * FROM t7 WHERE a MATCH 'number' AND a MATCH 'four' +} {unable to use function MATCH in the requested context} +error_test 7.3.2 { + SELECT * FROM t7, t8 WHERE a MATCH 'number' AND a MATCH 'four' +} {unable to use function MATCH in the requested context} +error_test 7.3.3 { + SELECT * FROM t7, t8 WHERE b MATCH 'letter' AND b MATCH 'd' +} {unable to use function MATCH in the requested context} +read_test 7.3.4 { + SELECT * FROM t7, t8 WHERE a MATCH 'number' AND b MATCH 'letter' +} {{number four} {letter D} {number four} {letter E} {number five} {letter D} {number five} {letter E}} +read_test 7.3.5 { + SELECT * FROM t7 WHERE a MATCH 'number' AND docid = 4 +} {{number four}} + +#------------------------------------------------------------------------- +# Test the quoting of FTS3 table column names. Names may be quoted using +# any of "", '', ``` or []. +# +set DO_MALLOC_TEST 0 +ddl_test 8.1.1 { CREATE VIRTUAL TABLE t9a USING fts3("c1", [c2]) } +ddl_test 8.1.2 { CREATE VIRTUAL TABLE t9b USING fts3('c1', `c2`) } +read_test 8.1.3 { PRAGMA table_info(t9a) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0} +read_test 8.1.4 { PRAGMA table_info(t9b) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0} +ddl_test 8.2.1 { CREATE VIRTUAL TABLE t9c USING fts3("c""1", 'c''2') } +read_test 8.2.2 { PRAGMA table_info(t9c) } {0 c\"1 {} 0 {} 0 1 c'2 {} 0 {} 0} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test that FTS3 tables can be renamed using the ALTER RENAME command. +# OOM errors are tested during ALTER RENAME commands also. +# +foreach DO_MALLOC_TEST {0 1 2} { + db close + file delete -force test.db test.db-journal + sqlite3 db test.db + if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 } + + ddl_test 9.1.1 { CREATE VIRTUAL TABLE t10 USING fts3(x) } + write_test 9.1.2 t10_content { INSERT INTO t10 VALUES('fts3 tables') } + write_test 9.1.3 t10_content { INSERT INTO t10 VALUES('are renameable') } + + read_test 9.1.4 { + SELECT * FROM t10 WHERE t10 MATCH 'table*' + } {{fts3 tables}} + read_test 9.1.5 { + SELECT * FROM t10 WHERE x MATCH 'rename*' + } {{are renameable}} + + ddl_test 9.1.6 { ALTER TABLE t10 RENAME TO t11 } + + read_test 9.1.7 { + SELECT * FROM t11 WHERE t11 MATCH 'table*' + } {{fts3 tables}} + read_test 9.1.8 { + SELECT * FROM t11 WHERE x MATCH 'rename*' + } {{are renameable}} +} +#------------------------------------------------------------------------- + +#------------------------------------------------------------------------- +# Test a couple of cases involving corrupt data structures: +# +# 1) A case where a document referenced by the full-text index is +# not present in the %_content table. +# +# 2) A badly formatted b-tree segment node. +# +set DO_MALLOC_TEST 0 +ddl_test 10.1.1 { CREATE VIRTUAL TABLE ta USING fts3 } +write_test 10.1.2 ta_content { + INSERT INTO ta VALUES('During a summer vacation in 1790') } +write_test 10.1.3 ta_content { + INSERT INTO ta VALUES('Wordsworth went on a walking tour') } +write_test 10.1.4 ta_content { DELETE FROM ta_content WHERE rowid = 2 } +read_test 10.1.5 { + SELECT * FROM ta WHERE ta MATCH 'summer' +} {{During a summer vacation in 1790}} +error_test 10.1.6 { + SELECT * FROM ta WHERE ta MATCH 'walking' +} {database disk image is malformed} + +write_test 10.2.1 ta_content { DELETE FROM ta } +write_test 10.2.2 ta_content { + INSERT INTO ta VALUES('debate demonstrated the rising difficulty') } +write_test 10.2.3 ta_content { + INSERT INTO ta VALUES('Google released its browser beta') } + +set blob [db one {SELECT root FROM ta_segdir WHERE rowid = 2}] +binary scan $blob "a6 a3 a*" start middle end +set middle "\x0E\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x06\x06" +set blob [binary format "a6 a* a*" $start $middle $end] +write_test 10.2.4 ta_segdir { + UPDATE ta_segdir SET root = $blob WHERE rowid = 2 +} +error_test 10.2.5 { + SELECT * FROM ta WHERE ta MATCH 'beta' +} {database disk image is malformed} + finish_test diff --git a/test/fkey2.test b/test/fkey2.test index dc6529be..10a624ef 100644 --- a/test/fkey2.test +++ b/test/fkey2.test @@ -1003,6 +1003,169 @@ ifcapable altertable { do_test fkey2-14.2.2.7 { execsql { INSERT INTO t3 VALUES(1, NULL, 1) } } {} + + # Repeat for TEMP tables + # + drop_all_tables + do_test fkey2-14.1tmp.1 { + # Adding a column with a REFERENCES clause is not supported. + execsql { + CREATE TEMP TABLE t1(a PRIMARY KEY); + CREATE TEMP TABLE t2(a, b); + } + catchsql { ALTER TABLE t2 ADD COLUMN c REFERENCES t1 } + } {0 {}} + do_test fkey2-14.1tmp.2 { + catchsql { ALTER TABLE t2 ADD COLUMN d DEFAULT NULL REFERENCES t1 } + } {0 {}} + do_test fkey2-14.1tmp.3 { + catchsql { ALTER TABLE t2 ADD COLUMN e REFERENCES t1 DEFAULT NULL} + } {0 {}} + do_test fkey2-14.1tmp.4 { + catchsql { ALTER TABLE t2 ADD COLUMN f REFERENCES t1 DEFAULT 'text'} + } {1 {Cannot add a REFERENCES column with non-NULL default value}} + do_test fkey2-14.1tmp.5 { + catchsql { ALTER TABLE t2 ADD COLUMN g DEFAULT CURRENT_TIME REFERENCES t1 } + } {1 {Cannot add a REFERENCES column with non-NULL default value}} + do_test fkey2-14.1tmp.6 { + execsql { + PRAGMA foreign_keys = off; + ALTER TABLE t2 ADD COLUMN h DEFAULT 'text' REFERENCES t1; + PRAGMA foreign_keys = on; + SELECT sql FROM sqlite_temp_master WHERE name='t2'; + } + } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}} + + do_test fkey2-14.2tmp.1.1 { + test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3 + } {{CREATE TABLE t1(a REFERENCES "t3")}} + do_test fkey2-14.2tmp.1.2 { + test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t4 t3 + } {{CREATE TABLE t1(a REFERENCES t2)}} + do_test fkey2-14.2tmp.1.3 { + test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3 + } {{CREATE TABLE t1(a REFERENCES "t3")}} + + # Test ALTER TABLE RENAME TABLE a bit. + # + do_test fkey2-14.2tmp.2.1 { + drop_all_tables + execsql { + CREATE TEMP TABLE t1(a PRIMARY KEY, b REFERENCES t1); + CREATE TEMP TABLE t2(a PRIMARY KEY, b REFERENCES t1, c REFERENCES t2); + CREATE TEMP TABLE t3(a REFERENCES t1, b REFERENCES t2, c REFERENCES t1); + } + execsql { SELECT sql FROM sqlite_temp_master WHERE type = 'table'} + } [list \ + {CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t1)} \ + {CREATE TABLE t2(a PRIMARY KEY, b REFERENCES t1, c REFERENCES t2)} \ + {CREATE TABLE t3(a REFERENCES t1, b REFERENCES t2, c REFERENCES t1)} \ + ] + do_test fkey2-14.2tmp.2.2 { + execsql { ALTER TABLE t1 RENAME TO t4 } + execsql { SELECT sql FROM sqlite_temp_master WHERE type = 'table'} + } [list \ + {CREATE TABLE "t4"(a PRIMARY KEY, b REFERENCES "t4")} \ + {CREATE TABLE t2(a PRIMARY KEY, b REFERENCES "t4", c REFERENCES t2)} \ + {CREATE TABLE t3(a REFERENCES "t4", b REFERENCES t2, c REFERENCES "t4")} \ + ] + do_test fkey2-14.2tmp.2.3 { + catchsql { INSERT INTO t3 VALUES(1, 2, 3) } + } {1 {foreign key constraint failed}} + do_test fkey2-14.2tmp.2.4 { + execsql { INSERT INTO t4 VALUES(1, NULL) } + } {} + do_test fkey2-14.2tmp.2.5 { + catchsql { UPDATE t4 SET b = 5 } + } {1 {foreign key constraint failed}} + do_test fkey2-14.2tmp.2.6 { + catchsql { UPDATE t4 SET b = 1 } + } {0 {}} + do_test fkey2-14.2tmp.2.7 { + execsql { INSERT INTO t3 VALUES(1, NULL, 1) } + } {} + + # Repeat for ATTACH-ed tables + # + drop_all_tables + do_test fkey2-14.1aux.1 { + # Adding a column with a REFERENCES clause is not supported. + execsql { + ATTACH ':memory:' AS aux; + CREATE TABLE aux.t1(a PRIMARY KEY); + CREATE TABLE aux.t2(a, b); + } + catchsql { ALTER TABLE t2 ADD COLUMN c REFERENCES t1 } + } {0 {}} + do_test fkey2-14.1aux.2 { + catchsql { ALTER TABLE t2 ADD COLUMN d DEFAULT NULL REFERENCES t1 } + } {0 {}} + do_test fkey2-14.1aux.3 { + catchsql { ALTER TABLE t2 ADD COLUMN e REFERENCES t1 DEFAULT NULL} + } {0 {}} + do_test fkey2-14.1aux.4 { + catchsql { ALTER TABLE t2 ADD COLUMN f REFERENCES t1 DEFAULT 'text'} + } {1 {Cannot add a REFERENCES column with non-NULL default value}} + do_test fkey2-14.1aux.5 { + catchsql { ALTER TABLE t2 ADD COLUMN g DEFAULT CURRENT_TIME REFERENCES t1 } + } {1 {Cannot add a REFERENCES column with non-NULL default value}} + do_test fkey2-14.1aux.6 { + execsql { + PRAGMA foreign_keys = off; + ALTER TABLE t2 ADD COLUMN h DEFAULT 'text' REFERENCES t1; + PRAGMA foreign_keys = on; + SELECT sql FROM aux.sqlite_master WHERE name='t2'; + } + } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}} + + do_test fkey2-14.2aux.1.1 { + test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t2 t3 + } {{CREATE TABLE t1(a REFERENCES "t3")}} + do_test fkey2-14.2aux.1.2 { + test_rename_parent {CREATE TABLE t1(a REFERENCES t2)} t4 t3 + } {{CREATE TABLE t1(a REFERENCES t2)}} + do_test fkey2-14.2aux.1.3 { + test_rename_parent {CREATE TABLE t1(a REFERENCES "t2")} t2 t3 + } {{CREATE TABLE t1(a REFERENCES "t3")}} + + # Test ALTER TABLE RENAME TABLE a bit. + # + do_test fkey2-14.2aux.2.1 { + drop_all_tables + execsql { + CREATE TABLE aux.t1(a PRIMARY KEY, b REFERENCES t1); + CREATE TABLE aux.t2(a PRIMARY KEY, b REFERENCES t1, c REFERENCES t2); + CREATE TABLE aux.t3(a REFERENCES t1, b REFERENCES t2, c REFERENCES t1); + } + execsql { SELECT sql FROM aux.sqlite_master WHERE type = 'table'} + } [list \ + {CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t1)} \ + {CREATE TABLE t2(a PRIMARY KEY, b REFERENCES t1, c REFERENCES t2)} \ + {CREATE TABLE t3(a REFERENCES t1, b REFERENCES t2, c REFERENCES t1)} \ + ] + do_test fkey2-14.2aux.2.2 { + execsql { ALTER TABLE t1 RENAME TO t4 } + execsql { SELECT sql FROM aux.sqlite_master WHERE type = 'table'} + } [list \ + {CREATE TABLE "t4"(a PRIMARY KEY, b REFERENCES "t4")} \ + {CREATE TABLE t2(a PRIMARY KEY, b REFERENCES "t4", c REFERENCES t2)} \ + {CREATE TABLE t3(a REFERENCES "t4", b REFERENCES t2, c REFERENCES "t4")} \ + ] + do_test fkey2-14.2aux.2.3 { + catchsql { INSERT INTO t3 VALUES(1, 2, 3) } + } {1 {foreign key constraint failed}} + do_test fkey2-14.2aux.2.4 { + execsql { INSERT INTO t4 VALUES(1, NULL) } + } {} + do_test fkey2-14.2aux.2.5 { + catchsql { UPDATE t4 SET b = 5 } + } {1 {foreign key constraint failed}} + do_test fkey2-14.2aux.2.6 { + catchsql { UPDATE t4 SET b = 1 } + } {0 {}} + do_test fkey2-14.2aux.2.7 { + execsql { INSERT INTO t3 VALUES(1, NULL, 1) } + } {} } do_test fkey-2.14.3.1 { diff --git a/test/fts3_common.tcl b/test/fts3_common.tcl index f954f5c3..ce6678d6 100644 --- a/test/fts3_common.tcl +++ b/test/fts3_common.tcl @@ -208,7 +208,7 @@ proc fts3_doclist {tbl term where} { proc gobble_varint {varname} { upvar $varname blob - set n [read_varint $blob ret] + set n [read_fts3varint $blob ret] set blob [string range $blob $n end] return $ret } @@ -310,36 +310,65 @@ proc fts3_read {tbl where varname} { # match the expected results passed via parameter $result. # proc do_select_test {name sql result} { - doPassiveTest $name $sql [list 0 $result] + uplevel [list doPassiveTest 0 $name $sql [list 0 $result]] +} + +proc do_restart_select_test {name sql result} { + uplevel [list doPassiveTest 1 $name $sql [list 0 $result]] } proc do_error_test {name sql error} { - doPassiveTest $name $sql [list 1 $error] + uplevel [list doPassiveTest 0 $name $sql [list 1 $error]] } -proc doPassiveTest {name sql catchres} { +proc doPassiveTest {isRestart name sql catchres} { if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } - if {$::DO_MALLOC_TEST} { - set answers [list {1 {out of memory}} $catchres] - set modes [list 100000 transient 1 persistent] - } else { - set answers [list $catchres] - set modes [list 0 nofail] + switch $::DO_MALLOC_TEST { + 0 { # No malloc failures. + do_test $name [list set {} [uplevel [list catchsql $sql]]] $catchres + return + } + 1 { # Simulate transient failures. + set nRepeat 1 + set zName "transient" + set nStartLimit 100000 + set nBackup 1 + } + 2 { # Simulate persistent failures. + set nRepeat 1 + set zName "persistent" + set nStartLimit 100000 + set nBackup 1 + } + 3 { # Simulate transient failures with extra brute force. + set nRepeat 100000 + set zName "ridiculous" + set nStartLimit 1 + set nBackup 10 + } } + + # The set of acceptable results from running [catchsql $sql]. + # + set answers [list {1 {out of memory}} $catchres] set str [join $answers " OR "] - foreach {nRepeat zName} $modes { - for {set iFail 1} 1 {incr iFail} { - if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} + set nFail 1 + for {set iLimit $nStartLimit} {$nFail} {incr iLimit} { + for {set iFail 1} {$nFail && $iFail<=$iLimit} {incr iFail} { + for {set iTest 0} {$iTest<$nBackup && ($iFail-$iTest)>0} {incr iTest} { - set res [catchsql $sql] - if {[lsearch -exact $answers $res]>=0} { - set res $str + if {$isRestart} { sqlite3 db test.db } + + sqlite3_memdebug_fail [expr $iFail-$iTest] -repeat $nRepeat + set res [uplevel [list catchsql $sql]] + if {[lsearch -exact $answers $res]>=0} { set res $str } + set testname "$name.$zName.$iFail" + do_test "$name.$zName.$iLimit.$iFail" [list set {} $res] $str + + set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] } - do_test $name.$zName.$iFail [list set {} $res] $str - set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] - if {$nFail==0} break } } } diff --git a/test/fts3ac.test b/test/fts3ac.test index 72e5410c..84da924b 100644 --- a/test/fts3ac.test +++ b/test/fts3ac.test @@ -1131,39 +1131,36 @@ do_test fts3ac-4.2 { SELECT snippet(email) FROM email WHERE email MATCH 'christmas candlelight' } -} {{... place.? What do you think about going here Christmas -eve?? They have an 11:00 a.m. service and a candlelight service at 5:00 p.m., -among others. ...}} +} {{...here Christmas +eve?? They have an 11:00 a.m. service and a candlelight service...}} do_test fts3ac-4.3 { execsql { SELECT snippet(email) FROM email WHERE email MATCH 'deal sheet potential reuse' } -} {{EOL-Accenture Deal Sheet ... intent - Review Enron asset base for potential reuse/ licensing - Contract negotiations ...}} +} {{EOL-Accenture Deal Sheet...asset base for potential reuse/ licensing + Contract negotiations...}} do_test fts3ac-4.4 { execsql { SELECT snippet(email,'<<<','>>>',' ') FROM email WHERE email MATCH 'deal sheet potential reuse' } -} {{EOL-Accenture <<>> <<>> intent - Review Enron asset base for <<>> <<>>/ licensing - Contract negotiations }} +} {{EOL-Accenture <<>> <<>> asset base for <<>> <<>>/ licensing + Contract negotiations }} do_test fts3ac-4.5 { execsql { SELECT snippet(email,'<<<','>>>',' ') FROM email WHERE email MATCH 'first things' } -} {{Re: <<>> Polish Deal! Congrats! <<>> seem to be building rapidly now on the }} +} {{Re: <<>> Polish Deal! Congrats! <<>> seem to be building rapidly now }} do_test fts3ac-4.6 { execsql { SELECT snippet(email) FROM email WHERE email MATCH 'chris is here' } -} {{chris.germany@enron.com ... Sounds good to me. I bet this is next to the Warick?? Hotel. ... place.? What do you think about going here Christmas -eve?? They have an 11:00 a.m. ...}} +} {{...chris.germany@enron.com'" <chris...bet this is next to...about going here Christmas +eve...}} do_test fts3ac-4.7 { execsql { SELECT snippet(email) FROM email @@ -1171,19 +1168,15 @@ do_test fts3ac-4.7 { } } {{Erin: -Pursuant to your request, attached are the Schedule to ...}} +Pursuant to your request, attached are the Schedule to the ISDA Master Agreement, together...}} do_test fts3ac-4.8 { execsql { SELECT snippet(email) FROM email WHERE email MATCH 'ancillary load davis' } -} {{pete.davis@enron.com ... Start Date: 4/22/01; HourAhead hour: 3; No ancillary schedules awarded. -Variances detected. -Variances detected in Load schedule. +} {{pete.davis@enron.com...3; No ancillary schedules awarded...detected in Load schedule. - LOG MESSAGES: - -PARSING ...}} + LOG...}} # Combinations of AND and OR operators: # @@ -1192,22 +1185,35 @@ do_test fts3ac-5.1 { SELECT snippet(email) FROM email WHERE email MATCH 'questar enron OR com' } -} {{matt.smith@enron.com ... six reports: - -31 Keystone Receipts +} {{matt.smith@enron.com...31 Keystone Receipts 15 Questar Pipeline -40 Rockies Production -22 West_2 ...}} +40 Rockies...}} + do_test fts3ac-5.2 { execsql { SELECT snippet(email) FROM email WHERE email MATCH 'enron OR com questar' } -} {{matt.smith@enron.com ... six reports: - -31 Keystone Receipts +} {{matt.smith@enron.com...31 Keystone Receipts 15 Questar Pipeline -40 Rockies Production -22 West_2 ...}} +40 Rockies...}} + +#------------------------------------------------------------------------- +# Test a problem reported on the mailing list. +# +do_test fts3ac-6.1 { + execsql { + CREATE VIRTUAL TABLE ft USING fts3(one, two); + INSERT INTO ft VALUES('', 'foo'); + INSERT INTO ft VALUES('foo', 'foo'); + SELECT offsets(ft) FROM ft WHERE ft MATCH 'foo'; + } +} {{1 0 0 3} {0 0 0 3 1 0 0 3}} +do_test fts3ac-6.2 { + execsql { + DELETE FROM ft WHERE one = 'foo'; + SELECT offsets(ft) FROM ft WHERE ft MATCH 'foo'; + } +} {{1 0 0 3}} finish_test diff --git a/test/fts3ag.test b/test/fts3ag.test index 34ef9497..18ed5ae3 100644 --- a/test/fts3ag.test +++ b/test/fts3ag.test @@ -73,7 +73,7 @@ do_test fts3ag-1.9 { # No support for all-except queries. do_test fts3ag-1.10 { catchsql {SELECT rowid FROM t1 WHERE t1 MATCH '-this -something'} -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [-this -something]}} # Test that docListOrMerge() correctly handles reaching the end of one # doclist before it reaches the end of the other. diff --git a/test/fts3al.test b/test/fts3al.test index be01ecb7..02cc0d16 100644 --- a/test/fts3al.test +++ b/test/fts3al.test @@ -53,6 +53,10 @@ do_test fts3al-1.3 { # # The trailing and leading hi-bit chars help with code which tests for # isspace() to coalesce multiple spaces. +# +# UPDATE: The above is no longer true; there is no such code in fts3. +# But leave the test in just the same. +# set word "\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80" set phrase1 "$word $word $word target $word $word $word" @@ -64,6 +68,6 @@ db eval "INSERT INTO t4 (content) VALUES ('$phrase2')" do_test fts3al-1.4 { execsql {SELECT rowid, length(snippet(t4)) FROM t4 WHERE t4 MATCH 'target'} -} {1 111 2 117} +} {1 241 2 247} finish_test diff --git a/test/fts3cov.test b/test/fts3cov.test new file mode 100644 index 00000000..d3fe4fa8 --- /dev/null +++ b/test/fts3cov.test @@ -0,0 +1,373 @@ +# 2009 December 03 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The tests in this file are structural coverage tests. They are designed +# to complement the tests in fts3rnd.test and fts3doc.test. Between them, +# the three files should provide full coverage of the fts3 extension code. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set DO_MALLOC_TEST 0 + +#-------------------------------------------------------------------------- +# When it first needs to read a block from the %_segments table, the FTS3 +# module compiles an SQL statement for that purpose. The statement is +# stored and reused each subsequent time a block is read. This test case +# tests the effects of an OOM error occuring while compiling the statement. +# +# Similarly, when FTS3 first needs to scan through a set of segment leaves +# to find a set of documents that matches a term, it allocates a string +# containing the text of the required SQL, and compiles one or more +# statements to traverse the leaves. This test case tests that OOM errors +# that occur while allocating this string and statement are handled correctly +# also. +# +do_test fts3cov-1.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts3(x); + INSERT INTO t1(t1) VALUES('nodesize=24'); + BEGIN; + INSERT INTO t1 VALUES('Is the night chilly and dark?'); + INSERT INTO t1 VALUES('The night is chilly, but not dark.'); + INSERT INTO t1 VALUES('The thin gray cloud is spread on high,'); + INSERT INTO t1 VALUES('It covers but not hides the sky.'); + COMMIT; + SELECT count(*)>0 FROM t1_segments; + } +} {1} + +set DO_MALLOC_TEST 1 +do_restart_select_test fts3cov-1.2 { + SELECT docid FROM t1 WHERE t1 MATCH 'chilly'; +} {1 2} +set DO_MALLOC_TEST 0 + +#-------------------------------------------------------------------------- +# When querying the full-text index, if an expected internal node block is +# missing from the %_segments table, or if a NULL value is stored in the +# %_segments table instead of a binary blob, database corruption should be +# reported. +# +# Even with tiny 24 byte nodes, it takes a fair bit of data to produce a +# segment b-tree that uses the %_segments table to store internal nodes. +# +do_test fts3cov-2.1 { + execsql { + INSERT INTO t1(t1) VALUES('nodesize=24'); + BEGIN; + INSERT INTO t1 VALUES('The moon is behind, and at the full;'); + INSERT INTO t1 VALUES('And yet she looks both small and dull.'); + INSERT INTO t1 VALUES('The night is chill, the cloud is gray:'); + INSERT INTO t1 VALUES('''T is a month before the month of May,'); + INSERT INTO t1 VALUES('And the Spring comes slowly up this way.'); + INSERT INTO t1 VALUES('The lovely lady, Christabel,'); + INSERT INTO t1 VALUES('Whom her father loves so well,'); + INSERT INTO t1 VALUES('What makes her in the wood so late,'); + INSERT INTO t1 VALUES('A furlong from the castle gate?'); + INSERT INTO t1 VALUES('She had dreams all yesternight'); + INSERT INTO t1 VALUES('Of her own betrothed knight;'); + INSERT INTO t1 VALUES('And she in the midnight wood will pray'); + INSERT INTO t1 VALUES('For the weal of her lover that''s far away.'); + COMMIT; + + INSERT INTO t1(t1) VALUES('optimize'); + SELECT substr(hex(root), 1, 2) FROM t1_segdir; + } +} {03} + +# Test the "missing entry" case: +do_test fts3cov-2.1 { + set root [db one {SELECT root FROM t1_segdir}] + read_fts3varint [string range $root 1 end] left_child + execsql { DELETE FROM t1_segments WHERE blockid = $left_child } +} {} +do_error_test fts3cov-2.2 { + SELECT * FROM t1 WHERE t1 MATCH 'c*' +} {database disk image is malformed} + +# Test the "replaced with NULL" case: +do_test fts3cov-2.3 { + execsql { INSERT INTO t1_segments VALUES($left_child, NULL) } +} {} +do_error_test fts3cov-2.4 { + SELECT * FROM t1 WHERE t1 MATCH 'cloud' +} {database disk image is malformed} + +#-------------------------------------------------------------------------- +# The following tests are to test the effects of OOM errors while storing +# terms in the pending-hash table. Specifically, while creating doclist +# blobs to store in the table. More specifically, to test OOM errors while +# appending column numbers to doclists. For example, if a doclist consists +# of: +# +# 0x01 +# +# The following tests check that malloc errors encountered while appending +# the "0x01 " data to the dynamically growable blob used to +# accumulate the doclist in memory are handled correctly. +# +do_test fts3cov-3.1 { + set cols [list] + set vals [list] + for {set i 0} {$i < 120} {incr i} { + lappend cols "col$i" + lappend vals "'word'" + } + execsql "CREATE VIRTUAL TABLE t2 USING fts3([join $cols ,])" +} {} +set DO_MALLOC_TEST 1 +do_write_test fts3cov-3.2 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(1, [join $vals ,]) +" +do_write_test fts3cov-3.3 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(200, [join $vals ,]) +" +do_write_test fts3cov-3.4 t2_content " + INSERT INTO t2(docid, [join $cols ,]) VALUES(60000, [join $vals ,]) +" + +#------------------------------------------------------------------------- +# If too much data accumulates in the pending-terms hash table, it is +# flushed to the database automatically, even if the transaction has not +# finished. The following tests check the effects of encountering an OOM +# while doing this. +# +do_test fts3cov-4.1 { + execsql { + CREATE VIRTUAL TABLE t3 USING fts3(x); + INSERT INTO t3(t3) VALUES('nodesize=24'); + INSERT INTO t3(t3) VALUES('maxpending=100'); + } +} {} +set DO_MALLOC_TEST 1 +do_write_test fts3cov-4.2 t3_content { + INSERT INTO t3(docid, x) + SELECT 1, 'Then Christabel stretched forth her hand,' UNION ALL + SELECT 3, 'And comforted fair Geraldine:' UNION ALL + SELECT 4, '''O well, bright dame, may you command' UNION ALL + SELECT 5, 'The service of Sir Leoline;' UNION ALL + SELECT 2, 'And gladly our stout chivalry' UNION ALL + SELECT 7, 'Will he send forth, and friends withal,' UNION ALL + SELECT 8, 'To guide and guard you safe and free' UNION ALL + SELECT 6, 'Home to your noble father''s hall.''' +} + +#------------------------------------------------------------------------- +# When building the internal tree structure for each segment b-tree, FTS3 +# assumes that the content of each internal node will be less than +# $nodesize bytes, where $nodesize is the advisory node size. If this turns +# out to be untrue, then an extra buffer must be malloc'd for each term. +# This test case tests these paths and the effects of said mallocs failing +# by inserting insert a document with some fairly large terms into a +# full-text table with a very small node-size. +# +# Test this handling of large terms in three contexts: +# +# 1. When flushing the pending-terms table. +# 2. When optimizing the data structures using the INSERT syntax. +# 2. When optimizing the data structures using the deprecated SELECT syntax. +# +do_test fts3cov-5.1 { + execsql { + CREATE VIRTUAL TABLE t4 USING fts3(x); + INSERT INTO t4(t4) VALUES('nodesize=24'); + } +} {} +set DO_MALLOC_TEST 1 + +# Test when flushing pending-terms table. +do_write_test fts3cov-5.2 t4_content { + INSERT INTO t4 + SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL + SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL + SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL + SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL + SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL + SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC' +} + +# Test when optimizing via INSERT. +do_test fts3cov-5.3 { execsql { INSERT INTO t4 VALUES('extra!') } } {} +do_write_test fts3cov-5.2 t4_segments { INSERT INTO t4(t4) VALUES('optimize') } + +# Test when optimizing via SELECT. +do_test fts3cov-5.5 { execsql { INSERT INTO t4 VALUES('more extra!') } } {} +do_write_test fts3cov-5.6 t4_segments { + SELECT * FROM (SELECT optimize(t4) FROM t4 LIMIT 1) + EXCEPT SELECT 'Index optimized' +} + +#------------------------------------------------------------------------- +# When merging all segments at a given level to create a single segment +# at level+1, FTS3 runs a query of the form: +# +# SELECT count(*) FROM %_segdir WHERE level = ? +# +# The query is compiled the first time this operation is required and +# reused thereafter. This test aims to test the effects of an OOM while +# preparing and executing this query for the first time. +# +# Then, keep inserting rows into the table so that the effects of an OOM +# while re-executing the same query can also be tested. +# +do_test fts3cov-6.1 { + execsql { CREATE VIRTUAL TABLE t5 USING fts3(x) } + for {set i 0} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" } + execsql { SELECT count(*) FROM t5_segdir } +} {16} + +# First time. +db close +sqlite3 db test.db +do_write_test fts3cov-6.2 t5_content { + INSERT INTO t5 VALUES('segment number 16!'); +} + +# Second time. +do_test fts3cov-6.3 { + for {set i 1} {$i<16} {incr i} { execsql "INSERT INTO t5 VALUES('term$i')" } + execsql { SELECT count(*) FROM t5_segdir } +} {17} +do_write_test fts3cov-6.4 t5_content { + INSERT INTO t5 VALUES('segment number 16!'); +} + +#------------------------------------------------------------------------- +# Update the docid of a row. Test this in two scenarios: +# +# 1. When the row being updated is the only row in the table. +# 2. When it is not. +# +# The two cases above take different paths because in case 1 all data +# structures can simply be emptied before inserting the new row record. +# In case 2, the data structures actually have to be updated. +# +do_test fts3cov-7.1 { + execsql { + CREATE VIRTUAL TABLE t7 USING fts3(a, b, c); + INSERT INTO t7 VALUES('A', 'B', 'C'); + UPDATE t7 SET docid = 5; + SELECT docid, * FROM t7; + } +} {5 A B C} +do_test fts3cov-7.2 { + execsql { + INSERT INTO t7 VALUES('D', 'E', 'F'); + UPDATE t7 SET docid = 1 WHERE docid = 6; + SELECT docid, * FROM t7; + } +} {1 D E F 5 A B C} + +#------------------------------------------------------------------------- +# If a set of documents are modified within a transaction, the +# pending-terms table must be flushed each time a document with a docid +# less than or equal to the previous docid is modified. +# +# This test checks the effects of an OOM error occuring when the +# pending-terms table is flushed for this reason as part of a DELETE +# statement. +# +do_malloc_test fts3cov-8 -sqlprep { + BEGIN; + CREATE VIRTUAL TABLE t8 USING fts3; + INSERT INTO t8 VALUES('the output of each batch run'); + INSERT INTO t8 VALUES('(possibly a day''s work)'); + INSERT INTO t8 VALUES('was written to two separate disks'); + COMMIT; +} -sqlbody { + BEGIN; + DELETE FROM t8 WHERE rowid = 3; + DELETE FROM t8 WHERE rowid = 2; + DELETE FROM t8 WHERE rowid = 1; + COMMIT; +} + +#------------------------------------------------------------------------- +# Test some branches in the code that handles "special" inserts like: +# +# INSERT INTO t1(t1) VALUES('optimize'); +# +# Also test that an optimize (INSERT method) works on an empty table. +# +set DO_MALLOC_TEST 0 +do_test fts3cov-9.1 { + execsql { CREATE VIRTUAL TABLE xx USING fts3 } +} {} +do_error_test fts3cov-9.2 { + INSERT INTO xx(xx) VALUES('optimise'); -- British spelling +} {SQL logic error or missing database} +do_error_test fts3cov-9.3 { + INSERT INTO xx(xx) VALUES('short'); +} {SQL logic error or missing database} +do_error_test fts3cov-9.4 { + INSERT INTO xx(xx) VALUES('waytoolongtobecorrect'); +} {SQL logic error or missing database} +do_test fts3cov-9.5 { + execsql { INSERT INTO xx(xx) VALUES('optimize') } +} {} + +#------------------------------------------------------------------------- +# Test that a table can be optimized in the middle of a transaction when +# the pending-terms table is non-empty. This case involves some extra +# branches because data must be read not only from the database, but +# also from the pending-terms table. +# +do_malloc_test fts3cov-10 -sqlprep { + CREATE VIRTUAL TABLE t10 USING fts3; + INSERT INTO t10 VALUES('Optimising images for the web is a tricky business'); + BEGIN; + INSERT INTO t10 VALUES('You have to get the right balance between'); +} -sqlbody { + INSERT INTO t10(t10) VALUES('optimize'); +} + +#------------------------------------------------------------------------- +# Test a full-text query for a term that was once in the index, but is +# no longer. +# +do_test fts3cov-11.1 { + execsql { + CREATE VIRTUAL TABLE xx USING fts3; + INSERT INTO xx VALUES('one two three'); + INSERT INTO xx VALUES('four five six'); + DELETE FROM xx WHERE docid = 1; + } + execsql { SELECT * FROM xx WHERE xx MATCH 'two' } +} {} + + +do_malloc_test fts3cov-12 -sqlprep { + CREATE VIRTUAL TABLE t12 USING fts3; + INSERT INTO t12 VALUES('is one of the two togther'); + BEGIN; + INSERT INTO t12 VALUES('one which was appropriate at the time'); +} -sqlbody { + SELECT * FROM t12 WHERE t12 MATCH 'one' +} + +do_malloc_test fts3cov-13 -sqlprep { + PRAGMA encoding = 'UTF-16'; + CREATE VIRTUAL TABLE t13 USING fts3; + INSERT INTO t13 VALUES('two scalar functions'); + INSERT INTO t13 VALUES('scalar two functions'); + INSERT INTO t13 VALUES('functions scalar two'); +} -sqlbody { + SELECT snippet(t13, '%%', '%%', '#') FROM t13 WHERE t13 MATCH 'two'; + SELECT snippet(t13, '%%', '%%') FROM t13 WHERE t13 MATCH 'two'; + SELECT snippet(t13, '%%') FROM t13 WHERE t13 MATCH 'two'; +} + +finish_test diff --git a/test/fts3expr.test b/test/fts3expr.test index 4cedecb2..e7c4f655 100644 --- a/test/fts3expr.test +++ b/test/fts3expr.test @@ -336,53 +336,53 @@ do_test fts3expr-4.1 { # Mismatched parenthesis: do_test fts3expr-4.2.1 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example AND (hello OR world))' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [example AND (hello OR world))]}} do_test fts3expr-4.2.2 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example AND (hello OR world' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [example AND (hello OR world]}} do_test fts3expr-4.2.3 { catchsql { SELECT * FROM t1 WHERE t1 MATCH '(hello' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [(hello]}} do_test fts3expr-4.2.4 { catchsql { SELECT * FROM t1 WHERE t1 MATCH '(' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [(]}} do_test fts3expr-4.2.5 { catchsql { SELECT * FROM t1 WHERE t1 MATCH ')' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [)]}} do_test fts3expr-4.2.6 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example (hello world' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [example (hello world]}} # Unterminated quotation marks: do_test fts3expr-4.3.1 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example OR "hello world' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [example OR "hello world]}} do_test fts3expr-4.3.2 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'example OR hello world"' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [example OR hello world"]}} # Binary operators without the required operands. do_test fts3expr-4.4.1 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'OR hello world' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [OR hello world]}} do_test fts3expr-4.4.2 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'hello world OR' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [hello world OR]}} do_test fts3expr-4.4.3 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one (hello world OR) two' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [one (hello world OR) two]}} do_test fts3expr-4.4.4 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one (OR hello world) two' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [one (OR hello world) two]}} # NEAR operators with something other than phrases as arguments. do_test fts3expr-4.5.1 { catchsql { SELECT * FROM t1 WHERE t1 MATCH '(hello OR world) NEAR one' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [(hello OR world) NEAR one]}} do_test fts3expr-4.5.2 { catchsql { SELECT * FROM t1 WHERE t1 MATCH 'one NEAR (hello OR world)' } -} {1 {SQL logic error or missing database}} +} {1 {malformed MATCH expression: [one NEAR (hello OR world)]}} #------------------------------------------------------------------------ # The following OOM tests are designed to cover cases in fts3_expr.c. diff --git a/test/fts3malloc.test b/test/fts3malloc.test index 920dc0c1..5e793fbd 100644 --- a/test/fts3malloc.test +++ b/test/fts3malloc.test @@ -33,7 +33,8 @@ set DO_MALLOC_TEST 1 # fts3_malloc-2.*: Test OOM during SELECT operations. # fts3_malloc-3.*: Test OOM during SELECT operations with a larger database. # fts3_malloc-4.*: Test OOM during database write operations. -# +# fts3_malloc-5.*: Test that a couple of memory leaks that could follow +# OOM in tokenizer code have been fixed. # @@ -288,6 +289,16 @@ do_test fts3_malloc-4.2 { execsql { SELECT a FROM ft } } {two four {two four}} +do_write_test fts3_malloc-5.1 ft_content { + INSERT INTO ft VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken', 'cynics!') +} +do_test fts3_malloc-5.2 { + execsql { CREATE VIRTUAL TABLE ft8 USING fts3(x, tokenize porter) } +} {} +do_write_test fts3_malloc-5.3 ft_content { + INSERT INTO ft8 VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken') +} + finish_test diff --git a/test/fts3near.test b/test/fts3near.test index e824133b..9c4409e1 100644 --- a/test/fts3near.test +++ b/test/fts3near.test @@ -76,6 +76,17 @@ do_test fts3near-1.15 { execsql {SELECT docid FROM t1 WHERE content MATCH 'one NEAR two NEAR one'} } {3} +do_test fts3near-1.16 { + execsql { + SELECT docid FROM t1 WHERE content MATCH '"one three" NEAR/0 "four five"' + } +} {1} +do_test fts3near-1.17 { + execsql { + SELECT docid FROM t1 WHERE content MATCH '"four five" NEAR/0 "one three"' + } +} {1} + # Output format of the offsets() function: # @@ -154,6 +165,7 @@ do_test fts3near-3.6 { SELECT offsets(t1) FROM t1 WHERE content MATCH 'three NEAR/0 "two four"' } } {{0 0 8 5 0 1 14 3 0 2 18 4}} +breakpoint do_test fts3near-3.7 { execsql { SELECT offsets(t1) FROM t1 WHERE content MATCH '"two four" NEAR/0 three'} @@ -170,7 +182,7 @@ do_test fts3near-4.1 { execsql { SELECT snippet(t1) FROM t1 WHERE content MATCH 'specification NEAR supports' } -} {{... devices, handheld devices, etc. This specification also supports content positioning, downloadable fonts, ...}} +} {{...braille devices, handheld devices, etc. This specification also supports content positioning, downloadable fonts, table layout...}} do_test fts3near-5.1 { execsql { diff --git a/test/fts3query.test b/test/fts3query.test new file mode 100644 index 00000000..b5af50a7 --- /dev/null +++ b/test/fts3query.test @@ -0,0 +1,105 @@ +# 2009 December 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests of fts3 queries that have been useful during +# the development process as well as some that have been useful in tracking +# down bugs. They are not focused on any particular functionality. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +set DO_MALLOC_TEST 0 + +do_test fts3query-1.1 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts3(x); + BEGIN; + INSERT INTO t1 VALUES('The source code for SQLite is in the public'); + } +} {} + +do_select_test fts3query-1.2 { + SELECT * FROM t1; +} {{The source code for SQLite is in the public}} +do_select_test fts3query-1.3 { + SELECT * FROM t1 WHERE t1 MATCH 'sqlite' +} {{The source code for SQLite is in the public}} + +do_test fts3query-1.4 { execsql {COMMIT} } {} + +do_select_test fts3query-1.5 { + SELECT * FROM t1; +} {{The source code for SQLite is in the public}} +do_select_test fts3query-1.6 { + SELECT * FROM t1 WHERE t1 MATCH 'sqlite' +} {{The source code for SQLite is in the public}} + + +set sqlite_fts3_enable_parentheses 1 +do_test fts3query-2.1 { + execsql { + CREATE VIRTUAL TABLE zoink USING fts3; + INSERT INTO zoink VALUES('The apple falls far from the tree'); + } +} {} +do_test fts3query-2.2 { + execsql { + SELECT docid FROM zoink WHERE zoink MATCH '(apple oranges) AND apple' + } +} {} +do_test fts3query-2.3 { + execsql { + SELECT docid FROM zoink WHERE zoink MATCH 'apple AND (oranges apple)' + } +} {} +set sqlite_fts3_enable_parentheses 0 + +do_test fts3query-3.1 { + execsql { + CREATE VIRTUAL TABLE foobar using FTS3(description, tokenize porter); + INSERT INTO foobar (description) values (' + Filed under: Emerging Technologies, EV/Plug-in, Hybrid, Chevrolet, GM, + ZENN 2011 Chevy Volt - Click above for high-res image gallery There are + 16 days left in the month of December. Besides being time for most + Americans to kick their Christmas shopping sessions into high gear and + start planning their resolutions for 2010, it also means that there''s + precious little time for EEStor to "deliver functional technology" to + Zenn Motors as promised. Still, the promises held out by the secretive + company are too great for us to forget about entirely. We''d love for + EEStor''s claims to be independently verified and proven accurate, as + would just about anyone else looking to break free of petroleum in fav + '); + } +} {} + +do_test fts3query-3.2 { + execsql { SELECT docid FROM foobar WHERE description MATCH '"high sp d"' } +} {} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +do_test fts3query-3.3 { + execsql { SELECT mit(matchinfo(foobar)) FROM foobar WHERE foobar MATCH 'the' } +} {{1 1 3 3 1}} + +finish_test + diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0fb15122..2dcde0f0 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -9,6 +9,72 @@ # Brute force (random data) tests for FTS3. # +#------------------------------------------------------------------------- +# +# The FTS3 tests implemented in this file focus on testing that FTS3 +# returns the correct set of documents for various types of full-text +# query. This is done using pseudo-randomly generated data and queries. +# The expected result of each query is calculated using Tcl code. +# +# 1. The database is initialized to contain a single table with three +# columns. 100 rows are inserted into the table. Each of the three +# values in each row is a document consisting of between 0 and 100 +# terms. Terms are selected from a vocabulary of $G(nVocab) terms. +# +# 2. The following is performed 100 times: +# +# a. A row is inserted into the database. The row contents are +# generated as in step 1. The docid is a pseudo-randomly selected +# value between 0 and 1000000. +# +# b. A psuedo-randomly selected row is updated. One of its columns is +# set to contain a new document generated in the same way as the +# documents in step 1. +# +# c. A psuedo-randomly selected row is deleted. +# +# d. For each of several types of fts3 queries, 10 SELECT queries +# of the form: +# +# SELECT docid FROM WHERE MATCH '' +# +# are evaluated. The results are compared to those calculated by +# Tcl code in this file. The patterns used for the different query +# types are: +# +# 1. query = +# 2. query = +# 3. query = " " +# 4. query = " " +# 5. query = " " +# 6. query = NEAR +# 7. query = NEAR/11 NEAR/11 +# 8. query = OR +# 9. query = NOT +# 10. query = AND +# 11. query = NEAR OR NEAR +# 12. query = NEAR NOT NEAR +# 13. query = NEAR AND NEAR +# +# where is a term psuedo-randomly selected from the vocabulary +# and prefix is the first 2 characters of such a term followed by +# a "*" character. +# +# Every second iteration, steps (a) through (d) above are performed +# within a single transaction. This forces the queries in (d) to +# read data from both the database and the in-memory hash table +# that caches the full-text index entries created by steps (a), (b) +# and (c) until the transaction is committed. +# +# The procedure above is run 5 times, using advisory fts3 node sizes of 50, +# 500, 1000 and 2000 bytes. +# +# After the test using an advisory node-size of 50, an OOM test is run using +# the database. This test is similar to step (d) above, except that it tests +# the effects of transient and persistent OOM conditions encountered while +# executing each query. +# + set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -17,16 +83,22 @@ source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl +set G(nVocab) 100 + set nVocab 100 set lVocab [list] +expr srand(0) + # Generate a vocabulary of nVocab words. Each word is 3 characters long. # set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z} for {set i 0} {$i < $nVocab} {incr i} { + set len [expr int(rand()*3)+2] set word [lindex $lChar [expr int(rand()*26)]] append word [lindex $lChar [expr int(rand()*26)]] - append word [lindex $lChar [expr int(rand()*26)]] + if {$len>2} { append word [lindex $lChar [expr int(rand()*26)]] } + if {$len>3} { append word [lindex $lChar [expr int(rand()*26)]] } lappend lVocab $word } @@ -71,12 +143,59 @@ proc update_row {rowid} { proc simple_phrase {zPrefix} { set ret [list] - set pattern "*[string map {* \[a-z\]} $zPrefix]*" - foreach {key value} [array get ::t1] { - if {[string match $pattern $value]} { lappend ret $key } + + set reg [string map {* {[^ ]*}} $zPrefix] + set reg " $reg " + + foreach key [lsort -integer [array names ::t1]] { + set value $::t1($key) + set cnt [list] + foreach col $value { + if {[regexp $reg " $col "]} { lappend ret $key ; break } + } } - lsort -integer $ret + + #lsort -uniq -integer $ret + set ret } + +# This [proc] is used to test the FTS3 matchinfo() function. +# +proc simple_token_matchinfo {zToken} { + + set nDoc(0) 0 + set nDoc(1) 0 + set nDoc(2) 0 + set nHit(0) 0 + set nHit(1) 0 + set nHit(2) 0 + + + foreach key [array names ::t1] { + set value $::t1($key) + set a($key) [list] + foreach i {0 1 2} col $value { + set hit [llength [lsearch -all $col $zToken]] + lappend a($key) $hit + incr nHit($i) $hit + if {$hit>0} { incr nDoc($i) } + } + } + + set ret [list] + foreach docid [lsort -integer [array names a]] { + if { [lindex [lsort -integer $a($docid)] end] } { + set matchinfo [list 1 3] + foreach i {0 1 2} hit $a($docid) { + lappend matchinfo $hit $nHit($i) $nDoc($i) + } + lappend ret $docid $matchinfo + } + } + + set ret +} + proc simple_near {termlist nNear} { set ret [list] @@ -135,6 +254,14 @@ proc setop_and {A B} { return $ret } +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + set sqlite_fts3_enable_parentheses 1 foreach nodesize {50 500 1000 2000} { @@ -144,11 +271,20 @@ foreach nodesize {50 500 1000 2000} { # db transaction { catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c, test:$nodesize)" + execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" + execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" for {set i 0} {$i < 100} {incr i} { insert_row $i } } for {set iTest 1} {$iTest <= 100} {incr iTest} { + catchsql COMMIT + + set DO_MALLOC_TEST 0 + set nRep 10 + if {$iTest==100 && $nodesize==50} { + set DO_MALLOC_TEST 1 + set nRep 2 + } # Delete one row, update one row and insert one row. # @@ -163,11 +299,11 @@ foreach nodesize {50 500 1000 2000} { while {[info exists ::t1($iInsert)]} { set iInsert [expr {int(rand()*1000000)}] } - db transaction { + execsql BEGIN insert_row $iInsert update_row $iUpdate delete_row $iDelete - } + if {0==($iTest%2)} { execsql COMMIT } # Pick 10 terms from the vocabulary. Check that the results of querying # the database for the set of documents containing each of these terms @@ -176,75 +312,75 @@ foreach nodesize {50 500 1000 2000} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_test fts3rnd-1.$nodesize.$iTest.1.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $term } - } [simple_phrase $term] + do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term + } [simple_token_matchinfo $term] } # This time, use the first two characters of each term as a term prefix # to query for. Test that querying the Tcl array produces the same results # as querying the FTS3 table for the prefix. # - for {set i 0} {$i < 10} {incr i} { - set prefix [string range [random_term] 0 1] + for {set i 0} {$i < $nRep} {incr i} { + set prefix [string range [random_term] 0 end-1] set match "${prefix}*" - do_test fts3rnd-1.$nodesize.$iTest.2.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $match] } # Similar to the above, except for phrase queries. # - for {set i 0} {$i < 10} {incr i} { + for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" - do_test fts3rnd-1.$nodesize.$iTest.3.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } # Three word phrases. # - for {set i 0} {$i < 10} {incr i} { + for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" - do_test fts3rnd-1.$nodesize.$iTest.4.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } # Three word phrases made up of term-prefixes. # - for {set i 0} {$i < 10} {incr i} { - set query "[string range [random_term] 0 1]* " - append query "[string range [random_term] 0 1]* " - append query "[string range [random_term] 0 1]*" + for {set i 0} {$i < $nRep} {incr i} { + set query "[string range [random_term] 0 end-1]* " + append query "[string range [random_term] 0 end-1]* " + append query "[string range [random_term] 0 end-1]*" set match "\"$query\"" - do_test fts3rnd-1.$nodesize.$iTest.5.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $query] } # A NEAR query with terms as the arguments. # - for {set i 0} {$i < 10} {incr i} { + for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] - do_test fts3rnd-1.$nodesize.$iTest.6.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms 10] } # A 3-way NEAR query with terms as the arguments. # - for {set i 0} {$i < 10} {incr i} { + for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] set fts3 [execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }] - do_test fts3rnd-1.$nodesize.$iTest.7.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms $nNear] } @@ -255,12 +391,12 @@ foreach nodesize {50 500 1000 2000} { 9 NOT setop_not 10 AND setop_and } { - for {set i 0} {$i < 10} {incr i} { + for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" - do_test fts3rnd-1.$nodesize.$iTest.$tn.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc [simple_phrase $term1] [simple_phrase $term2]] } } @@ -272,20 +408,22 @@ foreach nodesize {50 500 1000 2000} { 9 NOT setop_not 10 AND setop_and } { - for {set i 0} {$i < 10} {incr i} { + for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term] set term2 [random_term] set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_test fts3rnd-1.$nodesize.$iTest.$tn.$i { - execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } + do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc \ [simple_near [list $term1 $term2] 10] \ [simple_near [list $term3 $term4] 10] ] } } + + catchsql COMMIT } } diff --git a/test/fts3snippet.test b/test/fts3snippet.test new file mode 100644 index 00000000..e7adaee0 --- /dev/null +++ b/test/fts3snippet.test @@ -0,0 +1,465 @@ +# 2010 January 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# The tests in this file test the FTS3 auxillary functions offsets(), +# snippet() and matchinfo() work. At time of writing, running this file +# provides full coverage of fts3_snippet.c. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is not defined, omit this file. +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set sqlite_fts3_enable_parentheses 1 +set DO_MALLOC_TEST 0 + +# Transform the list $L to its "normal" form. So that it can be compared to +# another list with the same set of elements using [string compare]. +# +proc normalize {L} { + set ret [list] + foreach l $L {lappend ret $l} + return $ret +} + +proc do_offsets_test {name expr args} { + set result [list] + foreach a $args { + lappend result [normalize $a] + } + do_select_test $name { + SELECT offsets(ft) FROM ft WHERE ft MATCH $expr + } $result +} + +# Document text used by a few tests. Contains the English names of all +# integers between 1 and 300. +# +set numbers [normalize { + one two three four five six seven eight nine ten eleven twelve thirteen + fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone + twentytwo twentythree twentyfour twentyfive twentysix twentyseven + twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour + thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone + fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight + fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix + fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour + sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone + seventytwo seventythree seventyfour seventyfive seventysix seventyseven + seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour + eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone + ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven + ninetyeight ninetynine onehundred onehundredone onehundredtwo + onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven + onehundredeight onehundrednine onehundredten onehundredeleven + onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen + onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen + onehundredtwenty onehundredtwentyone onehundredtwentytwo + onehundredtwentythree onehundredtwentyfour onehundredtwentyfive + onehundredtwentysix onehundredtwentyseven onehundredtwentyeight + onehundredtwentynine onehundredthirty onehundredthirtyone + onehundredthirtytwo onehundredthirtythree onehundredthirtyfour + onehundredthirtyfive onehundredthirtysix onehundredthirtyseven + onehundredthirtyeight onehundredthirtynine onehundredforty + onehundredfortyone onehundredfortytwo onehundredfortythree + onehundredfortyfour onehundredfortyfive onehundredfortysix + onehundredfortyseven onehundredfortyeight onehundredfortynine + onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree + onehundredfiftyfour onehundredfiftyfive onehundredfiftysix + onehundredfiftyseven onehundredfiftyeight onehundredfiftynine + onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree + onehundredsixtyfour onehundredsixtyfive onehundredsixtysix + onehundredsixtyseven onehundredsixtyeight onehundredsixtynine + onehundredseventy onehundredseventyone onehundredseventytwo + onehundredseventythree onehundredseventyfour onehundredseventyfive + onehundredseventysix onehundredseventyseven onehundredseventyeight + onehundredseventynine onehundredeighty onehundredeightyone + onehundredeightytwo onehundredeightythree onehundredeightyfour + onehundredeightyfive onehundredeightysix onehundredeightyseven + onehundredeightyeight onehundredeightynine onehundredninety + onehundredninetyone onehundredninetytwo onehundredninetythree + onehundredninetyfour onehundredninetyfive onehundredninetysix + onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred + twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive + twohundredsix twohundredseven twohundredeight twohundrednine twohundredten + twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen + twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen + twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo + twohundredtwentythree twohundredtwentyfour twohundredtwentyfive + twohundredtwentysix twohundredtwentyseven twohundredtwentyeight + twohundredtwentynine twohundredthirty twohundredthirtyone + twohundredthirtytwo twohundredthirtythree twohundredthirtyfour + twohundredthirtyfive twohundredthirtysix twohundredthirtyseven + twohundredthirtyeight twohundredthirtynine twohundredforty + twohundredfortyone twohundredfortytwo twohundredfortythree + twohundredfortyfour twohundredfortyfive twohundredfortysix + twohundredfortyseven twohundredfortyeight twohundredfortynine + twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree + twohundredfiftyfour twohundredfiftyfive twohundredfiftysix + twohundredfiftyseven twohundredfiftyeight twohundredfiftynine + twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree + twohundredsixtyfour twohundredsixtyfive twohundredsixtysix + twohundredsixtyseven twohundredsixtyeight twohundredsixtynine + twohundredseventy twohundredseventyone twohundredseventytwo + twohundredseventythree twohundredseventyfour twohundredseventyfive + twohundredseventysix twohundredseventyseven twohundredseventyeight + twohundredseventynine twohundredeighty twohundredeightyone + twohundredeightytwo twohundredeightythree twohundredeightyfour + twohundredeightyfive twohundredeightysix twohundredeightyseven + twohundredeightyeight twohundredeightynine twohundredninety + twohundredninetyone twohundredninetytwo twohundredninetythree + twohundredninetyfour twohundredninetyfive twohundredninetysix + twohundredninetyseven twohundredninetyeight twohundredninetynine + threehundred +}] + +foreach {DO_MALLOC_TEST enc} { + 0 utf8 + 1 utf8 + 1 utf16 +} { + + db close + file delete -force test.db + sqlite3 db test.db + sqlite3_db_config_lookaside db 0 0 0 + db eval "PRAGMA encoding = \"$enc\"" + + # Set variable $T to the test name prefix for this iteration of the loop. + # + set T "fts3snippet-$enc" + + ########################################################################## + # Test the offset function. + # + do_test $T.1.1 { + execsql { + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('xxx xxx xxx xxx'); + } + } {} + do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3} + do_offsets_test $T.1.3 {"xxx xxx"} { + 0 0 0 3 0 0 4 3 0 1 4 3 0 0 8 3 + 0 1 8 3 0 1 12 3 + } + do_offsets_test $T.1.4 {"xxx xxx" xxx} { + 0 0 0 3 0 2 0 3 0 0 4 3 0 1 4 3 + 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3 + 0 1 12 3 0 2 12 3 + } + do_offsets_test $T.1.5 {xxx "xxx xxx"} { + 0 0 0 3 0 1 0 3 0 0 4 3 0 1 4 3 + 0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3 + 0 0 12 3 0 2 12 3 + } + + do_test $T.2.1 { + set v1 [lrange $numbers 0 99] + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3(a, b); + INSERT INTO ft VALUES($v1, $numbers); + INSERT INTO ft VALUES($v1, NULL); + } + } {} + + set off [string first "twohundred " $numbers] + do_offsets_test $T.2.1 {twohundred} [list 1 0 $off 10] + + set off [string first "onehundred " $numbers] + do_offsets_test $T.2.2 {onehundred} \ + [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10] + + # Test a corruption case: + execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers } + do_error_test $T.2.3 { + SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred' + } {database disk image is malformed} + + ########################################################################## + # Test the snippet function. + # + proc do_snippet_test {name expr iCol nTok args} { + set res [list] + foreach a $args { lappend res [string trim $a] } + do_select_test $name { + SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr + } $res + } + do_test $T.3.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('one two three four five six seven eight nine ten'); + } + } {} + do_snippet_test $T.3.2 one 0 5 "{one} two three four five..." + do_snippet_test $T.3.3 two 0 5 "one {two} three four five..." + do_snippet_test $T.3.4 three 0 5 "one two {three} four five..." + do_snippet_test $T.3.5 four 0 5 "...two three {four} five six..." + do_snippet_test $T.3.6 five 0 5 "...three four {five} six seven..." + do_snippet_test $T.3.7 six 0 5 "...four five {six} seven eight..." + do_snippet_test $T.3.8 seven 0 5 "...five six {seven} eight nine..." + do_snippet_test $T.3.9 eight 0 5 "...six seven {eight} nine ten" + do_snippet_test $T.3.10 nine 0 5 "...six seven eight {nine} ten" + do_snippet_test $T.3.11 ten 0 5 "...six seven eight nine {ten}" + + do_test $T.4.1 { + execsql { + INSERT INTO ft VALUES( + 'one two three four five ' + || 'six seven eight nine ten ' + || 'eleven twelve thirteen fourteen fifteen ' + || 'sixteen seventeen eighteen nineteen twenty ' + || 'one two three four five ' + || 'six seven eight nine ten ' + || 'eleven twelve thirteen fourteen fifteen ' + || 'sixteen seventeen eighteen nineteen twenty' + ); + } + } {} + + do_snippet_test $T.4.2 {one nine} 0 5 { + {one} two three...eight {nine} ten + } { + {one} two three...eight {nine} ten... + } + + do_snippet_test $T.4.3 {one nine} 0 -5 { + {one} two three four five...six seven eight {nine} ten + } { + {one} two three four five...seven eight {nine} ten eleven... + } + do_snippet_test $T.4.3 {one nineteen} 0 -5 { + ...eighteen {nineteen} twenty {one} two... + } + do_snippet_test $T.4.4 {two nineteen} 0 -5 { + ...eighteen {nineteen} twenty one {two}... + } + do_snippet_test $T.4.5 {three nineteen} 0 -5 { + ...{nineteen} twenty one two {three}... + } + + do_snippet_test $T.4.6 {four nineteen} 0 -5 { + ...two three {four} five six...seventeen eighteen {nineteen} twenty one... + } + do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 { + ...seventeen eighteen {nineteen} twenty one...two three {four} five six... + } + + do_snippet_test $T.4.8 {four nineteen} 0 5 { + ...three {four} five...eighteen {nineteen} twenty... + } + do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 { + ...eighteen {nineteen} twenty...three {four} five... + } + do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 { + ...seventeen eighteen {nineteen} twenty one...two three {four} five six... + } + do_snippet_test $T.4.11 {four NOT (nineteen twentyone)} 0 5 { + ...two three {four} five six... + } { + ...two three {four} five six... + } + do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 { + ...two three {four} five six... + } { + ...two three {four} five six... + } + + do_test $T.5.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3(a, b, c); + INSERT INTO ft VALUES( + 'one two three four five', + 'four five six seven eight', + 'seven eight nine ten eleven' + ); + } + } {} + + do_snippet_test $T.5.2 {five} -1 3 {...three four {five}} + do_snippet_test $T.5.3 {five} 0 3 {...three four {five}} + do_snippet_test $T.5.4 {five} 1 3 {four {five} six...} + do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...} + + do_test $T.5.6 { + execsql { UPDATE ft SET b = NULL } + } {} + + do_snippet_test $T.5.7 {five} -1 3 {...three four {five}} + do_snippet_test $T.5.8 {five} 0 3 {...three four {five}} + do_snippet_test $T.5.9 {five} 1 3 {} + do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...} + + do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 { + {one} two three...{seven} {eight} {nine}... + } + + do_test $T.6.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3(x); + INSERT INTO ft VALUES($numbers); + } + } {} + do_snippet_test $T.6.2 { + one fifty onehundred onehundredfifty twohundredfifty threehundred + } -1 4 { + {one}...{fifty}...{onehundred}...{onehundredfifty}... + } + do_snippet_test $T.6.3 { + one fifty onehundred onehundredfifty twohundredfifty threehundred + } -1 -4 { + {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone... + } + + do_test $T.7.1 { + execsql { + BEGIN; + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3(x); + } + set testresults [list] + for {set i 1} {$i < 150} {incr i} { + set commas [string repeat , $i] + execsql {INSERT INTO ft VALUES('one' || $commas || 'two')} + lappend testresults "{one}$commas{two}" + } + execsql COMMIT + } {} + eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults + + ########################################################################## + # Test the matchinfo function. + # + proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r + } + db func mit mit + proc do_matchinfo_test {name expr args} { + set res [list] + foreach a $args { lappend res [normalize $a] } + do_select_test $name { + SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr + } $res + } + do_test $T.8.1 { + set ten {one two three four five six seven eight nine ten} + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES($ten); + INSERT INTO ft VALUES($ten || ' ' || $ten); + } + } {} + + do_matchinfo_test $T.8.2 "one" {1 1 1 3 2} {1 1 2 3 2} + do_matchinfo_test $T.8.3 "one NEAR/3 ten" {2 1 1 1 1 1 1 1} + do_matchinfo_test $T.8.4 "five NEAR/4 ten" \ + {2 1 1 3 2 1 3 2} {2 1 2 3 2 2 3 2} + do_matchinfo_test $T.8.5 "six NEAR/3 ten NEAR/3 two" \ + {3 1 1 1 1 1 1 1 1 1 1} + do_matchinfo_test $T.8.6 "five NEAR/4 ten NEAR/3 two" \ + {3 1 2 2 1 1 1 1 1 1 1} + + do_test $T.9.1 { + execsql { + DROP TABLE IF EXISTS ft; + CREATE VIRTUAL TABLE ft USING fts3(x, y); + } + foreach n {1 2 3} { + set v1 [lrange $numbers 0 [expr $n*100]] + set v2 [string trim [string repeat "$numbers " $n]] + set docid [expr $n * 1000000] + execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) } + } + } {} + do_matchinfo_test $T.9.2 {two*} \ + { 1 2 1 105 3 101 606 3} \ + { 1 2 3 105 3 202 606 3} \ + { 1 2 101 105 3 303 606 3} + + do_matchinfo_test $T.9.4 {"one* two*"} \ + { 1 2 1 5 3 2 12 3} \ + { 1 2 2 5 3 4 12 3} \ + { 1 2 2 5 3 6 12 3} + + do_matchinfo_test $T.9.5 {twohundredfifty} \ + { 1 2 0 1 1 1 6 3} \ + { 1 2 0 1 1 2 6 3} \ + { 1 2 1 1 1 3 6 3} + + do_matchinfo_test $T.9.6 {"threehundred one"} \ + { 1 2 0 0 0 1 3 2} \ + { 1 2 0 0 0 2 3 2} + + do_matchinfo_test $T.9.7 {one OR fivehundred} \ + { 2 2 1 3 3 1 6 3 0 0 0 0 0 0 } \ + { 2 2 1 3 3 2 6 3 0 0 0 0 0 0 } \ + { 2 2 1 3 3 3 6 3 0 0 0 0 0 0 } + + do_matchinfo_test $T.9.8 {two OR "threehundred one"} \ + { 2 2 1 3 3 1 6 3 0 0 0 0 3 2 } \ + { 2 2 1 3 3 2 6 3 0 0 0 1 3 2 } \ + { 2 2 1 3 3 3 6 3 0 0 0 2 3 2 } + + do_select_test $T.9.9 { + SELECT mit(matchinfo(ft)), mit(matchinfo(ft)) + FROM ft WHERE ft MATCH 'two OR "threehundred one"' + } [normalize { + {2 2 1 3 3 1 6 3 0 0 0 0 3 2} + {2 2 1 3 3 1 6 3 0 0 0 0 3 2} + {2 2 1 3 3 2 6 3 0 0 0 1 3 2} + {2 2 1 3 3 2 6 3 0 0 0 1 3 2} + {2 2 1 3 3 3 6 3 0 0 0 2 3 2} + {2 2 1 3 3 3 6 3 0 0 0 2 3 2} + }] + + # EVIDENCE-OF: R-56101-59725 If used within a SELECT that uses the + # "query by rowid" or "linear scan" strategies, then the snippet and + # offsets both return an an empty string, and the matchinfo function + # returns a blob value zero bytes in size. + # + set r 1000000 ;# A rowid that exists in table ft + do_select_test $T.10.0 { SELECT rowid FROM ft WHERE rowid = $r } $r + do_select_test $T.10.1 { + SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft; + } {0 text 0 text 0 text} + do_select_test $T.10.2 { + SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft WHERE rowid = $r + } {0 text} + do_select_test $T.10.3 { + SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft; + } {0 text 0 text 0 text} + do_select_test $T.10.4 { + SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft WHERE rowid = $r; + } {0 text} + do_select_test $T.10.5 { + SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft; + } {0 blob 0 blob 0 blob} + do_select_test $T.10.6 { + SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r + } {0 blob} +} + + +set sqlite_fts3_enable_parentheses 0 +finish_test diff --git a/test/fts4aa.test b/test/fts4aa.test new file mode 100644 index 00000000..c6fa3b9c --- /dev/null +++ b/test/fts4aa.test @@ -0,0 +1,1636 @@ +# 2010 February 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} +if {[db eval {SELECT sqlite_compileoption_used('ENABLE_FTS4')}]==0} { + finish_test + return +} + +do_test fts4aa-1.0 { +db eval { +CREATE VIRTUAL TABLE t1 USING fts4(words, tokenize porter); +BEGIN TRANSACTION; +INSERT INTO t1(docid,words) VALUES(1001001,'In the beginning God created the heaven and the earth.'); +INSERT INTO t1(docid,words) VALUES(1001002,'And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.'); +INSERT INTO t1(docid,words) VALUES(1001003,'And God said, Let there be light: and there was light.'); +INSERT INTO t1(docid,words) VALUES(1001004,'And God saw the light, that it was good: and God divided the light from the darkness.'); +INSERT INTO t1(docid,words) VALUES(1001005,'And God called the light Day, and the darkness he called Night. And the evening and the morning were the first day.'); +INSERT INTO t1(docid,words) VALUES(1001006,'And God said, Let there be a firmament in the midst of the waters, and let it divide the waters from the waters.'); +INSERT INTO t1(docid,words) VALUES(1001007,'And God made the firmament, and divided the waters which were under the firmament from the waters which were above the firmament: and it was so.'); +INSERT INTO t1(docid,words) VALUES(1001008,'And God called the firmament Heaven. And the evening and the morning were the second day.'); +INSERT INTO t1(docid,words) VALUES(1001009,'And God said, Let the waters under the heaven be gathered together unto one place, and let the dry land appear: and it was so.'); +INSERT INTO t1(docid,words) VALUES(1001010,'And God called the dry land Earth; and the gathering together of the waters called he Seas: and God saw that it was good.'); +INSERT INTO t1(docid,words) VALUES(1001011,'And God said, Let the earth bring forth grass, the herb yielding seed, and the fruit tree yielding fruit after his kind, whose seed is in itself, upon the earth: and it was so.'); +INSERT INTO t1(docid,words) VALUES(1001012,'And the earth brought forth grass, and herb yielding seed after his kind, and the tree yielding fruit, whose seed was in itself, after his kind: and God saw that it was good.'); +INSERT INTO t1(docid,words) VALUES(1001013,'And the evening and the morning were the third day.'); +INSERT INTO t1(docid,words) VALUES(1001014,'And God said, Let there be lights in the firmament of the heaven to divide the day from the night; and let them be for signs, and for seasons, and for days, and years:'); +INSERT INTO t1(docid,words) VALUES(1001015,'And let them be for lights in the firmament of the heaven to give light upon the earth: and it was so.'); +INSERT INTO t1(docid,words) VALUES(1001016,'And God made two great lights; the greater light to rule the day, and the lesser light to rule the night: he made the stars also.'); +INSERT INTO t1(docid,words) VALUES(1001017,'And God set them in the firmament of the heaven to give light upon the earth,'); +INSERT INTO t1(docid,words) VALUES(1001018,'And to rule over the day and over the night, and to divide the light from the darkness: and God saw that it was good.'); +INSERT INTO t1(docid,words) VALUES(1001019,'And the evening and the morning were the fourth day.'); +INSERT INTO t1(docid,words) VALUES(1001020,'And God said, Let the waters bring forth abundantly the moving creature that hath life, and fowl that may fly above the earth in the open firmament of heaven.'); +INSERT INTO t1(docid,words) VALUES(1001021,'And God created great whales, and every living creature that moveth, which the waters brought forth abundantly, after their kind, and every winged fowl after his kind: and God saw that it was good.'); +INSERT INTO t1(docid,words) VALUES(1001022,'And God blessed them, saying, Be fruitful, and multiply, and fill the waters in the seas, and let fowl multiply in the earth.'); +INSERT INTO t1(docid,words) VALUES(1001023,'And the evening and the morning were the fifth day.'); +INSERT INTO t1(docid,words) VALUES(1001024,'And God said, Let the earth bring forth the living creature after his kind, cattle, and creeping thing, and beast of the earth after his kind: and it was so.'); +INSERT INTO t1(docid,words) VALUES(1001025,'And God made the beast of the earth after his kind, and cattle after their kind, and every thing that creepeth upon the earth after his kind: and God saw that it was good.'); +INSERT INTO t1(docid,words) VALUES(1001026,'And God said, Let us make man in our image, after our likeness: and let them have dominion over the fish of the sea, and over the fowl of the air, and over the cattle, and over all the earth, and over every creeping thing that creepeth upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1001027,'So God created man in his own image, in the image of God created he him; male and female created he them.'); +INSERT INTO t1(docid,words) VALUES(1001028,'And God blessed them, and God said unto them, Be fruitful, and multiply, and replenish the earth, and subdue it: and have dominion over the fish of the sea, and over the fowl of the air, and over every living thing that moveth upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1001029,'And God said, Behold, I have given you every herb bearing seed, which is upon the face of all the earth, and every tree, in the which is the fruit of a tree yielding seed; to you it shall be for meat.'); +INSERT INTO t1(docid,words) VALUES(1001030,'And to every beast of the earth, and to every fowl of the air, and to every thing that creepeth upon the earth, wherein there is life, I have given every green herb for meat: and it was so.'); +INSERT INTO t1(docid,words) VALUES(1001031,'And God saw every thing that he had made, and, behold, it was very good. And the evening and the morning were the sixth day.'); +INSERT INTO t1(docid,words) VALUES(1002001,'Thus the heavens and the earth were finished, and all the host of them.'); +INSERT INTO t1(docid,words) VALUES(1002002,'And on the seventh day God ended his work which he had made; and he rested on the seventh day from all his work which he had made.'); +INSERT INTO t1(docid,words) VALUES(1002003,'And God blessed the seventh day, and sanctified it: because that in it he had rested from all his work which God created and made.'); +INSERT INTO t1(docid,words) VALUES(1002004,'These are the generations of the heavens and of the earth when they were created, in the day that the LORD God made the earth and the heavens,'); +INSERT INTO t1(docid,words) VALUES(1002005,'And every plant of the field before it was in the earth, and every herb of the field before it grew: for the LORD God had not caused it to rain upon the earth, and there was not a man to till the ground.'); +INSERT INTO t1(docid,words) VALUES(1002006,'But there went up a mist from the earth, and watered the whole face of the ground.'); +INSERT INTO t1(docid,words) VALUES(1002007,'And the LORD God formed man of the dust of the ground, and breathed into his nostrils the breath of life; and man became a living soul.'); +INSERT INTO t1(docid,words) VALUES(1002008,'And the LORD God planted a garden eastward in Eden; and there he put the man whom he had formed.'); +INSERT INTO t1(docid,words) VALUES(1002009,'And out of the ground made the LORD God to grow every tree that is pleasant to the sight, and good for food; the tree of life also in the midst of the garden, and the tree of knowledge of good and evil.'); +INSERT INTO t1(docid,words) VALUES(1002010,'And a river went out of Eden to water the garden; and from thence it was parted, and became into four heads.'); +INSERT INTO t1(docid,words) VALUES(1002011,'The name of the first is Pison: that is it which compasseth the whole land of Havilah, where there is gold;'); +INSERT INTO t1(docid,words) VALUES(1002012,'And the gold of that land is good: there is bdellium and the onyx stone.'); +INSERT INTO t1(docid,words) VALUES(1002013,'And the name of the second river is Gihon: the same is it that compasseth the whole land of Ethiopia.'); +INSERT INTO t1(docid,words) VALUES(1002014,'And the name of the third river is Hiddekel: that is it which goeth toward the east of Assyria. And the fourth river is Euphrates.'); +INSERT INTO t1(docid,words) VALUES(1002015,'And the LORD God took the man, and put him into the garden of Eden to dress it and to keep it.'); +INSERT INTO t1(docid,words) VALUES(1002016,'And the LORD God commanded the man, saying, Of every tree of the garden thou mayest freely eat:'); +INSERT INTO t1(docid,words) VALUES(1002017,'But of the tree of the knowledge of good and evil, thou shalt not eat of it: for in the day that thou eatest thereof thou shalt surely die.'); +INSERT INTO t1(docid,words) VALUES(1002018,'And the LORD God said, It is not good that the man should be alone; I will make him an help meet for him.'); +INSERT INTO t1(docid,words) VALUES(1002019,'And out of the ground the LORD God formed every beast of the field, and every fowl of the air; and brought them unto Adam to see what he would call them: and whatsoever Adam called every living creature, that was the name thereof.'); +INSERT INTO t1(docid,words) VALUES(1002020,'And Adam gave names to all cattle, and to the fowl of the air, and to every beast of the field; but for Adam there was not found an help meet for him.'); +INSERT INTO t1(docid,words) VALUES(1002021,'And the LORD God caused a deep sleep to fall upon Adam, and he slept: and he took one of his ribs, and closed up the flesh instead thereof;'); +INSERT INTO t1(docid,words) VALUES(1002022,'And the rib, which the LORD God had taken from man, made he a woman, and brought her unto the man.'); +INSERT INTO t1(docid,words) VALUES(1002023,'And Adam said, This is now bone of my bones, and flesh of my flesh: she shall be called Woman, because she was taken out of Man.'); +INSERT INTO t1(docid,words) VALUES(1002024,'Therefore shall a man leave his father and his mother, and shall cleave unto his wife: and they shall be one flesh.'); +INSERT INTO t1(docid,words) VALUES(1002025,'And they were both naked, the man and his wife, and were not ashamed.'); +INSERT INTO t1(docid,words) VALUES(1003001,'Now the serpent was more subtil than any beast of the field which the LORD God had made. And he said unto the woman, Yea, hath God said, Ye shall not eat of every tree of the garden?'); +INSERT INTO t1(docid,words) VALUES(1003002,'And the woman said unto the serpent, We may eat of the fruit of the trees of the garden:'); +INSERT INTO t1(docid,words) VALUES(1003003,'But of the fruit of the tree which is in the midst of the garden, God hath said, Ye shall not eat of it, neither shall ye touch it, lest ye die.'); +INSERT INTO t1(docid,words) VALUES(1003004,'And the serpent said unto the woman, Ye shall not surely die:'); +INSERT INTO t1(docid,words) VALUES(1003005,'For God doth know that in the day ye eat thereof, then your eyes shall be opened, and ye shall be as gods, knowing good and evil.'); +INSERT INTO t1(docid,words) VALUES(1003006,'And when the woman saw that the tree was good for food, and that it was pleasant to the eyes, and a tree to be desired to make one wise, she took of the fruit thereof, and did eat, and gave also unto her husband with her; and he did eat.'); +INSERT INTO t1(docid,words) VALUES(1003007,'And the eyes of them both were opened, and they knew that they were naked; and they sewed fig leaves together, and made themselves aprons.'); +INSERT INTO t1(docid,words) VALUES(1003008,'And they heard the voice of the LORD God walking in the garden in the cool of the day: and Adam and his wife hid themselves from the presence of the LORD God amongst the trees of the garden.'); +INSERT INTO t1(docid,words) VALUES(1003009,'And the LORD God called unto Adam, and said unto him, Where art thou?'); +INSERT INTO t1(docid,words) VALUES(1003010,'And he said, I heard thy voice in the garden, and I was afraid, because I was naked; and I hid myself.'); +INSERT INTO t1(docid,words) VALUES(1003011,'And he said, Who told thee that thou wast naked? Hast thou eaten of the tree, whereof I commanded thee that thou shouldest not eat?'); +INSERT INTO t1(docid,words) VALUES(1003012,'And the man said, The woman whom thou gavest to be with me, she gave me of the tree, and I did eat.'); +INSERT INTO t1(docid,words) VALUES(1003013,'And the LORD God said unto the woman, What is this that thou hast done? And the woman said, The serpent beguiled me, and I did eat.'); +INSERT INTO t1(docid,words) VALUES(1003014,'And the LORD God said unto the serpent, Because thou hast done this, thou art cursed above all cattle, and above every beast of the field; upon thy belly shalt thou go, and dust shalt thou eat all the days of thy life:'); +INSERT INTO t1(docid,words) VALUES(1003015,'And I will put enmity between thee and the woman, and between thy seed and her seed; it shall bruise thy head, and thou shalt bruise his heel.'); +INSERT INTO t1(docid,words) VALUES(1003016,'Unto the woman he said, I will greatly multiply thy sorrow and thy conception; in sorrow thou shalt bring forth children; and thy desire shall be to thy husband, and he shall rule over thee.'); +INSERT INTO t1(docid,words) VALUES(1003017,'And unto Adam he said, Because thou hast hearkened unto the voice of thy wife, and hast eaten of the tree, of which I commanded thee, saying, Thou shalt not eat of it: cursed is the ground for thy sake; in sorrow shalt thou eat of it all the days of thy life;'); +INSERT INTO t1(docid,words) VALUES(1003018,'Thorns also and thistles shall it bring forth to thee; and thou shalt eat the herb of the field;'); +INSERT INTO t1(docid,words) VALUES(1003019,'In the sweat of thy face shalt thou eat bread, till thou return unto the ground; for out of it wast thou taken: for dust thou art, and unto dust shalt thou return.'); +INSERT INTO t1(docid,words) VALUES(1003020,'And Adam called his wife''s name Eve; because she was the mother of all living.'); +INSERT INTO t1(docid,words) VALUES(1003021,'Unto Adam also and to his wife did the LORD God make coats of skins, and clothed them.'); +INSERT INTO t1(docid,words) VALUES(1003022,'And the LORD God said, Behold, the man is become as one of us, to know good and evil: and now, lest he put forth his hand, and take also of the tree of life, and eat, and live for ever:'); +INSERT INTO t1(docid,words) VALUES(1003023,'Therefore the LORD God sent him forth from the garden of Eden, to till the ground from whence he was taken.'); +INSERT INTO t1(docid,words) VALUES(1003024,'So he drove out the man; and he placed at the east of the garden of Eden Cherubims, and a flaming sword which turned every way, to keep the way of the tree of life.'); +INSERT INTO t1(docid,words) VALUES(1004001,'And Adam knew Eve his wife; and she conceived, and bare Cain, and said, I have gotten a man from the LORD.'); +INSERT INTO t1(docid,words) VALUES(1004002,'And she again bare his brother Abel. And Abel was a keeper of sheep, but Cain was a tiller of the ground.'); +INSERT INTO t1(docid,words) VALUES(1004003,'And in process of time it came to pass, that Cain brought of the fruit of the ground an offering unto the LORD.'); +INSERT INTO t1(docid,words) VALUES(1004004,'And Abel, he also brought of the firstlings of his flock and of the fat thereof. And the LORD had respect unto Abel and to his offering:'); +INSERT INTO t1(docid,words) VALUES(1004005,'But unto Cain and to his offering he had not respect. And Cain was very wroth, and his countenance fell.'); +INSERT INTO t1(docid,words) VALUES(1004006,'And the LORD said unto Cain, Why art thou wroth? and why is thy countenance fallen?'); +INSERT INTO t1(docid,words) VALUES(1004007,'If thou doest well, shalt thou not be accepted? and if thou doest not well, sin lieth at the door. And unto thee shall be his desire, and thou shalt rule over him.'); +INSERT INTO t1(docid,words) VALUES(1004008,'And Cain talked with Abel his brother: and it came to pass, when they were in the field, that Cain rose up against Abel his brother, and slew him.'); +INSERT INTO t1(docid,words) VALUES(1004009,'And the LORD said unto Cain, Where is Abel thy brother? And he said, I know not: Am I my brother''s keeper?'); +INSERT INTO t1(docid,words) VALUES(1004010,'And he said, What hast thou done? the voice of thy brother''s blood crieth unto me from the ground.'); +INSERT INTO t1(docid,words) VALUES(1004011,'And now art thou cursed from the earth, which hath opened her mouth to receive thy brother''s blood from thy hand;'); +INSERT INTO t1(docid,words) VALUES(1004012,'When thou tillest the ground, it shall not henceforth yield unto thee her strength; a fugitive and a vagabond shalt thou be in the earth.'); +INSERT INTO t1(docid,words) VALUES(1004013,'And Cain said unto the LORD, My punishment is greater than I can bear.'); +INSERT INTO t1(docid,words) VALUES(1004014,'Behold, thou hast driven me out this day from the face of the earth; and from thy face shall I be hid; and I shall be a fugitive and a vagabond in the earth; and it shall come to pass, that every one that findeth me shall slay me.'); +INSERT INTO t1(docid,words) VALUES(1004015,'And the LORD said unto him, Therefore whosoever slayeth Cain, vengeance shall be taken on him sevenfold. And the LORD set a mark upon Cain, lest any finding him should kill him.'); +INSERT INTO t1(docid,words) VALUES(1004016,'And Cain went out from the presence of the LORD, and dwelt in the land of Nod, on the east of Eden.'); +INSERT INTO t1(docid,words) VALUES(1004017,'And Cain knew his wife; and she conceived, and bare Enoch: and he builded a city, and called the name of the city, after the name of his son, Enoch.'); +INSERT INTO t1(docid,words) VALUES(1004018,'And unto Enoch was born Irad: and Irad begat Mehujael: and Mehujael begat Methusael: and Methusael begat Lamech.'); +INSERT INTO t1(docid,words) VALUES(1004019,'And Lamech took unto him two wives: the name of the one was Adah, and the name of the other Zillah.'); +INSERT INTO t1(docid,words) VALUES(1004020,'And Adah bare Jabal: he was the father of such as dwell in tents, and of such as have cattle.'); +INSERT INTO t1(docid,words) VALUES(1004021,'And his brother''s name was Jubal: he was the father of all such as handle the harp and organ.'); +INSERT INTO t1(docid,words) VALUES(1004022,'And Zillah, she also bare Tubalcain, an instructer of every artificer in brass and iron: and the sister of Tubalcain was Naamah.'); +INSERT INTO t1(docid,words) VALUES(1004023,'And Lamech said unto his wives, Adah and Zillah, Hear my voice; ye wives of Lamech, hearken unto my speech: for I have slain a man to my wounding, and a young man to my hurt.'); +INSERT INTO t1(docid,words) VALUES(1004024,'If Cain shall be avenged sevenfold, truly Lamech seventy and sevenfold.'); +INSERT INTO t1(docid,words) VALUES(1004025,'And Adam knew his wife again; and she bare a son, and called his name Seth: For God, said she, hath appointed me another seed instead of Abel, whom Cain slew.'); +INSERT INTO t1(docid,words) VALUES(1004026,'And to Seth, to him also there was born a son; and he called his name Enos: then began men to call upon the name of the LORD.'); +INSERT INTO t1(docid,words) VALUES(1005001,'This is the book of the generations of Adam. In the day that God created man, in the likeness of God made he him;'); +INSERT INTO t1(docid,words) VALUES(1005002,'Male and female created he them; and blessed them, and called their name Adam, in the day when they were created.'); +INSERT INTO t1(docid,words) VALUES(1005003,'And Adam lived an hundred and thirty years, and begat a son in his own likeness, and after his image; and called his name Seth:'); +INSERT INTO t1(docid,words) VALUES(1005004,'And the days of Adam after he had begotten Seth were eight hundred years: and he begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005005,'And all the days that Adam lived were nine hundred and thirty years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005006,'And Seth lived an hundred and five years, and begat Enos:'); +INSERT INTO t1(docid,words) VALUES(1005007,'And Seth lived after he begat Enos eight hundred and seven years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005008,'And all the days of Seth were nine hundred and twelve years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005009,'And Enos lived ninety years, and begat Cainan:'); +INSERT INTO t1(docid,words) VALUES(1005010,'And Enos lived after he begat Cainan eight hundred and fifteen years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005011,'And all the days of Enos were nine hundred and five years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005012,'And Cainan lived seventy years and begat Mahalaleel:'); +INSERT INTO t1(docid,words) VALUES(1005013,'And Cainan lived after he begat Mahalaleel eight hundred and forty years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005014,'And all the days of Cainan were nine hundred and ten years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005015,'And Mahalaleel lived sixty and five years, and begat Jared:'); +INSERT INTO t1(docid,words) VALUES(1005016,'And Mahalaleel lived after he begat Jared eight hundred and thirty years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005017,'And all the days of Mahalaleel were eight hundred ninety and five years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005018,'And Jared lived an hundred sixty and two years, and he begat Enoch:'); +INSERT INTO t1(docid,words) VALUES(1005019,'And Jared lived after he begat Enoch eight hundred years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005020,'And all the days of Jared were nine hundred sixty and two years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005021,'And Enoch lived sixty and five years, and begat Methuselah:'); +INSERT INTO t1(docid,words) VALUES(1005022,'And Enoch walked with God after he begat Methuselah three hundred years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005023,'And all the days of Enoch were three hundred sixty and five years:'); +INSERT INTO t1(docid,words) VALUES(1005024,'And Enoch walked with God: and he was not; for God took him.'); +INSERT INTO t1(docid,words) VALUES(1005025,'And Methuselah lived an hundred eighty and seven years, and begat Lamech.'); +INSERT INTO t1(docid,words) VALUES(1005026,'And Methuselah lived after he begat Lamech seven hundred eighty and two years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005027,'And all the days of Methuselah were nine hundred sixty and nine years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005028,'And Lamech lived an hundred eighty and two years, and begat a son:'); +INSERT INTO t1(docid,words) VALUES(1005029,'And he called his name Noah, saying, This same shall comfort us concerning our work and toil of our hands, because of the ground which the LORD hath cursed.'); +INSERT INTO t1(docid,words) VALUES(1005030,'And Lamech lived after he begat Noah five hundred ninety and five years, and begat sons and daughters:'); +INSERT INTO t1(docid,words) VALUES(1005031,'And all the days of Lamech were seven hundred seventy and seven years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1005032,'And Noah was five hundred years old: and Noah begat Shem, Ham, and Japheth.'); +INSERT INTO t1(docid,words) VALUES(1006001,'And it came to pass, when men began to multiply on the face of the earth, and daughters were born unto them,'); +INSERT INTO t1(docid,words) VALUES(1006002,'That the sons of God saw the daughters of men that they were fair; and they took them wives of all which they chose.'); +INSERT INTO t1(docid,words) VALUES(1006003,'And the LORD said, My spirit shall not always strive with man, for that he also is flesh: yet his days shall be an hundred and twenty years.'); +INSERT INTO t1(docid,words) VALUES(1006004,'There were giants in the earth in those days; and also after that, when the sons of God came in unto the daughters of men, and they bare children to them, the same became mighty men which were of old, men of renown.'); +INSERT INTO t1(docid,words) VALUES(1006005,'And God saw that the wickedness of man was great in the earth, and that every imagination of the thoughts of his heart was only evil continually.'); +INSERT INTO t1(docid,words) VALUES(1006006,'And it repented the LORD that he had made man on the earth, and it grieved him at his heart.'); +INSERT INTO t1(docid,words) VALUES(1006007,'And the LORD said, I will destroy man whom I have created from the face of the earth; both man, and beast, and the creeping thing, and the fowls of the air; for it repenteth me that I have made them.'); +INSERT INTO t1(docid,words) VALUES(1006008,'But Noah found grace in the eyes of the LORD.'); +INSERT INTO t1(docid,words) VALUES(1006009,'These are the generations of Noah: Noah was a just man and perfect in his generations, and Noah walked with God.'); +INSERT INTO t1(docid,words) VALUES(1006010,'And Noah begat three sons, Shem, Ham, and Japheth.'); +INSERT INTO t1(docid,words) VALUES(1006011,'The earth also was corrupt before God, and the earth was filled with violence.'); +INSERT INTO t1(docid,words) VALUES(1006012,'And God looked upon the earth, and, behold, it was corrupt; for all flesh had corrupted his way upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1006013,'And God said unto Noah, The end of all flesh is come before me; for the earth is filled with violence through them; and, behold, I will destroy them with the earth.'); +INSERT INTO t1(docid,words) VALUES(1006014,'Make thee an ark of gopher wood; rooms shalt thou make in the ark, and shalt pitch it within and without with pitch.'); +INSERT INTO t1(docid,words) VALUES(1006015,'And this is the fashion which thou shalt make it of: The length of the ark shall be three hundred cubits, the breadth of it fifty cubits, and the height of it thirty cubits.'); +INSERT INTO t1(docid,words) VALUES(1006016,'A window shalt thou make to the ark, and in a cubit shalt thou finish it above; and the door of the ark shalt thou set in the side thereof; with lower, second, and third stories shalt thou make it.'); +INSERT INTO t1(docid,words) VALUES(1006017,'And, behold, I, even I, do bring a flood of waters upon the earth, to destroy all flesh, wherein is the breath of life, from under heaven; and every thing that is in the earth shall die.'); +INSERT INTO t1(docid,words) VALUES(1006018,'But with thee will I establish my covenant; and thou shalt come into the ark, thou, and thy sons, and thy wife, and thy sons'' wives with thee.'); +INSERT INTO t1(docid,words) VALUES(1006019,'And of every living thing of all flesh, two of every sort shalt thou bring into the ark, to keep them alive with thee; they shall be male and female.'); +INSERT INTO t1(docid,words) VALUES(1006020,'Of fowls after their kind, and of cattle after their kind, of every creeping thing of the earth after his kind, two of every sort shall come unto thee, to keep them alive.'); +INSERT INTO t1(docid,words) VALUES(1006021,'And take thou unto thee of all food that is eaten, and thou shalt gather it to thee; and it shall be for food for thee, and for them.'); +INSERT INTO t1(docid,words) VALUES(1006022,'Thus did Noah; according to all that God commanded him, so did he.'); +INSERT INTO t1(docid,words) VALUES(1007001,'And the LORD said unto Noah, Come thou and all thy house into the ark; for thee have I seen righteous before me in this generation.'); +INSERT INTO t1(docid,words) VALUES(1007002,'Of every clean beast thou shalt take to thee by sevens, the male and his female: and of beasts that are not clean by two, the male and his female.'); +INSERT INTO t1(docid,words) VALUES(1007003,'Of fowls also of the air by sevens, the male and the female; to keep seed alive upon the face of all the earth.'); +INSERT INTO t1(docid,words) VALUES(1007004,'For yet seven days, and I will cause it to rain upon the earth forty days and forty nights; and every living substance that I have made will I destroy from off the face of the earth.'); +INSERT INTO t1(docid,words) VALUES(1007005,'And Noah did according unto all that the LORD commanded him.'); +INSERT INTO t1(docid,words) VALUES(1007006,'And Noah was six hundred years old when the flood of waters was upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1007007,'And Noah went in, and his sons, and his wife, and his sons'' wives with him, into the ark, because of the waters of the flood.'); +INSERT INTO t1(docid,words) VALUES(1007008,'Of clean beasts, and of beasts that are not clean, and of fowls, and of every thing that creepeth upon the earth,'); +INSERT INTO t1(docid,words) VALUES(1007009,'There went in two and two unto Noah into the ark, the male and the female, as God had commanded Noah.'); +INSERT INTO t1(docid,words) VALUES(1007010,'And it came to pass after seven days, that the waters of the flood were upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1007011,'In the six hundredth year of Noah''s life, in the second month, the seventeenth day of the month, the same day were all the fountains of the great deep broken up, and the windows of heaven were opened.'); +INSERT INTO t1(docid,words) VALUES(1007012,'And the rain was upon the earth forty days and forty nights.'); +INSERT INTO t1(docid,words) VALUES(1007013,'In the selfsame day entered Noah, and Shem, and Ham, and Japheth, the sons of Noah, and Noah''s wife, and the three wives of his sons with them, into the ark;'); +INSERT INTO t1(docid,words) VALUES(1007014,'They, and every beast after his kind, and all the cattle after their kind, and every creeping thing that creepeth upon the earth after his kind, and every fowl after his kind, every bird of every sort.'); +INSERT INTO t1(docid,words) VALUES(1007015,'And they went in unto Noah into the ark, two and two of all flesh, wherein is the breath of life.'); +INSERT INTO t1(docid,words) VALUES(1007016,'And they that went in, went in male and female of all flesh, as God had commanded him: and the LORD shut him in.'); +INSERT INTO t1(docid,words) VALUES(1007017,'And the flood was forty days upon the earth; and the waters increased, and bare up the ark, and it was lift up above the earth.'); +INSERT INTO t1(docid,words) VALUES(1007018,'And the waters prevailed, and were increased greatly upon the earth; and the ark went upon the face of the waters.'); +INSERT INTO t1(docid,words) VALUES(1007019,'And the waters prevailed exceedingly upon the earth; and all the high hills, that were under the whole heaven, were covered.'); +INSERT INTO t1(docid,words) VALUES(1007020,'Fifteen cubits upward did the waters prevail; and the mountains were covered.'); +INSERT INTO t1(docid,words) VALUES(1007021,'And all flesh died that moved upon the earth, both of fowl, and of cattle, and of beast, and of every creeping thing that creepeth upon the earth, and every man:'); +INSERT INTO t1(docid,words) VALUES(1007022,'All in whose nostrils was the breath of life, of all that was in the dry land, died.'); +INSERT INTO t1(docid,words) VALUES(1007023,'And every living substance was destroyed which was upon the face of the ground, both man, and cattle, and the creeping things, and the fowl of the heaven; and they were destroyed from the earth: and Noah only remained alive, and they that were with him in the ark.'); +INSERT INTO t1(docid,words) VALUES(1007024,'And the waters prevailed upon the earth an hundred and fifty days.'); +INSERT INTO t1(docid,words) VALUES(1008001,'And God remembered Noah, and every living thing, and all the cattle that was with him in the ark: and God made a wind to pass over the earth, and the waters asswaged;'); +INSERT INTO t1(docid,words) VALUES(1008002,'The fountains also of the deep and the windows of heaven were stopped, and the rain from heaven was restrained;'); +INSERT INTO t1(docid,words) VALUES(1008003,'And the waters returned from off the earth continually: and after the end of the hundred and fifty days the waters were abated.'); +INSERT INTO t1(docid,words) VALUES(1008004,'And the ark rested in the seventh month, on the seventeenth day of the month, upon the mountains of Ararat.'); +INSERT INTO t1(docid,words) VALUES(1008005,'And the waters decreased continually until the tenth month: in the tenth month, on the first day of the month, were the tops of the mountains seen.'); +INSERT INTO t1(docid,words) VALUES(1008006,'And it came to pass at the end of forty days, that Noah opened the window of the ark which he had made:'); +INSERT INTO t1(docid,words) VALUES(1008007,'And he sent forth a raven, which went forth to and fro, until the waters were dried up from off the earth.'); +INSERT INTO t1(docid,words) VALUES(1008008,'Also he sent forth a dove from him, to see if the waters were abated from off the face of the ground;'); +INSERT INTO t1(docid,words) VALUES(1008009,'But the dove found no rest for the sole of her foot, and she returned unto him into the ark, for the waters were on the face of the whole earth: then he put forth his hand, and took her, and pulled her in unto him into the ark.'); +INSERT INTO t1(docid,words) VALUES(1008010,'And he stayed yet other seven days; and again he sent forth the dove out of the ark;'); +INSERT INTO t1(docid,words) VALUES(1008011,'And the dove came in to him in the evening; and, lo, in her mouth was an olive leaf pluckt off: so Noah knew that the waters were abated from off the earth.'); +INSERT INTO t1(docid,words) VALUES(1008012,'And he stayed yet other seven days; and sent forth the dove; which returned not again unto him any more.'); +INSERT INTO t1(docid,words) VALUES(1008013,'And it came to pass in the six hundredth and first year, in the first month, the first day of the month, the waters were dried up from off the earth: and Noah removed the covering of the ark, and looked, and, behold, the face of the ground was dry.'); +INSERT INTO t1(docid,words) VALUES(1008014,'And in the second month, on the seven and twentieth day of the month, was the earth dried.'); +INSERT INTO t1(docid,words) VALUES(1008015,'And God spake unto Noah, saying,'); +INSERT INTO t1(docid,words) VALUES(1008016,'Go forth of the ark, thou, and thy wife, and thy sons, and thy sons'' wives with thee.'); +INSERT INTO t1(docid,words) VALUES(1008017,'Bring forth with thee every living thing that is with thee, of all flesh, both of fowl, and of cattle, and of every creeping thing that creepeth upon the earth; that they may breed abundantly in the earth, and be fruitful, and multiply upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1008018,'And Noah went forth, and his sons, and his wife, and his sons'' wives with him:'); +INSERT INTO t1(docid,words) VALUES(1008019,'Every beast, every creeping thing, and every fowl, and whatsoever creepeth upon the earth, after their kinds, went forth out of the ark.'); +INSERT INTO t1(docid,words) VALUES(1008020,'And Noah builded an altar unto the LORD; and took of every clean beast, and of every clean fowl, and offered burnt offerings on the altar.'); +INSERT INTO t1(docid,words) VALUES(1008021,'And the LORD smelled a sweet savour; and the LORD said in his heart, I will not again curse the ground any more for man''s sake; for the imagination of man''s heart is evil from his youth; neither will I again smite any more every thing living, as I have done.'); +INSERT INTO t1(docid,words) VALUES(1008022,'While the earth remaineth, seedtime and harvest, and cold and heat, and summer and winter, and day and night shall not cease.'); +INSERT INTO t1(docid,words) VALUES(1009001,'And God blessed Noah and his sons, and said unto them, Be fruitful, and multiply, and replenish the earth.'); +INSERT INTO t1(docid,words) VALUES(1009002,'And the fear of you and the dread of you shall be upon every beast of the earth, and upon every fowl of the air, upon all that moveth upon the earth, and upon all the fishes of the sea; into your hand are they delivered.'); +INSERT INTO t1(docid,words) VALUES(1009003,'Every moving thing that liveth shall be meat for you; even as the green herb have I given you all things.'); +INSERT INTO t1(docid,words) VALUES(1009004,'But flesh with the life thereof, which is the blood thereof, shall ye not eat.'); +INSERT INTO t1(docid,words) VALUES(1009005,'And surely your blood of your lives will I require; at the hand of every beast will I require it, and at the hand of man; at the hand of every man''s brother will I require the life of man.'); +INSERT INTO t1(docid,words) VALUES(1009006,'Whoso sheddeth man''s blood, by man shall his blood be shed: for in the image of God made he man.'); +INSERT INTO t1(docid,words) VALUES(1009007,'And you, be ye fruitful, and multiply; bring forth abundantly in the earth, and multiply therein.'); +INSERT INTO t1(docid,words) VALUES(1009008,'And God spake unto Noah, and to his sons with him, saying,'); +INSERT INTO t1(docid,words) VALUES(1009009,'And I, behold, I establish my covenant with you, and with your seed after you;'); +INSERT INTO t1(docid,words) VALUES(1009010,'And with every living creature that is with you, of the fowl, of the cattle, and of every beast of the earth with you; from all that go out of the ark, to every beast of the earth.'); +INSERT INTO t1(docid,words) VALUES(1009011,'And I will establish my covenant with you, neither shall all flesh be cut off any more by the waters of a flood; neither shall there any more be a flood to destroy the earth.'); +INSERT INTO t1(docid,words) VALUES(1009012,'And God said, This is the token of the covenant which I make between me and you and every living creature that is with you, for perpetual generations:'); +INSERT INTO t1(docid,words) VALUES(1009013,'I do set my bow in the cloud, and it shall be for a token of a covenant between me and the earth.'); +INSERT INTO t1(docid,words) VALUES(1009014,'And it shall come to pass, when I bring a cloud over the earth, that the bow shall be seen in the cloud:'); +INSERT INTO t1(docid,words) VALUES(1009015,'And I will remember my covenant, which is between me and you and every living creature of all flesh; and the waters shall no more become a flood to destroy all flesh.'); +INSERT INTO t1(docid,words) VALUES(1009016,'And the bow shall be in the cloud; and I will look upon it, that I may remember the everlasting covenant between God and every living creature of all flesh that is upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1009017,'And God said unto Noah, This is the token of the covenant, which I have established between me and all flesh that is upon the earth.'); +INSERT INTO t1(docid,words) VALUES(1009018,'And the sons of Noah, that went forth of the ark, were Shem, and Ham, and Japheth: and Ham is the father of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1009019,'These are the three sons of Noah: and of them was the whole earth overspread.'); +INSERT INTO t1(docid,words) VALUES(1009020,'And Noah began to be an husbandman, and he planted a vineyard:'); +INSERT INTO t1(docid,words) VALUES(1009021,'And he drank of the wine, and was drunken; and he was uncovered within his tent.'); +INSERT INTO t1(docid,words) VALUES(1009022,'And Ham, the father of Canaan, saw the nakedness of his father, and told his two brethren without.'); +INSERT INTO t1(docid,words) VALUES(1009023,'And Shem and Japheth took a garment, and laid it upon both their shoulders, and went backward, and covered the nakedness of their father; and their faces were backward, and they saw not their father''s nakedness.'); +INSERT INTO t1(docid,words) VALUES(1009024,'And Noah awoke from his wine, and knew what his younger son had done unto him.'); +INSERT INTO t1(docid,words) VALUES(1009025,'And he said, Cursed be Canaan; a servant of servants shall he be unto his brethren.'); +INSERT INTO t1(docid,words) VALUES(1009026,'And he said, Blessed be the LORD God of Shem; and Canaan shall be his servant.'); +INSERT INTO t1(docid,words) VALUES(1009027,'God shall enlarge Japheth, and he shall dwell in the tents of Shem; and Canaan shall be his servant.'); +INSERT INTO t1(docid,words) VALUES(1009028,'And Noah lived after the flood three hundred and fifty years.'); +INSERT INTO t1(docid,words) VALUES(1009029,'And all the days of Noah were nine hundred and fifty years: and he died.'); +INSERT INTO t1(docid,words) VALUES(1010001,'Now these are the generations of the sons of Noah, Shem, Ham, and Japheth: and unto them were sons born after the flood.'); +INSERT INTO t1(docid,words) VALUES(1010002,'The sons of Japheth; Gomer, and Magog, and Madai, and Javan, and Tubal, and Meshech, and Tiras.'); +INSERT INTO t1(docid,words) VALUES(1010003,'And the sons of Gomer; Ashkenaz, and Riphath, and Togarmah.'); +INSERT INTO t1(docid,words) VALUES(1010004,'And the sons of Javan; Elishah, and Tarshish, Kittim, and Dodanim.'); +INSERT INTO t1(docid,words) VALUES(1010005,'By these were the isles of the Gentiles divided in their lands; every one after his tongue, after their families, in their nations.'); +INSERT INTO t1(docid,words) VALUES(1010006,'And the sons of Ham; Cush, and Mizraim, and Phut, and Canaan.'); +INSERT INTO t1(docid,words) VALUES(1010007,'And the sons of Cush; Seba, and Havilah, and Sabtah, and Raamah, and Sabtechah: and the sons of Raamah; Sheba, and Dedan.'); +INSERT INTO t1(docid,words) VALUES(1010008,'And Cush begat Nimrod: he began to be a mighty one in the earth.'); +INSERT INTO t1(docid,words) VALUES(1010009,'He was a mighty hunter before the LORD: wherefore it is said, Even as Nimrod the mighty hunter before the LORD.'); +INSERT INTO t1(docid,words) VALUES(1010010,'And the beginning of his kingdom was Babel, and Erech, and Accad, and Calneh, in the land of Shinar.'); +INSERT INTO t1(docid,words) VALUES(1010011,'Out of that land went forth Asshur, and builded Nineveh, and the city Rehoboth, and Calah,'); +INSERT INTO t1(docid,words) VALUES(1010012,'And Resen between Nineveh and Calah: the same is a great city.'); +INSERT INTO t1(docid,words) VALUES(1010013,'And Mizraim begat Ludim, and Anamim, and Lehabim, and Naphtuhim,'); +INSERT INTO t1(docid,words) VALUES(1010014,'And Pathrusim, and Casluhim, (out of whom came Philistim,) and Caphtorim.'); +INSERT INTO t1(docid,words) VALUES(1010015,'And Canaan begat Sidon his first born, and Heth,'); +INSERT INTO t1(docid,words) VALUES(1010016,'And the Jebusite, and the Amorite, and the Girgasite,'); +INSERT INTO t1(docid,words) VALUES(1010017,'And the Hivite, and the Arkite, and the Sinite,'); +INSERT INTO t1(docid,words) VALUES(1010018,'And the Arvadite, and the Zemarite, and the Hamathite: and afterward were the families of the Canaanites spread abroad.'); +INSERT INTO t1(docid,words) VALUES(1010019,'And the border of the Canaanites was from Sidon, as thou comest to Gerar, unto Gaza; as thou goest, unto Sodom, and Gomorrah, and Admah, and Zeboim, even unto Lasha.'); +INSERT INTO t1(docid,words) VALUES(1010020,'These are the sons of Ham, after their families, after their tongues, in their countries, and in their nations.'); +INSERT INTO t1(docid,words) VALUES(1010021,'Unto Shem also, the father of all the children of Eber, the brother of Japheth the elder, even to him were children born.'); +INSERT INTO t1(docid,words) VALUES(1010022,'The children of Shem; Elam, and Asshur, and Arphaxad, and Lud, and Aram.'); +INSERT INTO t1(docid,words) VALUES(1010023,'And the children of Aram; Uz, and Hul, and Gether, and Mash.'); +INSERT INTO t1(docid,words) VALUES(1010024,'And Arphaxad begat Salah; and Salah begat Eber.'); +INSERT INTO t1(docid,words) VALUES(1010025,'And unto Eber were born two sons: the name of one was Peleg; for in his days was the earth divided; and his brother''s name was Joktan.'); +INSERT INTO t1(docid,words) VALUES(1010026,'And Joktan begat Almodad, and Sheleph, and Hazarmaveth, and Jerah,'); +INSERT INTO t1(docid,words) VALUES(1010027,'And Hadoram, and Uzal, and Diklah,'); +INSERT INTO t1(docid,words) VALUES(1010028,'And Obal, and Abimael, and Sheba,'); +INSERT INTO t1(docid,words) VALUES(1010029,'And Ophir, and Havilah, and Jobab: all these were the sons of Joktan.'); +INSERT INTO t1(docid,words) VALUES(1010030,'And their dwelling was from Mesha, as thou goest unto Sephar a mount of the east.'); +INSERT INTO t1(docid,words) VALUES(1010031,'These are the sons of Shem, after their families, after their tongues, in their lands, after their nations.'); +INSERT INTO t1(docid,words) VALUES(1010032,'These are the families of the sons of Noah, after their generations, in their nations: and by these were the nations divided in the earth after the flood.'); +INSERT INTO t1(docid,words) VALUES(1011001,'And the whole earth was of one language, and of one speech.'); +INSERT INTO t1(docid,words) VALUES(1011002,'And it came to pass, as they journeyed from the east, that they found a plain in the land of Shinar; and they dwelt there.'); +INSERT INTO t1(docid,words) VALUES(1011003,'And they said one to another, Go to, let us make brick, and burn them thoroughly. And they had brick for stone, and slime had they for morter.'); +INSERT INTO t1(docid,words) VALUES(1011004,'And they said, Go to, let us build us a city and a tower, whose top may reach unto heaven; and let us make us a name, lest we be scattered abroad upon the face of the whole earth.'); +INSERT INTO t1(docid,words) VALUES(1011005,'And the LORD came down to see the city and the tower, which the children of men builded.'); +INSERT INTO t1(docid,words) VALUES(1011006,'And the LORD said, Behold, the people is one, and they have all one language; and this they begin to do: and now nothing will be restrained from them, which they have imagined to do.'); +INSERT INTO t1(docid,words) VALUES(1011007,'Go to, let us go down, and there confound their language, that they may not understand one another''s speech.'); +INSERT INTO t1(docid,words) VALUES(1011008,'So the LORD scattered them abroad from thence upon the face of all the earth: and they left off to build the city.'); +INSERT INTO t1(docid,words) VALUES(1011009,'Therefore is the name of it called Babel; because the LORD did there confound the language of all the earth: and from thence did the LORD scatter them abroad upon the face of all the earth.'); +INSERT INTO t1(docid,words) VALUES(1011010,'These are the generations of Shem: Shem was an hundred years old, and begat Arphaxad two years after the flood:'); +INSERT INTO t1(docid,words) VALUES(1011011,'And Shem lived after he begat Arphaxad five hundred years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011012,'And Arphaxad lived five and thirty years, and begat Salah:'); +INSERT INTO t1(docid,words) VALUES(1011013,'And Arphaxad lived after he begat Salah four hundred and three years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011014,'And Salah lived thirty years, and begat Eber:'); +INSERT INTO t1(docid,words) VALUES(1011015,'And Salah lived after he begat Eber four hundred and three years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011016,'And Eber lived four and thirty years, and begat Peleg:'); +INSERT INTO t1(docid,words) VALUES(1011017,'And Eber lived after he begat Peleg four hundred and thirty years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011018,'And Peleg lived thirty years, and begat Reu:'); +INSERT INTO t1(docid,words) VALUES(1011019,'And Peleg lived after he begat Reu two hundred and nine years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011020,'And Reu lived two and thirty years, and begat Serug:'); +INSERT INTO t1(docid,words) VALUES(1011021,'And Reu lived after he begat Serug two hundred and seven years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011022,'And Serug lived thirty years, and begat Nahor:'); +INSERT INTO t1(docid,words) VALUES(1011023,'And Serug lived after he begat Nahor two hundred years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011024,'And Nahor lived nine and twenty years, and begat Terah:'); +INSERT INTO t1(docid,words) VALUES(1011025,'And Nahor lived after he begat Terah an hundred and nineteen years, and begat sons and daughters.'); +INSERT INTO t1(docid,words) VALUES(1011026,'And Terah lived seventy years, and begat Abram, Nahor, and Haran.'); +INSERT INTO t1(docid,words) VALUES(1011027,'Now these are the generations of Terah: Terah begat Abram, Nahor, and Haran; and Haran begat Lot.'); +INSERT INTO t1(docid,words) VALUES(1011028,'And Haran died before his father Terah in the land of his nativity, in Ur of the Chaldees.'); +INSERT INTO t1(docid,words) VALUES(1011029,'And Abram and Nahor took them wives: the name of Abram''s wife was Sarai; and the name of Nahor''s wife, Milcah, the daughter of Haran, the father of Milcah, and the father of Iscah.'); +INSERT INTO t1(docid,words) VALUES(1011030,'But Sarai was barren; she had no child.'); +INSERT INTO t1(docid,words) VALUES(1011031,'And Terah took Abram his son, and Lot the son of Haran his son''s son, and Sarai his daughter in law, his son Abram''s wife; and they went forth with them from Ur of the Chaldees, to go into the land of Canaan; and they came unto Haran, and dwelt there.'); +INSERT INTO t1(docid,words) VALUES(1011032,'And the days of Terah were two hundred and five years: and Terah died in Haran.'); +INSERT INTO t1(docid,words) VALUES(1012001,'Now the LORD had said unto Abram, Get thee out of thy country, and from thy kindred, and from thy father''s house, unto a land that I will shew thee:'); +INSERT INTO t1(docid,words) VALUES(1012002,'And I will make of thee a great nation, and I will bless thee, and make thy name great; and thou shalt be a blessing:'); +INSERT INTO t1(docid,words) VALUES(1012003,'And I will bless them that bless thee, and curse him that curseth thee: and in thee shall all families of the earth be blessed.'); +INSERT INTO t1(docid,words) VALUES(1012004,'So Abram departed, as the LORD had spoken unto him; and Lot went with him: and Abram was seventy and five years old when he departed out of Haran.'); +INSERT INTO t1(docid,words) VALUES(1012005,'And Abram took Sarai his wife, and Lot his brother''s son, and all their substance that they had gathered, and the souls that they had gotten in Haran; and they went forth to go into the land of Canaan; and into the land of Canaan they came.'); +INSERT INTO t1(docid,words) VALUES(1012006,'And Abram passed through the land unto the place of Sichem, unto the plain of Moreh. And the Canaanite was then in the land.'); +INSERT INTO t1(docid,words) VALUES(1012007,'And the LORD appeared unto Abram, and said, Unto thy seed will I give this land: and there builded he an altar unto the LORD, who appeared unto him.'); +INSERT INTO t1(docid,words) VALUES(1012008,'And he removed from thence unto a mountain on the east of Bethel, and pitched his tent, having Bethel on the west, and Hai on the east: and there he builded an altar unto the LORD, and called upon the name of the LORD.'); +INSERT INTO t1(docid,words) VALUES(1012009,'And Abram journeyed, going on still toward the south.'); +INSERT INTO t1(docid,words) VALUES(1012010,'And there was a famine in the land: and Abram went down into Egypt to sojourn there; for the famine was grievous in the land.'); +INSERT INTO t1(docid,words) VALUES(1012011,'And it came to pass, when he was come near to enter into Egypt, that he said unto Sarai his wife, Behold now, I know that thou art a fair woman to look upon:'); +INSERT INTO t1(docid,words) VALUES(1012012,'Therefore it shall come to pass, when the Egyptians shall see thee, that they shall say, This is his wife: and they will kill me, but they will save thee alive.'); +INSERT INTO t1(docid,words) VALUES(1012013,'Say, I pray thee, thou art my sister: that it may be well with me for thy sake; and my soul shall live because of thee.'); +INSERT INTO t1(docid,words) VALUES(1012014,'And it came to pass, that, when Abram was come into Egypt, the Egyptians beheld the woman that she was very fair.'); +INSERT INTO t1(docid,words) VALUES(1012015,'The princes also of Pharaoh saw her, and commended her before Pharaoh: and the woman was taken into Pharaoh''s house.'); +INSERT INTO t1(docid,words) VALUES(1012016,'And he entreated Abram well for her sake: and he had sheep, and oxen, and he asses, and menservants, and maidservants, and she asses, and camels.'); +INSERT INTO t1(docid,words) VALUES(1012017,'And the LORD plagued Pharaoh and his house with great plagues because of Sarai Abram''s wife.'); +INSERT INTO t1(docid,words) VALUES(1012018,'And Pharaoh called Abram and said, What is this that thou hast done unto me? why didst thou not tell me that she was thy wife?'); +INSERT INTO t1(docid,words) VALUES(1012019,'Why saidst thou, She is my sister? so I might have taken her to me to wife: now therefore behold thy wife, take her, and go thy way.'); +INSERT INTO t1(docid,words) VALUES(1012020,'And Pharaoh commanded his men concerning him: and they sent him away, and his wife, and all that he had.'); +INSERT INTO t1(docid,words) VALUES(1013001,'And Abram went up out of Egypt, he, and his wife, and all that he had, and Lot with him, into the south.'); +INSERT INTO t1(docid,words) VALUES(1013002,'And Abram was very rich in cattle, in silver, and in gold.'); +INSERT INTO t1(docid,words) VALUES(1013003,'And he went on his journeys from the south even to Bethel, unto the place where his tent had been at the beginning, between Bethel and Hai;'); +INSERT INTO t1(docid,words) VALUES(1013004,'Unto the place of the altar, which he had make there at the first: and there Abram called on the name of the LORD.'); +INSERT INTO t1(docid,words) VALUES(1013005,'And Lot also, which went with Abram, had flocks, and herds, and tents.'); +INSERT INTO t1(docid,words) VALUES(1013006,'And the land was not able to bear them, that they might dwell together: for their substance was great, so that they could not dwell together.'); +INSERT INTO t1(docid,words) VALUES(1013007,'And there was a strife between the herdmen of Abram''s cattle and the herdmen of Lot''s cattle: and the Canaanite and the Perizzite dwelled then in the land.'); +INSERT INTO t1(docid,words) VALUES(1013008,'And Abram said unto Lot, Let there be no strife, I pray thee, between me and thee, and between my herdmen and thy herdmen; for we be brethren.'); +INSERT INTO t1(docid,words) VALUES(1013009,'Is not the whole land before thee? separate thyself, I pray thee, from me: if thou wilt take the left hand, then I will go to the right; or if thou depart to the right hand, then I will go to the left.'); +INSERT INTO t1(docid,words) VALUES(1013010,'And Lot lifted up his eyes, and beheld all the plain of Jordan, that it was well watered every where, before the LORD destroyed Sodom and Gomorrah, even as the garden of the LORD, like the land of Egypt, as thou comest unto Zoar.'); +INSERT INTO t1(docid,words) VALUES(1013011,'Then Lot chose him all the plain of Jordan; and Lot journeyed east: and they separated themselves the one from the other.'); +INSERT INTO t1(docid,words) VALUES(1013012,'Abram dwelled in the land of Canaan, and Lot dwelled in the cities of the plain, and pitched his tent toward Sodom.'); +INSERT INTO t1(docid,words) VALUES(1013013,'But the men of Sodom were wicked and sinners before the LORD exceedingly.'); +INSERT INTO t1(docid,words) VALUES(1013014,'And the LORD said unto Abram, after that Lot was separated from him, Lift up now thine eyes, and look from the place where thou art northward, and southward, and eastward, and westward:'); +INSERT INTO t1(docid,words) VALUES(1013015,'For all the land which thou seest, to thee will I give it, and to thy seed for ever.'); +INSERT INTO t1(docid,words) VALUES(1013016,'And I will make thy seed as the dust of the earth: so that if a man can number the dust of the earth, then shall thy seed also be numbered.'); +INSERT INTO t1(docid,words) VALUES(1013017,'Arise, walk through the land in the length of it and in the breadth of it; for I will give it unto thee.'); +INSERT INTO t1(docid,words) VALUES(1013018,'Then Abram removed his tent, and came and dwelt in the plain of Mamre, which is in Hebron, and built there an altar unto the LORD.'); +INSERT INTO t1(docid,words) VALUES(1014001,'And it came to pass in the days of Amraphel king of Shinar, Arioch king of Ellasar, Chedorlaomer king of Elam, and Tidal king of nations;'); +INSERT INTO t1(docid,words) VALUES(1014002,'That these made war with Bera king of Sodom, and with Birsha king of Gomorrah, Shinab king of Admah, and Shemeber king of Zeboiim, and the king of Bela, which is Zoar.'); +INSERT INTO t1(docid,words) VALUES(1014003,'All these were joined together in the vale of Siddim, which is the salt sea.'); +INSERT INTO t1(docid,words) VALUES(1014004,'Twelve years they served Chedorlaomer, and in the thirteenth year they rebelled.'); +INSERT INTO t1(docid,words) VALUES(1014005,'And in the fourteenth year came Chedorlaomer, and the kings that were with him, and smote the Rephaims in Ashteroth Karnaim, and the Zuzims in Ham, and the Emins in Shaveh Kiriathaim,'); +INSERT INTO t1(docid,words) VALUES(1014006,'And the Horites in their mount Seir, unto Elparan, which is by the wilderness.'); +INSERT INTO t1(docid,words) VALUES(1014007,'And they returned, and came to Enmishpat, which is Kadesh, and smote all the country of the Amalekites, and also the Amorites, that dwelt in Hazezontamar.'); +INSERT INTO t1(docid,words) VALUES(1014008,'And there went out the king of Sodom, and the king of Gomorrah, and the king of Admah, and the king of Zeboiim, and the king of Bela (the same is Zoar;) and they joined battle with them in the vale of Siddim;'); +INSERT INTO t1(docid,words) VALUES(1014009,'With Chedorlaomer the king of Elam, and with Tidal king of nations, and Amraphel king of Shinar, and Arioch king of Ellasar; four kings with five.'); +INSERT INTO t1(docid,words) VALUES(1014010,'And the vale of Siddim was full of slimepits; and the kings of Sodom and Gomorrah fled, and fell there; and they that remained fled to the mountain.'); +INSERT INTO t1(docid,words) VALUES(1014011,'And they took all the goods of Sodom and Gomorrah, and all their victuals, and went their way.'); +INSERT INTO t1(docid,words) VALUES(1014012,'And they took Lot, Abram''s brother''s son, who dwelt in Sodom, and his goods, and departed.'); +INSERT INTO t1(docid,words) VALUES(1014013,'And there came one that had escaped, and told Abram the Hebrew; for he dwelt in the plain of Mamre the Amorite, brother of Eshcol, and brother of Aner: and these were confederate with Abram.'); +INSERT INTO t1(docid,words) VALUES(1014014,'And when Abram heard that his brother was taken captive, he armed his trained servants, born in his own house, three hundred and eighteen, and pursued them unto Dan.'); +INSERT INTO t1(docid,words) VALUES(1014015,'And he divided himself against them, he and his servants, by night, and smote them, and pursued them unto Hobah, which is on the left hand of Damascus.'); +INSERT INTO t1(docid,words) VALUES(1014016,'And he brought back all the goods, and also brought again his brother Lot, and his goods, and the women also, and the people.'); +INSERT INTO t1(docid,words) VALUES(1014017,'And the king of Sodom went out to meet him after his return from the slaughter of Chedorlaomer, and of the kings that were with him, at the valley of Shaveh, which is the king''s dale.'); +INSERT INTO t1(docid,words) VALUES(1014018,'And Melchizedek king of Salem brought forth bread and wine: and he was the priest of the most high God.'); +INSERT INTO t1(docid,words) VALUES(1014019,'And he blessed him, and said, Blessed be Abram of the most high God, possessor of heaven and earth:'); +INSERT INTO t1(docid,words) VALUES(1014020,'And blessed be the most high God, which hath delivered thine enemies into thy hand. And he gave him tithes of all.'); +INSERT INTO t1(docid,words) VALUES(1014021,'And the king of Sodom said unto Abram, Give me the persons, and take the goods to thyself.'); +INSERT INTO t1(docid,words) VALUES(1014022,'And Abram said to the king of Sodom, I have lift up mine hand unto the LORD, the most high God, the possessor of heaven and earth,'); +INSERT INTO t1(docid,words) VALUES(1014023,'That I will not take from a thread even to a shoelatchet, and that I will not take any thing that is thine, lest thou shouldest say, I have made Abram rich:'); +INSERT INTO t1(docid,words) VALUES(1014024,'Save only that which the young men have eaten, and the portion of the men which went with me, Aner, Eshcol, and Mamre; let them take their portion.'); +INSERT INTO t1(docid,words) VALUES(1015001,'After these things the word of the LORD came unto Abram in a vision, saying, Fear not, Abram: I am thy shield, and thy exceeding great reward.'); +INSERT INTO t1(docid,words) VALUES(1015002,'And Abram said, LORD God, what wilt thou give me, seeing I go childless, and the steward of my house is this Eliezer of Damascus?'); +INSERT INTO t1(docid,words) VALUES(1015003,'And Abram said, Behold, to me thou hast given no seed: and, lo, one born in my house is mine heir.'); +INSERT INTO t1(docid,words) VALUES(1015004,'And, behold, the word of the LORD came unto him, saying, This shall not be thine heir; but he that shall come forth out of thine own bowels shall be thine heir.'); +INSERT INTO t1(docid,words) VALUES(1015005,'And he brought him forth abroad, and said, Look now toward heaven, and tell the stars, if thou be able to number them: and he said unto him, So shall thy seed be.'); +INSERT INTO t1(docid,words) VALUES(1015006,'And he believed in the LORD; and he counted it to him for righteousness.'); +INSERT INTO t1(docid,words) VALUES(1015007,'And he said unto him, I am the LORD that brought thee out of Ur of the Chaldees, to give thee this land to inherit it.'); +INSERT INTO t1(docid,words) VALUES(1015008,'And he said, LORD God, whereby shall I know that I shall inherit it?'); +INSERT INTO t1(docid,words) VALUES(1015009,'And he said unto him, Take me an heifer of three years old, and a she goat of three years old, and a ram of three years old, and a turtledove, and a young pigeon.'); +INSERT INTO t1(docid,words) VALUES(1015010,'And he took unto him all these, and divided them in the midst, and laid each piece one against another: but the birds divided he not.'); +INSERT INTO t1(docid,words) VALUES(1015011,'And when the fowls came down upon the carcases, Abram drove them away.'); +INSERT INTO t1(docid,words) VALUES(1015012,'And when the sun was going down, a deep sleep fell upon Abram; and, lo, an horror of great darkness fell upon him.'); +INSERT INTO t1(docid,words) VALUES(1015013,'And he said unto Abram, Know of a surety that thy seed shall be a stranger in a land that is not their''s, and shall serve them; and they shall afflict them four hundred years;'); +INSERT INTO t1(docid,words) VALUES(1015014,'And also that nation, whom they shall serve, will I judge: and afterward shall they come out with great substance.'); +INSERT INTO t1(docid,words) VALUES(1015015,'And thou shalt go to thy fathers in peace; thou shalt be buried in a good old age.'); +INSERT INTO t1(docid,words) VALUES(1015016,'But in the fourth generation they shall come hither again: for the iniquity of the Amorites is not yet full.'); +INSERT INTO t1(docid,words) VALUES(1015017,'And it came to pass, that, when the sun went down, and it was dark, behold a smoking furnace, and a burning lamp that passed between those pieces.'); +INSERT INTO t1(docid,words) VALUES(1015018,'In the same day the LORD made a covenant with Abram, saying, Unto thy seed have I given this land, from the river of Egypt unto the great river, the river Euphrates:'); +INSERT INTO t1(docid,words) VALUES(1015019,'The Kenites, and the Kenizzites, and the Kadmonites,'); +INSERT INTO t1(docid,words) VALUES(1015020,'And the Hittites, and the Perizzites, and the Rephaims,'); +INSERT INTO t1(docid,words) VALUES(1015021,'And the Amorites, and the Canaanites, and the Girgashites, and the Jebusites.'); +INSERT INTO t1(docid,words) VALUES(1016001,'Now Sarai Abram''s wife bare him no children: and she had an handmaid, an Egyptian, whose name was Hagar.'); +INSERT INTO t1(docid,words) VALUES(1016002,'And Sarai said unto Abram, Behold now, the LORD hath restrained me from bearing: I pray thee, go in unto my maid; it may be that I may obtain children by her. And Abram hearkened to the voice of Sarai.'); +INSERT INTO t1(docid,words) VALUES(1016003,'And Sarai Abram''s wife took Hagar her maid the Egyptian, after Abram had dwelt ten years in the land of Canaan, and gave her to her husband Abram to be his wife.'); +INSERT INTO t1(docid,words) VALUES(1016004,'And he went in unto Hagar, and she conceived: and when she saw that she had conceived, her mistress was despised in her eyes.'); +INSERT INTO t1(docid,words) VALUES(1016005,'And Sarai said unto Abram, My wrong be upon thee: I have given my maid into thy bosom; and when she saw that she had conceived, I was despised in her eyes: the LORD judge between me and thee.'); +INSERT INTO t1(docid,words) VALUES(1016006,'But Abram said unto Sarai, Behold, thy maid is in thine hand; do to her as it pleaseth thee. And when Sarai dealt hardly with her, she fled from her face.'); +INSERT INTO t1(docid,words) VALUES(1016007,'And the angel of the LORD found her by a fountain of water in the wilderness, by the fountain in the way to Shur.'); +INSERT INTO t1(docid,words) VALUES(1016008,'And he said, Hagar, Sarai''s maid, whence camest thou? and whither wilt thou go? And she said, I flee from the face of my mistress Sarai.'); +INSERT INTO t1(docid,words) VALUES(1016009,'And the angel of the LORD said unto her, Return to thy mistress, and submit thyself under her hands.'); +INSERT INTO t1(docid,words) VALUES(1016010,'And the angel of the LORD said unto her, I will multiply thy seed exceedingly, that it shall not be numbered for multitude.'); +INSERT INTO t1(docid,words) VALUES(1016011,'And the angel of the LORD said unto her, Behold, thou art with child and shalt bear a son, and shalt call his name Ishmael; because the LORD hath heard thy affliction.'); +INSERT INTO t1(docid,words) VALUES(1016012,'And he will be a wild man; his hand will be against every man, and every man''s hand against him; and he shall dwell in the presence of all his brethren.'); +INSERT INTO t1(docid,words) VALUES(1016013,'And she called the name of the LORD that spake unto her, Thou God seest me: for she said, Have I also here looked after him that seeth me?'); +INSERT INTO t1(docid,words) VALUES(1016014,'Wherefore the well was called Beerlahairoi; behold, it is between Kadesh and Bered.'); +INSERT INTO t1(docid,words) VALUES(1016015,'And Hagar bare Abram a son: and Abram called his son''s name, which Hagar bare, Ishmael.'); +INSERT INTO t1(docid,words) VALUES(1016016,'And Abram was fourscore and six years old, when Hagar bare Ishmael to Abram.'); +INSERT INTO t1(docid,words) VALUES(1017001,'And when Abram was ninety years old and nine, the LORD appeared to Abram, and said unto him, I am the Almighty God; walk before me, and be thou perfect.'); +INSERT INTO t1(docid,words) VALUES(1017002,'And I will make my covenant between me and thee, and will multiply thee exceedingly.'); +INSERT INTO t1(docid,words) VALUES(1017003,'And Abram fell on his face: and God talked with him, saying,'); +INSERT INTO t1(docid,words) VALUES(1017004,'As for me, behold, my covenant is with thee, and thou shalt be a father of many nations.'); +INSERT INTO t1(docid,words) VALUES(1017005,'Neither shall thy name any more be called Abram, but thy name shall be Abraham; for a father of many nations have I made thee.'); +INSERT INTO t1(docid,words) VALUES(1017006,'And I will make thee exceeding fruitful, and I will make nations of thee, and kings shall come out of thee.'); +INSERT INTO t1(docid,words) VALUES(1017007,'And I will establish my covenant between me and thee and thy seed after thee in their generations for an everlasting covenant, to be a God unto thee, and to thy seed after thee.'); +INSERT INTO t1(docid,words) VALUES(1017008,'And I will give unto thee, and to thy seed after thee, the land wherein thou art a stranger, all the land of Canaan, for an everlasting possession; and I will be their God.'); +INSERT INTO t1(docid,words) VALUES(1017009,'And God said unto Abraham, Thou shalt keep my covenant therefore, thou, and thy seed after thee in their generations.'); +INSERT INTO t1(docid,words) VALUES(1017010,'This is my covenant, which ye shall keep, between me and you and thy seed after thee; Every man child among you shall be circumcised.'); +INSERT INTO t1(docid,words) VALUES(1017011,'And ye shall circumcise the flesh of your foreskin; and it shall be a token of the covenant betwixt me and you.'); +INSERT INTO t1(docid,words) VALUES(1017012,'And he that is eight days old shall be circumcised among you, every man child in your generations, he that is born in the house, or bought with money of any stranger, which is not of thy seed.'); +INSERT INTO t1(docid,words) VALUES(1017013,'He that is born in thy house, and he that is bought with thy money, must needs be circumcised: and my covenant shall be in your flesh for an everlasting covenant.'); +INSERT INTO t1(docid,words) VALUES(1017014,'And the uncircumcised man child whose flesh of his foreskin is not circumcised, that soul shall be cut off from his people; he hath broken my covenant.'); +INSERT INTO t1(docid,words) VALUES(1017015,'And God said unto Abraham, As for Sarai thy wife, thou shalt not call her name Sarai, but Sarah shall her name be.'); +INSERT INTO t1(docid,words) VALUES(1017016,'And I will bless her, and give thee a son also of her: yea, I will bless her, and she shall be a mother of nations; kings of people shall be of her.'); +INSERT INTO t1(docid,words) VALUES(1017017,'Then Abraham fell upon his face, and laughed, and said in his heart, Shall a child be born unto him that is an hundred years old? and shall Sarah, that is ninety years old, bear?'); +INSERT INTO t1(docid,words) VALUES(1017018,'And Abraham said unto God, O that Ishmael might live before thee!'); +INSERT INTO t1(docid,words) VALUES(1017019,'And God said, Sarah thy wife shall bear thee a son indeed; and thou shalt call his name Isaac: and I will establish my covenant with him for an everlasting covenant, and with his seed after him.'); +INSERT INTO t1(docid,words) VALUES(1017020,'And as for Ishmael, I have heard thee: Behold, I have blessed him, and will make him fruitful, and will multiply him exceedingly; twelve princes shall he beget, and I will make him a great nation.'); +INSERT INTO t1(docid,words) VALUES(1017021,'But my covenant will I establish with Isaac, which Sarah shall bear unto thee at this set time in the next year.'); +INSERT INTO t1(docid,words) VALUES(1017022,'And he left off talking with him, and God went up from Abraham.'); +INSERT INTO t1(docid,words) VALUES(1017023,'And Abraham took Ishmael his son, and all that were born in his house, and all that were bought with his money, every male among the men of Abraham''s house; and circumcised the flesh of their foreskin in the selfsame day, as God had said unto him.'); +INSERT INTO t1(docid,words) VALUES(1017024,'And Abraham was ninety years old and nine, when he was circumcised in the flesh of his foreskin.'); +INSERT INTO t1(docid,words) VALUES(1017025,'And Ishmael his son was thirteen years old, when he was circumcised in the flesh of his foreskin.'); +INSERT INTO t1(docid,words) VALUES(1017026,'In the selfsame day was Abraham circumcised, and Ishmael his son.'); +INSERT INTO t1(docid,words) VALUES(1017027,'And all the men of his house, born in the house, and bought with money of the stranger, were circumcised with him.'); +INSERT INTO t1(docid,words) VALUES(1018001,'And the LORD appeared unto him in the plains of Mamre: and he sat in the tent door in the heat of the day;'); +INSERT INTO t1(docid,words) VALUES(1018002,'And he lift up his eyes and looked, and, lo, three men stood by him: and when he saw them, he ran to meet them from the tent door, and bowed himself toward the ground,'); +INSERT INTO t1(docid,words) VALUES(1018003,'And said, My LORD, if now I have found favour in thy sight, pass not away, I pray thee, from thy servant:'); +INSERT INTO t1(docid,words) VALUES(1018004,'Let a little water, I pray you, be fetched, and wash your feet, and rest yourselves under the tree:'); +INSERT INTO t1(docid,words) VALUES(1018005,'And I will fetch a morsel of bread, and comfort ye your hearts; after that ye shall pass on: for therefore are ye come to your servant. And they said, So do, as thou hast said.'); +INSERT INTO t1(docid,words) VALUES(1018006,'And Abraham hastened into the tent unto Sarah, and said, Make ready quickly three measures of fine meal, knead it, and make cakes upon the hearth.'); +INSERT INTO t1(docid,words) VALUES(1018007,'And Abraham ran unto the herd, and fetcht a calf tender and good, and gave it unto a young man; and he hasted to dress it.'); +INSERT INTO t1(docid,words) VALUES(1018008,'And he took butter, and milk, and the calf which he had dressed, and set it before them; and he stood by them under the tree, and they did eat.'); +INSERT INTO t1(docid,words) VALUES(1018009,'And they said unto him, Where is Sarah thy wife? And he said, Behold, in the tent.'); +INSERT INTO t1(docid,words) VALUES(1018010,'And he said, I will certainly return unto thee according to the time of life; and, lo, Sarah thy wife shall have a son. And Sarah heard it in the tent door, which was behind him.'); +INSERT INTO t1(docid,words) VALUES(1018011,'Now Abraham and Sarah were old and well stricken in age; and it ceased to be with Sarah after the manner of women.'); +INSERT INTO t1(docid,words) VALUES(1018012,'Therefore Sarah laughed within herself, saying, After I am waxed old shall I have pleasure, my lord being old also?'); +INSERT INTO t1(docid,words) VALUES(1018013,'And the LORD said unto Abraham, Wherefore did Sarah laugh, saying, Shall I of a surety bear a child, which am old?'); +INSERT INTO t1(docid,words) VALUES(1018014,'Is any thing too hard for the LORD? At the time appointed I will return unto thee, according to the time of life, and Sarah shall have a son.'); +INSERT INTO t1(docid,words) VALUES(1018015,'Then Sarah denied, saying, I laughed not; for she was afraid. And he said, Nay; but thou didst laugh.'); +INSERT INTO t1(docid,words) VALUES(1018016,'And the men rose up from thence, and looked toward Sodom: and Abraham went with them to bring them on the way.'); +INSERT INTO t1(docid,words) VALUES(1018017,'And the LORD said, Shall I hide from Abraham that thing which I do;'); +INSERT INTO t1(docid,words) VALUES(1018018,'Seeing that Abraham shall surely become a great and mighty nation, and all the nations of the earth shall be blessed in him?'); +INSERT INTO t1(docid,words) VALUES(1018019,'For I know him, that he will command his children and his household after him, and they shall keep the way of the LORD, to do justice and judgment; that the LORD may bring upon Abraham that which he hath spoken of him.'); +INSERT INTO t1(docid,words) VALUES(1018020,'And the LORD said, Because the cry of Sodom and Gomorrah is great, and because their sin is very grievous;'); +INSERT INTO t1(docid,words) VALUES(1018021,'I will go down now, and see whether they have done altogether according to the cry of it, which is come unto me; and if not, I will know.'); +INSERT INTO t1(docid,words) VALUES(1018022,'And the men turned their faces from thence, and went toward Sodom: but Abraham stood yet before the LORD.'); +INSERT INTO t1(docid,words) VALUES(1018023,'And Abraham drew near, and said, Wilt thou also destroy the righteous with the wicked?'); +INSERT INTO t1(docid,words) VALUES(1018024,'Peradventure there be fifty righteous within the city: wilt thou also destroy and not spare the place for the fifty righteous that are therein?'); +INSERT INTO t1(docid,words) VALUES(1018025,'That be far from thee to do after this manner, to slay the righteous with the wicked: and that the righteous should be as the wicked, that be far from thee: Shall not the Judge of all the earth do right?'); +INSERT INTO t1(docid,words) VALUES(1018026,'And the LORD said, If I find in Sodom fifty righteous within the city, then I will spare all the place for their sakes.'); +INSERT INTO t1(docid,words) VALUES(1018027,'And Abraham answered and said, Behold now, I have taken upon me to speak unto the LORD, which am but dust and ashes:'); +INSERT INTO t1(docid,words) VALUES(1018028,'Peradventure there shall lack five of the fifty righteous: wilt thou destroy all the city for lack of five? And he said, If I find there forty and five, I will not destroy it.'); +INSERT INTO t1(docid,words) VALUES(1018029,'And he spake unto him yet again, and said, Peradventure there shall be forty found there. And he said, I will not do it for forty''s sake.'); +INSERT INTO t1(docid,words) VALUES(1018030,'And he said unto him, Oh let not the LORD be angry, and I will speak: Peradventure there shall thirty be found there. And he said, I will not do it, if I find thirty there.'); +INSERT INTO t1(docid,words) VALUES(1018031,'And he said, Behold now, I have taken upon me to speak unto the LORD: Peradventure there shall be twenty found there. And he said, I will not destroy it for twenty''s sake.'); +INSERT INTO t1(docid,words) VALUES(1018032,'And he said, Oh let not the LORD be angry, and I will speak yet but this once: Peradventure ten shall be found there. And he said, I will not destroy it for ten''s sake.'); +INSERT INTO t1(docid,words) VALUES(1018033,'And the LORD went his way, as soon as he had left communing with Abraham: and Abraham returned unto his place.'); +INSERT INTO t1(docid,words) VALUES(1019001,'And there came two angels to Sodom at even; and Lot sat in the gate of Sodom: and Lot seeing them rose up to meet them; and he bowed himself with his face toward the ground;'); +INSERT INTO t1(docid,words) VALUES(1019002,'And he said, Behold now, my lords, turn in, I pray you, into your servant''s house, and tarry all night, and wash your feet, and ye shall rise up early, and go on your ways. And they said, Nay; but we will abide in the street all night.'); +INSERT INTO t1(docid,words) VALUES(1019003,'And he pressed upon them greatly; and they turned in unto him, and entered into his house; and he made them a feast, and did bake unleavened bread, and they did eat.'); +INSERT INTO t1(docid,words) VALUES(1019004,'But before they lay down, the men of the city, even the men of Sodom, compassed the house round, both old and young, all the people from every quarter:'); +INSERT INTO t1(docid,words) VALUES(1019005,'And they called unto Lot, and said unto him, Where are the men which came in to thee this night? bring them out unto us, that we may know them.'); +INSERT INTO t1(docid,words) VALUES(1019006,'And Lot went out at the door unto them, and shut the door after him,'); +INSERT INTO t1(docid,words) VALUES(1019007,'And said, I pray you, brethren, do not so wickedly.'); +INSERT INTO t1(docid,words) VALUES(1019008,'Behold now, I have two daughters which have not known man; let me, I pray you, bring them out unto you, and do ye to them as is good in your eyes: only unto these men do nothing; for therefore came they under the shadow of my roof.'); +INSERT INTO t1(docid,words) VALUES(1019009,'And they said, Stand back. And they said again, This one fellow came in to sojourn, and he will needs be a judge: now will we deal worse with thee, than with them. And they pressed sore upon the man, even Lot, and came near to break the door.'); +INSERT INTO t1(docid,words) VALUES(1019010,'But the men put forth their hand, and pulled Lot into the house to them, and shut to the door.'); +INSERT INTO t1(docid,words) VALUES(1019011,'And they smote the men that were at the door of the house with blindness, both small and great: so that they wearied themselves to find the door.'); +INSERT INTO t1(docid,words) VALUES(1019012,'And the men said unto Lot, Hast thou here any besides? son in law, and thy sons, and thy daughters, and whatsoever thou hast in the city, bring them out of this place:'); +INSERT INTO t1(docid,words) VALUES(1019013,'For we will destroy this place, because the cry of them is waxen great before the face of the LORD; and the LORD hath sent us to destroy it.'); +INSERT INTO t1(docid,words) VALUES(1019014,'And Lot went out, and spake unto his sons in law, which married his daughters, and said, Up, get you out of this place; for the LORD will destroy this city. But he seemed as one that mocked unto his sons in law.'); +INSERT INTO t1(docid,words) VALUES(1019015,'And when the morning arose, then the angels hastened Lot, saying, Arise, take thy wife, and thy two daughters, which are here; lest thou be consumed in the iniquity of the city.'); +INSERT INTO t1(docid,words) VALUES(1019016,'And while he lingered, the men laid hold upon his hand, and upon the hand of his wife, and upon the hand of his two daughters; the LORD being merciful unto him: and they brought him forth, and set him without the city.'); +INSERT INTO t1(docid,words) VALUES(1019017,'And it came to pass, when they had brought them forth abroad, that he said, Escape for thy life; look not behind thee, neither stay thou in all the plain; escape to the mountain, lest thou be consumed.'); +INSERT INTO t1(docid,words) VALUES(1019018,'And Lot said unto them, Oh, not so, my LORD:'); +INSERT INTO t1(docid,words) VALUES(1019019,'Behold now, thy servant hath found grace in thy sight, and thou hast magnified thy mercy, which thou hast shewed unto me in saving my life; and I cannot escape to the mountain, lest some evil take me, and I die:'); +INSERT INTO t1(docid,words) VALUES(1019020,'Behold now, this city is near to flee unto, and it is a little one: Oh, let me escape thither, (is it not a little one?) and my soul shall live.'); +INSERT INTO t1(docid,words) VALUES(1019021,'And he said unto him, See, I have accepted thee concerning this thing also, that I will not overthrow this city, for the which thou hast spoken.'); +INSERT INTO t1(docid,words) VALUES(1019022,'Haste thee, escape thither; for I cannot do anything till thou be come thither. Therefore the name of the city was called Zoar.'); +INSERT INTO t1(docid,words) VALUES(1019023,'The sun was risen upon the earth when Lot entered into Zoar.'); +INSERT INTO t1(docid,words) VALUES(1019024,'Then the LORD rained upon Sodom and upon Gomorrah brimstone and fire from the LORD out of heaven;'); +INSERT INTO t1(docid,words) VALUES(1019025,'And he overthrew those cities, and all the plain, and all the inhabitants of the cities, and that which grew upon the ground.'); +INSERT INTO t1(docid,words) VALUES(1019026,'But his wife looked back from behind him, and she became a pillar of salt.'); +INSERT INTO t1(docid,words) VALUES(1019027,'And Abraham gat up early in the morning to the place where he stood before the LORD:'); +INSERT INTO t1(docid,words) VALUES(1019028,'And he looked toward Sodom and Gomorrah, and toward all the land of the plain, and beheld, and, lo, the smoke of the country went up as the smoke of a furnace.'); +INSERT INTO t1(docid,words) VALUES(1019029,'And it came to pass, when God destroyed the cities of the plain, that God remembered Abraham, and sent Lot out of the midst of the overthrow, when he overthrew the cities in the which Lot dwelt.'); +INSERT INTO t1(docid,words) VALUES(1019030,'And Lot went up out of Zoar, and dwelt in the mountain, and his two daughters with him; for he feared to dwell in Zoar: and he dwelt in a cave, he and his two daughters.'); +INSERT INTO t1(docid,words) VALUES(1019031,'And the firstborn said unto the younger, Our father is old, and there is not a man in the earth to come in unto us after the manner of all the earth:'); +INSERT INTO t1(docid,words) VALUES(1019032,'Come, let us make our father drink wine, and we will lie with him, that we may preserve seed of our father.'); +INSERT INTO t1(docid,words) VALUES(1019033,'And they made their father drink wine that night: and the firstborn went in, and lay with her father; and he perceived not when she lay down, nor when she arose.'); +INSERT INTO t1(docid,words) VALUES(1019034,'And it came to pass on the morrow, that the firstborn said unto the younger, Behold, I lay yesternight with my father: let us make him drink wine this night also; and go thou in, and lie with him, that we may preserve seed of our father.'); +INSERT INTO t1(docid,words) VALUES(1019035,'And they made their father drink wine that night also: and the younger arose, and lay with him; and he perceived not when she lay down, nor when she arose.'); +INSERT INTO t1(docid,words) VALUES(1019036,'Thus were both the daughters of Lot with child by their father.'); +INSERT INTO t1(docid,words) VALUES(1019037,'And the first born bare a son, and called his name Moab: the same is the father of the Moabites unto this day.'); +INSERT INTO t1(docid,words) VALUES(1019038,'And the younger, she also bare a son, and called his name Benammi: the same is the father of the children of Ammon unto this day.'); +INSERT INTO t1(docid,words) VALUES(1020001,'And Abraham journeyed from thence toward the south country, and dwelled between Kadesh and Shur, and sojourned in Gerar.'); +INSERT INTO t1(docid,words) VALUES(1020002,'And Abraham said of Sarah his wife, She is my sister: and Abimelech king of Gerar sent, and took Sarah.'); +INSERT INTO t1(docid,words) VALUES(1020003,'But God came to Abimelech in a dream by night, and said to him, Behold, thou art but a dead man, for the woman which thou hast taken; for she is a man''s wife.'); +INSERT INTO t1(docid,words) VALUES(1020004,'But Abimelech had not come near her: and he said, LORD, wilt thou slay also a righteous nation?'); +INSERT INTO t1(docid,words) VALUES(1020005,'Said he not unto me, She is my sister? and she, even she herself said, He is my brother: in the integrity of my heart and innocency of my hands have I done this.'); +INSERT INTO t1(docid,words) VALUES(1020006,'And God said unto him in a dream, Yea, I know that thou didst this in the integrity of thy heart; for I also withheld thee from sinning against me: therefore suffered I thee not to touch her.'); +INSERT INTO t1(docid,words) VALUES(1020007,'Now therefore restore the man his wife; for he is a prophet, and he shall pray for thee, and thou shalt live: and if thou restore her not, know thou that thou shalt surely die, thou, and all that are thine.'); +INSERT INTO t1(docid,words) VALUES(1020008,'Therefore Abimelech rose early in the morning, and called all his servants, and told all these things in their ears: and the men were sore afraid.'); +INSERT INTO t1(docid,words) VALUES(1020009,'Then Abimelech called Abraham, and said unto him, What hast thou done unto us? and what have I offended thee, that thou hast brought on me and on my kingdom a great sin? thou hast done deeds unto me that ought not to be done.'); +INSERT INTO t1(docid,words) VALUES(1020010,'And Abimelech said unto Abraham, What sawest thou, that thou hast done this thing?'); +INSERT INTO t1(docid,words) VALUES(1020011,'And Abraham said, Because I thought, Surely the fear of God is not in this place; and they will slay me for my wife''s sake.'); +INSERT INTO t1(docid,words) VALUES(1020012,'And yet indeed she is my sister; she is the daughter of my father, but not the daughter of my mother; and she became my wife.'); +INSERT INTO t1(docid,words) VALUES(1020013,'And it came to pass, when God caused me to wander from my father''s house, that I said unto her, This is thy kindness which thou shalt shew unto me; at every place whither we shall come, say of me, He is my brother.'); +INSERT INTO t1(docid,words) VALUES(1020014,'And Abimelech took sheep, and oxen, and menservants, and womenservants, and gave them unto Abraham, and restored him Sarah his wife.'); +INSERT INTO t1(docid,words) VALUES(1020015,'And Abimelech said, Behold, my land is before thee: dwell where it pleaseth thee.'); +INSERT INTO t1(docid,words) VALUES(1020016,'And unto Sarah he said, Behold, I have given thy brother a thousand pieces of silver: behold, he is to thee a covering of the eyes, unto all that are with thee, and with all other: thus she was reproved.'); +INSERT INTO t1(docid,words) VALUES(1020017,'So Abraham prayed unto God: and God healed Abimelech, and his wife, and his maidservants; and they bare children.'); +INSERT INTO t1(docid,words) VALUES(1020018,'For the LORD had fast closed up all the wombs of the house of Abimelech, because of Sarah Abraham''s wife.'); +INSERT INTO t1(docid,words) VALUES(1021001,'And the LORD visited Sarah as he had said, and the LORD did unto Sarah as he had spoken.'); +INSERT INTO t1(docid,words) VALUES(1021002,'For Sarah conceived, and bare Abraham a son in his old age, at the set time of which God had spoken to him.'); +INSERT INTO t1(docid,words) VALUES(1021003,'And Abraham called the name of his son that was born unto him, whom Sarah bare to him, Isaac.'); +INSERT INTO t1(docid,words) VALUES(1021004,'And Abraham circumcised his son Isaac being eight days old, as God had commanded him.'); +INSERT INTO t1(docid,words) VALUES(1021005,'And Abraham was an hundred years old, when his son Isaac was born unto him.'); +INSERT INTO t1(docid,words) VALUES(1021006,'And Sarah said, God hath made me to laugh, so that all that hear will laugh with me.'); +INSERT INTO t1(docid,words) VALUES(1021007,'And she said, Who would have said unto Abraham, that Sarah should have given children suck? for I have born him a son in his old age.'); +INSERT INTO t1(docid,words) VALUES(1021008,'And the child grew, and was weaned: and Abraham made a great feast the same day that Isaac was weaned.'); +INSERT INTO t1(docid,words) VALUES(1021009,'And Sarah saw the son of Hagar the Egyptian, which she had born unto Abraham, mocking.'); +INSERT INTO t1(docid,words) VALUES(1021010,'Wherefore she said unto Abraham, Cast out this bondwoman and her son: for the son of this bondwoman shall not be heir with my son, even with Isaac.'); +INSERT INTO t1(docid,words) VALUES(1021011,'And the thing was very grievous in Abraham''s sight because of his son.'); +INSERT INTO t1(docid,words) VALUES(1021012,'And God said unto Abraham, Let it not be grievous in thy sight because of the lad, and because of thy bondwoman; in all that Sarah hath said unto thee, hearken unto her voice; for in Isaac shall thy seed be called.'); +INSERT INTO t1(docid,words) VALUES(1021013,'And also of the son of the bondwoman will I make a nation, because he is thy seed.'); +INSERT INTO t1(docid,words) VALUES(1021014,'And Abraham rose up early in the morning, and took bread, and a bottle of water, and gave it unto Hagar, putting it on her shoulder, and the child, and sent her away: and she departed, and wandered in the wilderness of Beersheba.'); +INSERT INTO t1(docid,words) VALUES(1021015,'And the water was spent in the bottle, and she cast the child under one of the shrubs.'); +INSERT INTO t1(docid,words) VALUES(1021016,'And she went, and sat her down over against him a good way off, as it were a bow shot: for she said, Let me not see the death of the child. And she sat over against him, and lift up her voice, and wept.'); +INSERT INTO t1(docid,words) VALUES(1021017,'And God heard the voice of the lad; and the angel of God called to Hagar out of heaven, and said unto her, What aileth thee, Hagar? fear not; for God hath heard the voice of the lad where he is.'); +INSERT INTO t1(docid,words) VALUES(1021018,'Arise, lift up the lad, and hold him in thine hand; for I will make him a great nation.'); +INSERT INTO t1(docid,words) VALUES(1021019,'And God opened her eyes, and she saw a well of water; and she went, and filled the bottle with water, and gave the lad drink.'); +INSERT INTO t1(docid,words) VALUES(1021020,'And God was with the lad; and he grew, and dwelt in the wilderness, and became an archer.'); +INSERT INTO t1(docid,words) VALUES(1021021,'And he dwelt in the wilderness of Paran: and his mother took him a wife out of the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1021022,'And it came to pass at that time, that Abimelech and Phichol the chief captain of his host spake unto Abraham, saying, God is with thee in all that thou doest:'); +INSERT INTO t1(docid,words) VALUES(1021023,'Now therefore swear unto me here by God that thou wilt not deal falsely with me, nor with my son, nor with my son''s son: but according to the kindness that I have done unto thee, thou shalt do unto me, and to the land wherein thou hast sojourned.'); +INSERT INTO t1(docid,words) VALUES(1021024,'And Abraham said, I will swear.'); +INSERT INTO t1(docid,words) VALUES(1021025,'And Abraham reproved Abimelech because of a well of water, which Abimelech''s servants had violently taken away.'); +INSERT INTO t1(docid,words) VALUES(1021026,'And Abimelech said, I wot not who hath done this thing; neither didst thou tell me, neither yet heard I of it, but to day.'); +INSERT INTO t1(docid,words) VALUES(1021027,'And Abraham took sheep and oxen, and gave them unto Abimelech; and both of them made a covenant.'); +INSERT INTO t1(docid,words) VALUES(1021028,'And Abraham set seven ewe lambs of the flock by themselves.'); +INSERT INTO t1(docid,words) VALUES(1021029,'And Abimelech said unto Abraham, What mean these seven ewe lambs which thou hast set by themselves?'); +INSERT INTO t1(docid,words) VALUES(1021030,'And he said, For these seven ewe lambs shalt thou take of my hand, that they may be a witness unto me, that I have digged this well.'); +INSERT INTO t1(docid,words) VALUES(1021031,'Wherefore he called that place Beersheba; because there they sware both of them.'); +INSERT INTO t1(docid,words) VALUES(1021032,'Thus they made a covenant at Beersheba: then Abimelech rose up, and Phichol the chief captain of his host, and they returned into the land of the Philistines.'); +INSERT INTO t1(docid,words) VALUES(1021033,'And Abraham planted a grove in Beersheba, and called there on the name of the LORD, the everlasting God.'); +INSERT INTO t1(docid,words) VALUES(1021034,'And Abraham sojourned in the Philistines'' land many days.'); +INSERT INTO t1(docid,words) VALUES(1022001,'And it came to pass after these things, that God did tempt Abraham, and said unto him, Abraham: and he said, Behold, here I am.'); +INSERT INTO t1(docid,words) VALUES(1022002,'And he said, Take now thy son, thine only son Isaac, whom thou lovest, and get thee into the land of Moriah; and offer him there for a burnt offering upon one of the mountains which I will tell thee of.'); +INSERT INTO t1(docid,words) VALUES(1022003,'And Abraham rose up early in the morning, and saddled his ass, and took two of his young men with him, and Isaac his son, and clave the wood for the burnt offering, and rose up, and went unto the place of which God had told him.'); +INSERT INTO t1(docid,words) VALUES(1022004,'Then on the third day Abraham lifted up his eyes, and saw the place afar off.'); +INSERT INTO t1(docid,words) VALUES(1022005,'And Abraham said unto his young men, Abide ye here with the ass; and I and the lad will go yonder and worship, and come again to you.'); +INSERT INTO t1(docid,words) VALUES(1022006,'And Abraham took the wood of the burnt offering, and laid it upon Isaac his son; and he took the fire in his hand, and a knife; and they went both of them together.'); +INSERT INTO t1(docid,words) VALUES(1022007,'And Isaac spake unto Abraham his father, and said, My father: and he said, Here am I, my son. And he said, Behold the fire and the wood: but where is the lamb for a burnt offering?'); +INSERT INTO t1(docid,words) VALUES(1022008,'And Abraham said, My son, God will provide himself a lamb for a burnt offering: so they went both of them together.'); +INSERT INTO t1(docid,words) VALUES(1022009,'And they came to the place which God had told him of; and Abraham built an altar there, and laid the wood in order, and bound Isaac his son, and laid him on the altar upon the wood.'); +INSERT INTO t1(docid,words) VALUES(1022010,'And Abraham stretched forth his hand, and took the knife to slay his son.'); +INSERT INTO t1(docid,words) VALUES(1022011,'And the angel of the LORD called unto him out of heaven, and said, Abraham, Abraham: and he said, Here am I.'); +INSERT INTO t1(docid,words) VALUES(1022012,'And he said, Lay not thine hand upon the lad, neither do thou any thing unto him: for now I know that thou fearest God, seeing thou hast not withheld thy son, thine only son from me.'); +INSERT INTO t1(docid,words) VALUES(1022013,'And Abraham lifted up his eyes, and looked, and behold behind him a ram caught in a thicket by his horns: and Abraham went and took the ram, and offered him up for a burnt offering in the stead of his son.'); +INSERT INTO t1(docid,words) VALUES(1022014,'And Abraham called the name of that place Jehovahjireh: as it is said to this day, In the mount of the LORD it shall be seen.'); +INSERT INTO t1(docid,words) VALUES(1022015,'And the angel of the LORD called unto Abraham out of heaven the second time,'); +INSERT INTO t1(docid,words) VALUES(1022016,'And said, By myself have I sworn, saith the LORD, for because thou hast done this thing, and hast not withheld thy son, thine only son:'); +INSERT INTO t1(docid,words) VALUES(1022017,'That in blessing I will bless thee, and in multiplying I will multiply thy seed as the stars of the heaven, and as the sand which is upon the sea shore; and thy seed shall possess the gate of his enemies;'); +INSERT INTO t1(docid,words) VALUES(1022018,'And in thy seed shall all the nations of the earth be blessed; because thou hast obeyed my voice.'); +INSERT INTO t1(docid,words) VALUES(1022019,'So Abraham returned unto his young men, and they rose up and went together to Beersheba; and Abraham dwelt at Beersheba.'); +INSERT INTO t1(docid,words) VALUES(1022020,'And it came to pass after these things, that it was told Abraham, saying, Behold, Milcah, she hath also born children unto thy brother Nahor;'); +INSERT INTO t1(docid,words) VALUES(1022021,'Huz his firstborn, and Buz his brother, and Kemuel the father of Aram,'); +INSERT INTO t1(docid,words) VALUES(1022022,'And Chesed, and Hazo, and Pildash, and Jidlaph, and Bethuel.'); +INSERT INTO t1(docid,words) VALUES(1022023,'And Bethuel begat Rebekah: these eight Milcah did bear to Nahor, Abraham''s brother.'); +INSERT INTO t1(docid,words) VALUES(1022024,'And his concubine, whose name was Reumah, she bare also Tebah, and Gaham, and Thahash, and Maachah.'); +INSERT INTO t1(docid,words) VALUES(1023001,'And Sarah was an hundred and seven and twenty years old: these were the years of the life of Sarah.'); +INSERT INTO t1(docid,words) VALUES(1023002,'And Sarah died in Kirjatharba; the same is Hebron in the land of Canaan: and Abraham came to mourn for Sarah, and to weep for her.'); +INSERT INTO t1(docid,words) VALUES(1023003,'And Abraham stood up from before his dead, and spake unto the sons of Heth, saying,'); +INSERT INTO t1(docid,words) VALUES(1023004,'I am a stranger and a sojourner with you: give me a possession of a buryingplace with you, that I may bury my dead out of my sight.'); +INSERT INTO t1(docid,words) VALUES(1023005,'And the children of Heth answered Abraham, saying unto him,'); +INSERT INTO t1(docid,words) VALUES(1023006,'Hear us, my lord: thou art a mighty prince among us: in the choice of our sepulchres bury thy dead; none of us shall withhold from thee his sepulchre, but that thou mayest bury thy dead.'); +INSERT INTO t1(docid,words) VALUES(1023007,'And Abraham stood up, and bowed himself to the people of the land, even to the children of Heth.'); +INSERT INTO t1(docid,words) VALUES(1023008,'And he communed with them, saying, If it be your mind that I should bury my dead out of my sight; hear me, and intreat for me to Ephron the son of Zohar,'); +INSERT INTO t1(docid,words) VALUES(1023009,'That he may give me the cave of Machpelah, which he hath, which is in the end of his field; for as much money as it is worth he shall give it me for a possession of a buryingplace amongst you.'); +INSERT INTO t1(docid,words) VALUES(1023010,'And Ephron dwelt among the children of Heth: and Ephron the Hittite answered Abraham in the audience of the children of Heth, even of all that went in at the gate of his city, saying,'); +INSERT INTO t1(docid,words) VALUES(1023011,'Nay, my lord, hear me: the field give I thee, and the cave that is therein, I give it thee; in the presence of the sons of my people give I it thee: bury thy dead.'); +INSERT INTO t1(docid,words) VALUES(1023012,'And Abraham bowed down himself before the people of the land.'); +INSERT INTO t1(docid,words) VALUES(1023013,'And he spake unto Ephron in the audience of the people of the land, saying, But if thou wilt give it, I pray thee, hear me: I will give thee money for the field; take it of me, and I will bury my dead there.'); +INSERT INTO t1(docid,words) VALUES(1023014,'And Ephron answered Abraham, saying unto him,'); +INSERT INTO t1(docid,words) VALUES(1023015,'My lord, hearken unto me: the land is worth four hundred shekels of silver; what is that betwixt me and thee? bury therefore thy dead.'); +INSERT INTO t1(docid,words) VALUES(1023016,'And Abraham hearkened unto Ephron; and Abraham weighed to Ephron the silver, which he had named in the audience of the sons of Heth, four hundred shekels of silver, current money with the merchant.'); +INSERT INTO t1(docid,words) VALUES(1023017,'And the field of Ephron which was in Machpelah, which was before Mamre, the field, and the cave which was therein, and all the trees that were in the field, that were in all the borders round about, were made sure'); +INSERT INTO t1(docid,words) VALUES(1023018,'Unto Abraham for a possession in the presence of the children of Heth, before all that went in at the gate of his city.'); +INSERT INTO t1(docid,words) VALUES(1023019,'And after this, Abraham buried Sarah his wife in the cave of the field of Machpelah before Mamre: the same is Hebron in the land of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1023020,'And the field, and the cave that is therein, were made sure unto Abraham for a possession of a buryingplace by the sons of Heth.'); +INSERT INTO t1(docid,words) VALUES(1024001,'And Abraham was old, and well stricken in age: and the LORD had blessed Abraham in all things.'); +INSERT INTO t1(docid,words) VALUES(1024002,'And Abraham said unto his eldest servant of his house, that ruled over all that he had, Put, I pray thee, thy hand under my thigh:'); +INSERT INTO t1(docid,words) VALUES(1024003,'And I will make thee swear by the LORD, the God of heaven, and the God of the earth, that thou shalt not take a wife unto my son of the daughters of the Canaanites, among whom I dwell:'); +INSERT INTO t1(docid,words) VALUES(1024004,'But thou shalt go unto my country, and to my kindred, and take a wife unto my son Isaac.'); +INSERT INTO t1(docid,words) VALUES(1024005,'And the servant said unto him, Peradventure the woman will not be willing to follow me unto this land: must I needs bring thy son again unto the land from whence thou camest?'); +INSERT INTO t1(docid,words) VALUES(1024006,'And Abraham said unto him, Beware thou that thou bring not my son thither again.'); +INSERT INTO t1(docid,words) VALUES(1024007,'The LORD God of heaven, which took me from my father''s house, and from the land of my kindred, and which spake unto me, and that sware unto me, saying, Unto thy seed will I give this land; he shall send his angel before thee, and thou shalt take a wife unto my son from thence.'); +INSERT INTO t1(docid,words) VALUES(1024008,'And if the woman will not be willing to follow thee, then thou shalt be clear from this my oath: only bring not my son thither again.'); +INSERT INTO t1(docid,words) VALUES(1024009,'And the servant put his hand under the thigh of Abraham his master, and sware to him concerning that matter.'); +INSERT INTO t1(docid,words) VALUES(1024010,'And the servant took ten camels of the camels of his master, and departed; for all the goods of his master were in his hand: and he arose, and went to Mesopotamia, unto the city of Nahor.'); +INSERT INTO t1(docid,words) VALUES(1024011,'And he made his camels to kneel down without the city by a well of water at the time of the evening, even the time that women go out to draw water.'); +INSERT INTO t1(docid,words) VALUES(1024012,'And he said O LORD God of my master Abraham, I pray thee, send me good speed this day, and shew kindness unto my master Abraham.'); +INSERT INTO t1(docid,words) VALUES(1024013,'Behold, I stand here by the well of water; and the daughters of the men of the city come out to draw water:'); +INSERT INTO t1(docid,words) VALUES(1024014,'And let it come to pass, that the damsel to whom I shall say, Let down thy pitcher, I pray thee, that I may drink; and she shall say, Drink, and I will give thy camels drink also: let the same be she that thou hast appointed for thy servant Isaac; and thereby shall I know that thou hast shewed kindness unto my master.'); +INSERT INTO t1(docid,words) VALUES(1024015,'And it came to pass, before he had done speaking, that, behold, Rebekah came out, who was born to Bethuel, son of Milcah, the wife of Nahor, Abraham''s brother, with her pitcher upon her shoulder.'); +INSERT INTO t1(docid,words) VALUES(1024016,'And the damsel was very fair to look upon, a virgin, neither had any man known her: and she went down to the well, and filled her pitcher, and came up.'); +INSERT INTO t1(docid,words) VALUES(1024017,'And the servant ran to meet her, and said, Let me, I pray thee, drink a little water of thy pitcher.'); +INSERT INTO t1(docid,words) VALUES(1024018,'And she said, Drink, my lord: and she hasted, and let down her pitcher upon her hand, and gave him drink.'); +INSERT INTO t1(docid,words) VALUES(1024019,'And when she had done giving him drink, she said, I will draw water for thy camels also, until they have done drinking.'); +INSERT INTO t1(docid,words) VALUES(1024020,'And she hasted, and emptied her pitcher into the trough, and ran again unto the well to draw water, and drew for all his camels.'); +INSERT INTO t1(docid,words) VALUES(1024021,'And the man wondering at her held his peace, to wit whether the LORD had made his journey prosperous or not.'); +INSERT INTO t1(docid,words) VALUES(1024022,'And it came to pass, as the camels had done drinking, that the man took a golden earring of half a shekel weight, and two bracelets for her hands of ten shekels weight of gold;'); +INSERT INTO t1(docid,words) VALUES(1024023,'And said, Whose daughter art thou? tell me, I pray thee: is there room in thy father''s house for us to lodge in?'); +INSERT INTO t1(docid,words) VALUES(1024024,'And she said unto him, I am the daughter of Bethuel the son of Milcah, which she bare unto Nahor.'); +INSERT INTO t1(docid,words) VALUES(1024025,'She said moreover unto him, We have both straw and provender enough, and room to lodge in.'); +INSERT INTO t1(docid,words) VALUES(1024026,'And the man bowed down his head, and worshipped the LORD.'); +INSERT INTO t1(docid,words) VALUES(1024027,'And he said, Blessed be the LORD God of my master Abraham, who hath not left destitute my master of his mercy and his truth: I being in the way, the LORD led me to the house of my master''s brethren.'); +INSERT INTO t1(docid,words) VALUES(1024028,'And the damsel ran, and told them of her mother''s house these things.'); +INSERT INTO t1(docid,words) VALUES(1024029,'And Rebekah had a brother, and his name was Laban: and Laban ran out unto the man, unto the well.'); +INSERT INTO t1(docid,words) VALUES(1024030,'And it came to pass, when he saw the earring and bracelets upon his sister''s hands, and when he heard the words of Rebekah his sister, saying, Thus spake the man unto me; that he came unto the man; and, behold, he stood by the camels at the well.'); +INSERT INTO t1(docid,words) VALUES(1024031,'And he said, Come in, thou blessed of the LORD; wherefore standest thou without? for I have prepared the house, and room for the camels.'); +INSERT INTO t1(docid,words) VALUES(1024032,'And the man came into the house: and he ungirded his camels, and gave straw and provender for the camels, and water to wash his feet, and the men''s feet that were with him.'); +INSERT INTO t1(docid,words) VALUES(1024033,'And there was set meat before him to eat: but he said, I will not eat, until I have told mine errand. And he said, Speak on.'); +INSERT INTO t1(docid,words) VALUES(1024034,'And he said, I am Abraham''s servant.'); +INSERT INTO t1(docid,words) VALUES(1024035,'And the LORD hath blessed my master greatly; and he is become great: and he hath given him flocks, and herds, and silver, and gold, and menservants, and maidservants, and camels, and asses.'); +INSERT INTO t1(docid,words) VALUES(1024036,'And Sarah my master''s wife bare a son to my master when she was old: and unto him hath he given all that he hath.'); +INSERT INTO t1(docid,words) VALUES(1024037,'And my master made me swear, saying, Thou shalt not take a wife to my son of the daughters of the Canaanites, in whose land I dwell:'); +INSERT INTO t1(docid,words) VALUES(1024038,'But thou shalt go unto my father''s house, and to my kindred, and take a wife unto my son.'); +INSERT INTO t1(docid,words) VALUES(1024039,'And I said unto my master, Peradventure the woman will not follow me.'); +INSERT INTO t1(docid,words) VALUES(1024040,'And he said unto me, The LORD, before whom I walk, will send his angel with thee, and prosper thy way; and thou shalt take a wife for my son of my kindred, and of my father''s house:'); +INSERT INTO t1(docid,words) VALUES(1024041,'Then shalt thou be clear from this my oath, when thou comest to my kindred; and if they give not thee one, thou shalt be clear from my oath.'); +INSERT INTO t1(docid,words) VALUES(1024042,'And I came this day unto the well, and said, O LORD God of my master Abraham, if now thou do prosper my way which I go:'); +INSERT INTO t1(docid,words) VALUES(1024043,'Behold, I stand by the well of water; and it shall come to pass, that when the virgin cometh forth to draw water, and I say to her, Give me, I pray thee, a little water of thy pitcher to drink;'); +INSERT INTO t1(docid,words) VALUES(1024044,'And she say to me, Both drink thou, and I will also draw for thy camels: let the same be the woman whom the LORD hath appointed out for my master''s son.'); +INSERT INTO t1(docid,words) VALUES(1024045,'And before I had done speaking in mine heart, behold, Rebekah came forth with her pitcher on her shoulder; and she went down unto the well, and drew water: and I said unto her, Let me drink, I pray thee.'); +INSERT INTO t1(docid,words) VALUES(1024046,'And she made haste, and let down her pitcher from her shoulder, and said, Drink, and I will give thy camels drink also: so I drank, and she made the camels drink also.'); +INSERT INTO t1(docid,words) VALUES(1024047,'And I asked her, and said, Whose daughter art thou? And she said, the daughter of Bethuel, Nahor''s son, whom Milcah bare unto him: and I put the earring upon her face, and the bracelets upon her hands.'); +INSERT INTO t1(docid,words) VALUES(1024048,'And I bowed down my head, and worshipped the LORD, and blessed the LORD God of my master Abraham, which had led me in the right way to take my master''s brother''s daughter unto his son.'); +INSERT INTO t1(docid,words) VALUES(1024049,'And now if ye will deal kindly and truly with my master, tell me: and if not, tell me; that I may turn to the right hand, or to the left.'); +INSERT INTO t1(docid,words) VALUES(1024050,'Then Laban and Bethuel answered and said, The thing proceedeth from the LORD: we cannot speak unto thee bad or good.'); +INSERT INTO t1(docid,words) VALUES(1024051,'Behold, Rebekah is before thee, take her, and go, and let her be thy master''s son''s wife, as the LORD hath spoken.'); +INSERT INTO t1(docid,words) VALUES(1024052,'And it came to pass, that, when Abraham''s servant heard their words, he worshipped the LORD, bowing himself to the earth.'); +INSERT INTO t1(docid,words) VALUES(1024053,'And the servant brought forth jewels of silver, and jewels of gold, and raiment, and gave them to Rebekah: he gave also to her brother and to her mother precious things.'); +INSERT INTO t1(docid,words) VALUES(1024054,'And they did eat and drink, he and the men that were with him, and tarried all night; and they rose up in the morning, and he said, Send me away unto my master.'); +INSERT INTO t1(docid,words) VALUES(1024055,'And her brother and her mother said, Let the damsel abide with us a few days, at the least ten; after that she shall go.'); +INSERT INTO t1(docid,words) VALUES(1024056,'And he said unto them, Hinder me not, seeing the LORD hath prospered my way; send me away that I may go to my master.'); +INSERT INTO t1(docid,words) VALUES(1024057,'And they said, We will call the damsel, and enquire at her mouth.'); +INSERT INTO t1(docid,words) VALUES(1024058,'And they called Rebekah, and said unto her, Wilt thou go with this man? And she said, I will go.'); +INSERT INTO t1(docid,words) VALUES(1024059,'And they sent away Rebekah their sister, and her nurse, and Abraham''s servant, and his men.'); +INSERT INTO t1(docid,words) VALUES(1024060,'And they blessed Rebekah, and said unto her, Thou art our sister, be thou the mother of thousands of millions, and let thy seed possess the gate of those which hate them.'); +INSERT INTO t1(docid,words) VALUES(1024061,'And Rebekah arose, and her damsels, and they rode upon the camels, and followed the man: and the servant took Rebekah, and went his way.'); +INSERT INTO t1(docid,words) VALUES(1024062,'And Isaac came from the way of the well Lahairoi; for he dwelt in the south country.'); +INSERT INTO t1(docid,words) VALUES(1024063,'And Isaac went out to meditate in the field at the eventide: and he lifted up his eyes, and saw, and, behold, the camels were coming.'); +INSERT INTO t1(docid,words) VALUES(1024064,'And Rebekah lifted up her eyes, and when she saw Isaac, she lighted off the camel.'); +INSERT INTO t1(docid,words) VALUES(1024065,'For she had said unto the servant, What man is this that walketh in the field to meet us? And the servant had said, It is my master: therefore she took a vail, and covered herself.'); +INSERT INTO t1(docid,words) VALUES(1024066,'And the servant told Isaac all things that he had done.'); +INSERT INTO t1(docid,words) VALUES(1024067,'And Isaac brought her into his mother Sarah''s tent, and took Rebekah, and she became his wife; and he loved her: and Isaac was comforted after his mother''s death.'); +INSERT INTO t1(docid,words) VALUES(1025001,'Then again Abraham took a wife, and her name was Keturah.'); +INSERT INTO t1(docid,words) VALUES(1025002,'And she bare him Zimran, and Jokshan, and Medan, and Midian, and Ishbak, and Shuah.'); +INSERT INTO t1(docid,words) VALUES(1025003,'And Jokshan begat Sheba, and Dedan. And the sons of Dedan were Asshurim, and Letushim, and Leummim.'); +INSERT INTO t1(docid,words) VALUES(1025004,'And the sons of Midian; Ephah, and Epher, and Hanoch, and Abidah, and Eldaah. All these were the children of Keturah.'); +INSERT INTO t1(docid,words) VALUES(1025005,'And Abraham gave all that he had unto Isaac.'); +INSERT INTO t1(docid,words) VALUES(1025006,'But unto the sons of the concubines, which Abraham had, Abraham gave gifts, and sent them away from Isaac his son, while he yet lived, eastward, unto the east country.'); +INSERT INTO t1(docid,words) VALUES(1025007,'And these are the days of the years of Abraham''s life which he lived, an hundred threescore and fifteen years.'); +INSERT INTO t1(docid,words) VALUES(1025008,'Then Abraham gave up the ghost, and died in a good old age, an old man, and full of years; and was gathered to his people.'); +INSERT INTO t1(docid,words) VALUES(1025009,'And his sons Isaac and Ishmael buried him in the cave of Machpelah, in the field of Ephron the son of Zohar the Hittite, which is before Mamre;'); +INSERT INTO t1(docid,words) VALUES(1025010,'The field which Abraham purchased of the sons of Heth: there was Abraham buried, and Sarah his wife.'); +INSERT INTO t1(docid,words) VALUES(1025011,'And it came to pass after the death of Abraham, that God blessed his son Isaac; and Isaac dwelt by the well Lahairoi.'); +INSERT INTO t1(docid,words) VALUES(1025012,'Now these are the generations of Ishmael, Abraham''s son, whom Hagar the Egyptian, Sarah''s handmaid, bare unto Abraham:'); +INSERT INTO t1(docid,words) VALUES(1025013,'And these are the names of the sons of Ishmael, by their names, according to their generations: the firstborn of Ishmael, Nebajoth; and Kedar, and Adbeel, and Mibsam,'); +INSERT INTO t1(docid,words) VALUES(1025014,'And Mishma, and Dumah, and Massa,'); +INSERT INTO t1(docid,words) VALUES(1025015,'Hadar, and Tema, Jetur, Naphish, and Kedemah:'); +INSERT INTO t1(docid,words) VALUES(1025016,'These are the sons of Ishmael, and these are their names, by their towns, and by their castles; twelve princes according to their nations.'); +INSERT INTO t1(docid,words) VALUES(1025017,'And these are the years of the life of Ishmael, an hundred and thirty and seven years: and he gave up the ghost and died; and was gathered unto his people.'); +INSERT INTO t1(docid,words) VALUES(1025018,'And they dwelt from Havilah unto Shur, that is before Egypt, as thou goest toward Assyria: and he died in the presence of all his brethren.'); +INSERT INTO t1(docid,words) VALUES(1025019,'And these are the generations of Isaac, Abraham''s son: Abraham begat Isaac:'); +INSERT INTO t1(docid,words) VALUES(1025020,'And Isaac was forty years old when he took Rebekah to wife, the daughter of Bethuel the Syrian of Padanaram, the sister to Laban the Syrian.'); +INSERT INTO t1(docid,words) VALUES(1025021,'And Isaac intreated the LORD for his wife, because she was barren: and the LORD was intreated of him, and Rebekah his wife conceived.'); +INSERT INTO t1(docid,words) VALUES(1025022,'And the children struggled together within her; and she said, If it be so, why am I thus? And she went to enquire of the LORD.'); +INSERT INTO t1(docid,words) VALUES(1025023,'And the LORD said unto her, Two nations are in thy womb, and two manner of people shall be separated from thy bowels; and the one people shall be stronger than the other people; and the elder shall serve the younger.'); +INSERT INTO t1(docid,words) VALUES(1025024,'And when her days to be delivered were fulfilled, behold, there were twins in her womb.'); +INSERT INTO t1(docid,words) VALUES(1025025,'And the first came out red, all over like an hairy garment; and they called his name Esau.'); +INSERT INTO t1(docid,words) VALUES(1025026,'And after that came his brother out, and his hand took hold on Esau''s heel; and his name was called Jacob: and Isaac was threescore years old when she bare them.'); +INSERT INTO t1(docid,words) VALUES(1025027,'And the boys grew: and Esau was a cunning hunter, a man of the field; and Jacob was a plain man, dwelling in tents.'); +INSERT INTO t1(docid,words) VALUES(1025028,'And Isaac loved Esau, because he did eat of his venison: but Rebekah loved Jacob.'); +INSERT INTO t1(docid,words) VALUES(1025029,'And Jacob sod pottage: and Esau came from the field, and he was faint:'); +INSERT INTO t1(docid,words) VALUES(1025030,'And Esau said to Jacob, Feed me, I pray thee, with that same red pottage; for I am faint: therefore was his name called Edom.'); +INSERT INTO t1(docid,words) VALUES(1025031,'And Jacob said, Sell me this day thy birthright.'); +INSERT INTO t1(docid,words) VALUES(1025032,'And Esau said, Behold, I am at the point to die: and what profit shall this birthright do to me?'); +INSERT INTO t1(docid,words) VALUES(1025033,'And Jacob said, Swear to me this day; and he sware unto him: and he sold his birthright unto Jacob.'); +INSERT INTO t1(docid,words) VALUES(1025034,'Then Jacob gave Esau bread and pottage of lentiles; and he did eat and drink, and rose up, and went his way: thus Esau despised his birthright.'); +INSERT INTO t1(docid,words) VALUES(1026001,'And there was a famine in the land, beside the first famine that was in the days of Abraham. And Isaac went unto Abimelech king of the Philistines unto Gerar.'); +INSERT INTO t1(docid,words) VALUES(1026002,'And the LORD appeared unto him, and said, Go not down into Egypt; dwell in the land which I shall tell thee of:'); +INSERT INTO t1(docid,words) VALUES(1026003,'Sojourn in this land, and I will be with thee, and will bless thee; for unto thee, and unto thy seed, I will give all these countries, and I will perform the oath which I sware unto Abraham thy father;'); +INSERT INTO t1(docid,words) VALUES(1026004,'And I will make thy seed to multiply as the stars of heaven, and will give unto thy seed all these countries; and in thy seed shall all the nations of the earth be blessed;'); +INSERT INTO t1(docid,words) VALUES(1026005,'Because that Abraham obeyed my voice, and kept my charge, my commandments, my statutes, and my laws.'); +INSERT INTO t1(docid,words) VALUES(1026006,'And Isaac dwelt in Gerar:'); +INSERT INTO t1(docid,words) VALUES(1026007,'And the men of the place asked him of his wife; and he said, She is my sister: for he feared to say, She is my wife; lest, said he, the men of the place should kill me for Rebekah; because she was fair to look upon.'); +INSERT INTO t1(docid,words) VALUES(1026008,'And it came to pass, when he had been there a long time, that Abimelech king of the Philistines looked out at a window, and saw, and, behold, Isaac was sporting with Rebekah his wife.'); +INSERT INTO t1(docid,words) VALUES(1026009,'And Abimelech called Isaac, and said, Behold, of a surety she is thy wife; and how saidst thou, She is my sister? And Isaac said unto him, Because I said, Lest I die for her.'); +INSERT INTO t1(docid,words) VALUES(1026010,'And Abimelech said, What is this thou hast done unto us? one of the people might lightly have lien with thy wife, and thou shouldest have brought guiltiness upon us.'); +INSERT INTO t1(docid,words) VALUES(1026011,'And Abimelech charged all his people, saying, He that toucheth this man or his wife shall surely be put to death.'); +INSERT INTO t1(docid,words) VALUES(1026012,'Then Isaac sowed in that land, and received in the same year an hundredfold: and the LORD blessed him.'); +INSERT INTO t1(docid,words) VALUES(1026013,'And the man waxed great, and went forward, and grew until he became very great:'); +INSERT INTO t1(docid,words) VALUES(1026014,'For he had possession of flocks, and possession of herds, and great store of servants: and the Philistines envied him.'); +INSERT INTO t1(docid,words) VALUES(1026015,'For all the wells which his father''s servants had digged in the days of Abraham his father, the Philistines had stopped them, and filled them with earth.'); +INSERT INTO t1(docid,words) VALUES(1026016,'And Abimelech said unto Isaac, Go from us; for thou art much mightier than we.'); +INSERT INTO t1(docid,words) VALUES(1026017,'And Isaac departed thence, and pitched his tent in the valley of Gerar, and dwelt there.'); +INSERT INTO t1(docid,words) VALUES(1026018,'And Isaac digged again the wells of water, which they had digged in the days of Abraham his father; for the Philistines had stopped them after the death of Abraham: and he called their names after the names by which his father had called them.'); +INSERT INTO t1(docid,words) VALUES(1026019,'And Isaac''s servants digged in the valley, and found there a well of springing water.'); +INSERT INTO t1(docid,words) VALUES(1026020,'And the herdmen of Gerar did strive with Isaac''s herdmen, saying, The water is ours: and he called the name of the well Esek; because they strove with him.'); +INSERT INTO t1(docid,words) VALUES(1026021,'And they digged another well, and strove for that also: and he called the name of it Sitnah.'); +INSERT INTO t1(docid,words) VALUES(1026022,'And he removed from thence, and digged another well; and for that they strove not: and he called the name of it Rehoboth; and he said, For now the LORD hath made room for us, and we shall be fruitful in the land.'); +INSERT INTO t1(docid,words) VALUES(1026023,'And he went up from thence to Beersheba.'); +INSERT INTO t1(docid,words) VALUES(1026024,'And the LORD appeared unto him the same night, and said, I am the God of Abraham thy father: fear not, for I am with thee, and will bless thee, and multiply thy seed for my servant Abraham''s sake.'); +INSERT INTO t1(docid,words) VALUES(1026025,'And he builded an altar there, and called upon the name of the LORD, and pitched his tent there: and there Isaac''s servants digged a well.'); +INSERT INTO t1(docid,words) VALUES(1026026,'Then Abimelech went to him from Gerar, and Ahuzzath one of his friends, and Phichol the chief captain of his army.'); +INSERT INTO t1(docid,words) VALUES(1026027,'And Isaac said unto them, Wherefore come ye to me, seeing ye hate me, and have sent me away from you?'); +INSERT INTO t1(docid,words) VALUES(1026028,'And they said, We saw certainly that the LORD was with thee: and we said, Let there be now an oath betwixt us, even betwixt us and thee, and let us make a covenant with thee;'); +INSERT INTO t1(docid,words) VALUES(1026029,'That thou wilt do us no hurt, as we have not touched thee, and as we have done unto thee nothing but good, and have sent thee away in peace: thou art now the blessed of the LORD.'); +INSERT INTO t1(docid,words) VALUES(1026030,'And he made them a feast, and they did eat and drink.'); +INSERT INTO t1(docid,words) VALUES(1026031,'And they rose up betimes in the morning, and sware one to another: and Isaac sent them away, and they departed from him in peace.'); +INSERT INTO t1(docid,words) VALUES(1026032,'And it came to pass the same day, that Isaac''s servants came, and told him concerning the well which they had digged, and said unto him, We have found water.'); +INSERT INTO t1(docid,words) VALUES(1026033,'And he called it Shebah: therefore the name of the city is Beersheba unto this day.'); +INSERT INTO t1(docid,words) VALUES(1026034,'And Esau was forty years old when he took to wife Judith the daughter of Beeri the Hittite, and Bashemath the daughter of Elon the Hittite:'); +INSERT INTO t1(docid,words) VALUES(1026035,'Which were a grief of mind unto Isaac and to Rebekah.'); +INSERT INTO t1(docid,words) VALUES(1027001,'And it came to pass, that when Isaac was old, and his eyes were dim, so that he could not see, he called Esau his eldest son, and said unto him, My son: and he said unto him, Behold, here am I.'); +INSERT INTO t1(docid,words) VALUES(1027002,'And he said, Behold now, I am old, I know not the day of my death:'); +INSERT INTO t1(docid,words) VALUES(1027003,'Now therefore take, I pray thee, thy weapons, thy quiver and thy bow, and go out to the field, and take me some venison;'); +INSERT INTO t1(docid,words) VALUES(1027004,'And make me savoury meat, such as I love, and bring it to me, that I may eat; that my soul may bless thee before I die.'); +INSERT INTO t1(docid,words) VALUES(1027005,'And Rebekah heard when Isaac spake to Esau his son. And Esau went to the field to hunt for venison, and to bring it.'); +INSERT INTO t1(docid,words) VALUES(1027006,'And Rebekah spake unto Jacob her son, saying, Behold, I heard thy father speak unto Esau thy brother, saying,'); +INSERT INTO t1(docid,words) VALUES(1027007,'Bring me venison, and make me savoury meat, that I may eat, and bless thee before the LORD before my death.'); +INSERT INTO t1(docid,words) VALUES(1027008,'Now therefore, my son, obey my voice according to that which I command thee.'); +INSERT INTO t1(docid,words) VALUES(1027009,'Go now to the flock, and fetch me from thence two good kids of the goats; and I will make them savoury meat for thy father, such as he loveth:'); +INSERT INTO t1(docid,words) VALUES(1027010,'And thou shalt bring it to thy father, that he may eat, and that he may bless thee before his death.'); +INSERT INTO t1(docid,words) VALUES(1027011,'And Jacob said to Rebekah his mother, Behold, Esau my brother is a hairy man, and I am a smooth man:'); +INSERT INTO t1(docid,words) VALUES(1027012,'My father peradventure will feel me, and I shall seem to him as a deceiver; and I shall bring a curse upon me, and not a blessing.'); +INSERT INTO t1(docid,words) VALUES(1027013,'And his mother said unto him, Upon me be thy curse, my son: only obey my voice, and go fetch me them.'); +INSERT INTO t1(docid,words) VALUES(1027014,'And he went, and fetched, and brought them to his mother: and his mother made savoury meat, such as his father loved.'); +INSERT INTO t1(docid,words) VALUES(1027015,'And Rebekah took goodly raiment of her eldest son Esau, which were with her in the house, and put them upon Jacob her younger son:'); +INSERT INTO t1(docid,words) VALUES(1027016,'And she put the skins of the kids of the goats upon his hands, and upon the smooth of his neck:'); +INSERT INTO t1(docid,words) VALUES(1027017,'And she gave the savoury meat and the bread, which she had prepared, into the hand of her son Jacob.'); +INSERT INTO t1(docid,words) VALUES(1027018,'And he came unto his father, and said, My father: and he said, Here am I; who art thou, my son?'); +INSERT INTO t1(docid,words) VALUES(1027019,'And Jacob said unto his father, I am Esau thy first born; I have done according as thou badest me: arise, I pray thee, sit and eat of my venison, that thy soul may bless me.'); +INSERT INTO t1(docid,words) VALUES(1027020,'And Isaac said unto his son, How is it that thou hast found it so quickly, my son? And he said, Because the LORD thy God brought it to me.'); +INSERT INTO t1(docid,words) VALUES(1027021,'And Isaac said unto Jacob, Come near, I pray thee, that I may feel thee, my son, whether thou be my very son Esau or not.'); +INSERT INTO t1(docid,words) VALUES(1027022,'And Jacob went near unto Isaac his father; and he felt him, and said, The voice is Jacob''s voice, but the hands are the hands of Esau.'); +INSERT INTO t1(docid,words) VALUES(1027023,'And he discerned him not, because his hands were hairy, as his brother Esau''s hands: so he blessed him.'); +INSERT INTO t1(docid,words) VALUES(1027024,'And he said, Art thou my very son Esau? And he said, I am.'); +INSERT INTO t1(docid,words) VALUES(1027025,'And he said, Bring it near to me, and I will eat of my son''s venison, that my soul may bless thee. And he brought it near to him, and he did eat: and he brought him wine and he drank.'); +INSERT INTO t1(docid,words) VALUES(1027026,'And his father Isaac said unto him, Come near now, and kiss me, my son.'); +INSERT INTO t1(docid,words) VALUES(1027027,'And he came near, and kissed him: and he smelled the smell of his raiment, and blessed him, and said, See, the smell of my son is as the smell of a field which the LORD hath blessed:'); +INSERT INTO t1(docid,words) VALUES(1027028,'Therefore God give thee of the dew of heaven, and the fatness of the earth, and plenty of corn and wine:'); +INSERT INTO t1(docid,words) VALUES(1027029,'Let people serve thee, and nations bow down to thee: be lord over thy brethren, and let thy mother''s sons bow down to thee: cursed be every one that curseth thee, and blessed be he that blesseth thee.'); +INSERT INTO t1(docid,words) VALUES(1027030,'And it came to pass, as soon as Isaac had made an end of blessing Jacob, and Jacob was yet scarce gone out from the presence of Isaac his father, that Esau his brother came in from his hunting.'); +INSERT INTO t1(docid,words) VALUES(1027031,'And he also had made savoury meat, and brought it unto his father, and said unto his father, Let my father arise, and eat of his son''s venison, that thy soul may bless me.'); +INSERT INTO t1(docid,words) VALUES(1027032,'And Isaac his father said unto him, Who art thou? And he said, I am thy son, thy firstborn Esau.'); +INSERT INTO t1(docid,words) VALUES(1027033,'And Isaac trembled very exceedingly, and said, Who? where is he that hath taken venison, and brought it me, and I have eaten of all before thou camest, and have blessed him? yea, and he shall be blessed.'); +INSERT INTO t1(docid,words) VALUES(1027034,'And when Esau heard the words of his father, he cried with a great and exceeding bitter cry, and said unto his father, Bless me, even me also, O my father.'); +INSERT INTO t1(docid,words) VALUES(1027035,'And he said, Thy brother came with subtilty, and hath taken away thy blessing.'); +INSERT INTO t1(docid,words) VALUES(1027036,'And he said, Is not he rightly named Jacob? for he hath supplanted me these two times: he took away my birthright; and, behold, now he hath taken away my blessing. And he said, Hast thou not reserved a blessing for me?'); +INSERT INTO t1(docid,words) VALUES(1027037,'And Isaac answered and said unto Esau, Behold, I have made him thy lord, and all his brethren have I given to him for servants; and with corn and wine have I sustained him: and what shall I do now unto thee, my son?'); +INSERT INTO t1(docid,words) VALUES(1027038,'And Esau said unto his father, Hast thou but one blessing, my father? bless me, even me also, O my father. And Esau lifted up his voice, and wept.'); +INSERT INTO t1(docid,words) VALUES(1027039,'And Isaac his father answered and said unto him, Behold, thy dwelling shall be the fatness of the earth, and of the dew of heaven from above;'); +INSERT INTO t1(docid,words) VALUES(1027040,'And by thy sword shalt thou live, and shalt serve thy brother; and it shall come to pass when thou shalt have the dominion, that thou shalt break his yoke from off thy neck.'); +INSERT INTO t1(docid,words) VALUES(1027041,'And Esau hated Jacob because of the blessing wherewith his father blessed him: and Esau said in his heart, The days of mourning for my father are at hand; then will I slay my brother Jacob.'); +INSERT INTO t1(docid,words) VALUES(1027042,'And these words of Esau her elder son were told to Rebekah: and she sent and called Jacob her younger son, and said unto him, Behold, thy brother Esau, as touching thee, doth comfort himself, purposing to kill thee.'); +INSERT INTO t1(docid,words) VALUES(1027043,'Now therefore, my son, obey my voice; arise, flee thou to Laban my brother to Haran;'); +INSERT INTO t1(docid,words) VALUES(1027044,'And tarry with him a few days, until thy brother''s fury turn away;'); +INSERT INTO t1(docid,words) VALUES(1027045,'Until thy brother''s anger turn away from thee, and he forget that which thou hast done to him: then I will send, and fetch thee from thence: why should I be deprived also of you both in one day?'); +INSERT INTO t1(docid,words) VALUES(1027046,'And Rebekah said to Isaac, I am weary of my life because of the daughters of Heth: if Jacob take a wife of the daughters of Heth, such as these which are of the daughters of the land, what good shall my life do me?'); +INSERT INTO t1(docid,words) VALUES(1028001,'And Isaac called Jacob, and blessed him, and charged him, and said unto him, Thou shalt not take a wife of the daughters of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1028002,'Arise, go to Padanaram, to the house of Bethuel thy mother''s father; and take thee a wife from thence of the daughers of Laban thy mother''s brother.'); +INSERT INTO t1(docid,words) VALUES(1028003,'And God Almighty bless thee, and make thee fruitful, and multiply thee, that thou mayest be a multitude of people;'); +INSERT INTO t1(docid,words) VALUES(1028004,'And give thee the blessing of Abraham, to thee, and to thy seed with thee; that thou mayest inherit the land wherein thou art a stranger, which God gave unto Abraham.'); +INSERT INTO t1(docid,words) VALUES(1028005,'And Isaac sent away Jacob: and he went to Padanaram unto Laban, son of Bethuel the Syrian, the brother of Rebekah, Jacob''s and Esau''s mother.'); +INSERT INTO t1(docid,words) VALUES(1028006,'When Esau saw that Isaac had blessed Jacob, and sent him away to Padanaram, to take him a wife from thence; and that as he blessed him he gave him a charge, saying, Thou shalt not take a wife of the daughers of Canaan;'); +INSERT INTO t1(docid,words) VALUES(1028007,'And that Jacob obeyed his father and his mother, and was gone to Padanaram;'); +INSERT INTO t1(docid,words) VALUES(1028008,'And Esau seeing that the daughters of Canaan pleased not Isaac his father;'); +INSERT INTO t1(docid,words) VALUES(1028009,'Then went Esau unto Ishmael, and took unto the wives which he had Mahalath the daughter of Ishmael Abraham''s son, the sister of Nebajoth, to be his wife.'); +INSERT INTO t1(docid,words) VALUES(1028010,'And Jacob went out from Beersheba, and went toward Haran.'); +INSERT INTO t1(docid,words) VALUES(1028011,'And he lighted upon a certain place, and tarried there all night, because the sun was set; and he took of the stones of that place, and put them for his pillows, and lay down in that place to sleep.'); +INSERT INTO t1(docid,words) VALUES(1028012,'And he dreamed, and behold a ladder set up on the earth, and the top of it reached to heaven: and behold the angels of God ascending and descending on it.'); +INSERT INTO t1(docid,words) VALUES(1028013,'And, behold, the LORD stood above it, and said, I am the LORD God of Abraham thy father, and the God of Isaac: the land whereon thou liest, to thee will I give it, and to thy seed;'); +INSERT INTO t1(docid,words) VALUES(1028014,'And thy seed shall be as the dust of the earth, and thou shalt spread abroad to the west, and to the east, and to the north, and to the south: and in thee and in thy seed shall all the families of the earth be blessed.'); +INSERT INTO t1(docid,words) VALUES(1028015,'And, behold, I am with thee, and will keep thee in all places whither thou goest, and will bring thee again into this land; for I will not leave thee, until I have done that which I have spoken to thee of.'); +INSERT INTO t1(docid,words) VALUES(1028016,'And Jacob awaked out of his sleep, and he said, Surely the LORD is in this place; and I knew it not.'); +INSERT INTO t1(docid,words) VALUES(1028017,'And he was afraid, and said, How dreadful is this place! this is none other but the house of God, and this is the gate of heaven.'); +INSERT INTO t1(docid,words) VALUES(1028018,'And Jacob rose up early in the morning, and took the stone that he had put for his pillows, and set it up for a pillar, and poured oil upon the top of it.'); +INSERT INTO t1(docid,words) VALUES(1028019,'And he called the name of that place Bethel: but the name of that city was called Luz at the first.'); +INSERT INTO t1(docid,words) VALUES(1028020,'And Jacob vowed a vow, saying, If God will be with me, and will keep me in this way that I go, and will give me bread to eat, and raiment to put on,'); +INSERT INTO t1(docid,words) VALUES(1028021,'So that I come again to my father''s house in peace; then shall the LORD be my God:'); +INSERT INTO t1(docid,words) VALUES(1028022,'And this stone, which I have set for a pillar, shall be God''s house: and of all that thou shalt give me I will surely give the tenth unto thee.'); +INSERT INTO t1(docid,words) VALUES(1029001,'Then Jacob went on his journey, and came into the land of the people of the east.'); +INSERT INTO t1(docid,words) VALUES(1029002,'And he looked, and behold a well in the field, and, lo, there were three flocks of sheep lying by it; for out of that well they watered the flocks: and a great stone was upon the well''s mouth.'); +INSERT INTO t1(docid,words) VALUES(1029003,'And thither were all the flocks gathered: and they rolled the stone from the well''s mouth, and watered the sheep, and put the stone again upon the well''s mouth in his place.'); +INSERT INTO t1(docid,words) VALUES(1029004,'And Jacob said unto them, My brethren, whence be ye? And they said, Of Haran are we.'); +INSERT INTO t1(docid,words) VALUES(1029005,'And he said unto them, Know ye Laban the son of Nahor? And they said, We know him.'); +INSERT INTO t1(docid,words) VALUES(1029006,'And he said unto them, Is he well? And they said, He is well: and, behold, Rachel his daughter cometh with the sheep.'); +INSERT INTO t1(docid,words) VALUES(1029007,'And he said, Lo, it is yet high day, neither is it time that the cattle should be gathered together: water ye the sheep, and go and feed them.'); +INSERT INTO t1(docid,words) VALUES(1029008,'And they said, We cannot, until all the flocks be gathered together, and till they roll the stone from the well''s mouth; then we water the sheep.'); +INSERT INTO t1(docid,words) VALUES(1029009,'And while he yet spake with them, Rachel came with her father''s sheep; for she kept them.'); +INSERT INTO t1(docid,words) VALUES(1029010,'And it came to pass, when Jacob saw Rachel the daughter of Laban his mother''s brother, and the sheep of Laban his mother''s brother, that Jacob went near, and rolled the stone from the well''s mouth, and watered the flock of Laban his mother''s brother.'); +INSERT INTO t1(docid,words) VALUES(1029011,'And Jacob kissed Rachel, and lifted up his voice, and wept.'); +INSERT INTO t1(docid,words) VALUES(1029012,'And Jacob told Rachel that he was her father''s brother, and that he was Rebekah''s son: and she ran and told her father.'); +INSERT INTO t1(docid,words) VALUES(1029013,'And it came to pass, when Laban heard the tidings of Jacob his sister''s son, that he ran to meet him, and embraced him, and kissed him, and brought him to his house. And he told Laban all these things.'); +INSERT INTO t1(docid,words) VALUES(1029014,'And Laban said to him, Surely thou art my bone and my flesh. And he abode with him the space of a month.'); +INSERT INTO t1(docid,words) VALUES(1029015,'And Laban said unto Jacob, Because thou art my brother, shouldest thou therefore serve me for nought? tell me, what shall thy wages be?'); +INSERT INTO t1(docid,words) VALUES(1029016,'And Laban had two daughters: the name of the elder was Leah, and the name of the younger was Rachel.'); +INSERT INTO t1(docid,words) VALUES(1029017,'Leah was tender eyed; but Rachel was beautiful and well favoured.'); +INSERT INTO t1(docid,words) VALUES(1029018,'And Jacob loved Rachel; and said, I will serve thee seven years for Rachel thy younger daughter.'); +INSERT INTO t1(docid,words) VALUES(1029019,'And Laban said, It is better that I give her to thee, than that I should give her to another man: abide with me.'); +INSERT INTO t1(docid,words) VALUES(1029020,'And Jacob served seven years for Rachel; and they seemed unto him but a few days, for the love he had to her.'); +INSERT INTO t1(docid,words) VALUES(1029021,'And Jacob said unto Laban, Give me my wife, for my days are fulfilled, that I may go in unto her.'); +INSERT INTO t1(docid,words) VALUES(1029022,'And Laban gathered together all the men of the place, and made a feast.'); +INSERT INTO t1(docid,words) VALUES(1029023,'And it came to pass in the evening, that he took Leah his daughter, and brought her to him; and he went in unto her.'); +INSERT INTO t1(docid,words) VALUES(1029024,'And Laban gave unto his daughter Leah Zilpah his maid for an handmaid.'); +INSERT INTO t1(docid,words) VALUES(1029025,'And it came to pass, that in the morning, behold, it was Leah: and he said to Laban, What is this thou hast done unto me? did not I serve with thee for Rachel? wherefore then hast thou beguiled me?'); +INSERT INTO t1(docid,words) VALUES(1029026,'And Laban said, It must not be so done in our country, to give the younger before the firstborn.'); +INSERT INTO t1(docid,words) VALUES(1029027,'Fulfil her week, and we will give thee this also for the service which thou shalt serve with me yet seven other years.'); +INSERT INTO t1(docid,words) VALUES(1029028,'And Jacob did so, and fulfilled her week: and he gave him Rachel his daughter to wife also.'); +INSERT INTO t1(docid,words) VALUES(1029029,'And Laban gave to Rachel his daughter Bilhah his handmaid to be her maid.'); +INSERT INTO t1(docid,words) VALUES(1029030,'And he went in also unto Rachel, and he loved also Rachel more than Leah, and served with him yet seven other years.'); +INSERT INTO t1(docid,words) VALUES(1029031,'And when the LORD saw that Leah was hated, he opened her womb: but Rachel was barren.'); +INSERT INTO t1(docid,words) VALUES(1029032,'And Leah conceived, and bare a son, and she called his name Reuben: for she said, Surely the LORD hath looked upon my affliction; now therefore my husband will love me.'); +INSERT INTO t1(docid,words) VALUES(1029033,'And she conceived again, and bare a son; and said, Because the LORD hath heard I was hated, he hath therefore given me this son also: and she called his name Simeon.'); +INSERT INTO t1(docid,words) VALUES(1029034,'And she conceived again, and bare a son; and said, Now this time will my husband be joined unto me, because I have born him three sons: therefore was his name called Levi.'); +INSERT INTO t1(docid,words) VALUES(1029035,'And she conceived again, and bare a son: and she said, Now will I praise the LORD: therefore she called his name Judah; and left bearing.'); +INSERT INTO t1(docid,words) VALUES(1030001,'And when Rachel saw that she bare Jacob no children, Rachel envied her sister; and said unto Jacob, Give me children, or else I die.'); +INSERT INTO t1(docid,words) VALUES(1030002,'And Jacob''s anger was kindled against Rachel: and he said, Am I in God''s stead, who hath withheld from thee the fruit of the womb?'); +INSERT INTO t1(docid,words) VALUES(1030003,'And she said, Behold my maid Bilhah, go in unto her; and she shall bear upon my knees, that I may also have children by her.'); +INSERT INTO t1(docid,words) VALUES(1030004,'And she gave him Bilhah her handmaid to wife: and Jacob went in unto her.'); +INSERT INTO t1(docid,words) VALUES(1030005,'And Bilhah conceived, and bare Jacob a son.'); +INSERT INTO t1(docid,words) VALUES(1030006,'And Rachel said, God hath judged me, and hath also heard my voice, and hath given me a son: therefore called she his name Dan.'); +INSERT INTO t1(docid,words) VALUES(1030007,'And Bilhah Rachel''s maid conceived again, and bare Jacob a second son.'); +INSERT INTO t1(docid,words) VALUES(1030008,'And Rachel said, With great wrestlings have I wrestled with my sister, and I have prevailed: and she called his name Naphtali.'); +INSERT INTO t1(docid,words) VALUES(1030009,'When Leah saw that she had left bearing, she took Zilpah her maid, and gave her Jacob to wife.'); +INSERT INTO t1(docid,words) VALUES(1030010,'And Zilpah Leah''s maid bare Jacob a son.'); +INSERT INTO t1(docid,words) VALUES(1030011,'And Leah said, A troop cometh: and she called his name Gad.'); +INSERT INTO t1(docid,words) VALUES(1030012,'And Zilpah Leah''s maid bare Jacob a second son.'); +INSERT INTO t1(docid,words) VALUES(1030013,'And Leah said, Happy am I, for the daughters will call me blessed: and she called his name Asher.'); +INSERT INTO t1(docid,words) VALUES(1030014,'And Reuben went in the days of wheat harvest, and found mandrakes in the field, and brought them unto his mother Leah. Then Rachel said to Leah, Give me, I pray thee, of thy son''s mandrakes.'); +INSERT INTO t1(docid,words) VALUES(1030015,'And she said unto her, Is it a small matter that thou hast taken my husband? and wouldest thou take away my son''s mandrakes also? And Rachel said, Therefore he shall lie with thee to night for thy son''s mandrakes.'); +INSERT INTO t1(docid,words) VALUES(1030016,'And Jacob came out of the field in the evening, and Leah went out to meet him, and said, Thou must come in unto me; for surely I have hired thee with my son''s mandrakes. And he lay with her that night.'); +INSERT INTO t1(docid,words) VALUES(1030017,'And God hearkened unto Leah, and she conceived, and bare Jacob the fifth son.'); +INSERT INTO t1(docid,words) VALUES(1030018,'And Leah said, God hath given me my hire, because I have given my maiden to my husband: and she called his name Issachar.'); +INSERT INTO t1(docid,words) VALUES(1030019,'And Leah conceived again, and bare Jacob the sixth son.'); +INSERT INTO t1(docid,words) VALUES(1030020,'And Leah said, God hath endued me with a good dowry; now will my husband dwell with me, because I have born him six sons: and she called his name Zebulun.'); +INSERT INTO t1(docid,words) VALUES(1030021,'And afterwards she bare a daughter, and called her name Dinah.'); +INSERT INTO t1(docid,words) VALUES(1030022,'And God remembered Rachel, and God hearkened to her, and opened her womb.'); +INSERT INTO t1(docid,words) VALUES(1030023,'And she conceived, and bare a son; and said, God hath taken away my reproach:'); +INSERT INTO t1(docid,words) VALUES(1030024,'And she called his name Joseph; and said, The LORD shall add to me another son.'); +INSERT INTO t1(docid,words) VALUES(1030025,'And it came to pass, when Rachel had born Joseph, that Jacob said unto Laban, Send me away, that I may go unto mine own place, and to my country.'); +INSERT INTO t1(docid,words) VALUES(1030026,'Give me my wives and my children, for whom I have served thee, and let me go: for thou knowest my service which I have done thee.'); +INSERT INTO t1(docid,words) VALUES(1030027,'And Laban said unto him, I pray thee, if I have found favour in thine eyes, tarry: for I have learned by experience that the LORD hath blessed me for thy sake.'); +INSERT INTO t1(docid,words) VALUES(1030028,'And he said, Appoint me thy wages, and I will give it.'); +INSERT INTO t1(docid,words) VALUES(1030029,'And he said unto him, Thou knowest how I have served thee, and how thy cattle was with me.'); +INSERT INTO t1(docid,words) VALUES(1030030,'For it was little which thou hadst before I came, and it is now increased unto a multitude; and the LORD hath blessed thee since my coming: and now when shall I provide for mine own house also?'); +INSERT INTO t1(docid,words) VALUES(1030031,'And he said, What shall I give thee? And Jacob said, Thou shalt not give me any thing: if thou wilt do this thing for me, I will again feed and keep thy flock.'); +INSERT INTO t1(docid,words) VALUES(1030032,'I will pass through all thy flock to day, removing from thence all the speckled and spotted cattle, and all the brown cattle among the sheep, and the spotted and speckled among the goats: and of such shall be my hire.'); +INSERT INTO t1(docid,words) VALUES(1030033,'So shall my righteousness answer for me in time to come, when it shall come for my hire before thy face: every one that is not speckled and spotted among the goats, and brown among the sheep, that shall be counted stolen with me.'); +INSERT INTO t1(docid,words) VALUES(1030034,'And Laban said, Behold, I would it might be according to thy word.'); +INSERT INTO t1(docid,words) VALUES(1030035,'And he removed that day the he goats that were ringstraked and spotted, and all the she goats that were speckled and spotted, and every one that had some white in it, and all the brown among the sheep, and gave them into the hand of his sons.'); +INSERT INTO t1(docid,words) VALUES(1030036,'And he set three days'' journey betwixt himself and Jacob: and Jacob fed the rest of Laban''s flocks.'); +INSERT INTO t1(docid,words) VALUES(1030037,'And Jacob took him rods of green poplar, and of the hazel and chesnut tree; and pilled white strakes in them, and made the white appear which was in the rods.'); +INSERT INTO t1(docid,words) VALUES(1030038,'And he set the rods which he had pilled before the flocks in the gutters in the watering troughs when the flocks came to drink, that they should conceive when they came to drink.'); +INSERT INTO t1(docid,words) VALUES(1030039,'And the flocks conceived before the rods, and brought forth cattle ringstraked, speckled, and spotted.'); +INSERT INTO t1(docid,words) VALUES(1030040,'And Jacob did separate the lambs, and set the faces of the flocks toward the ringstraked, and all the brown in the flock of Laban; and he put his own flocks by themselves, and put them not unto Laban''s cattle.'); +INSERT INTO t1(docid,words) VALUES(1030041,'And it came to pass, whensoever the stronger cattle did conceive, that Jacob laid the rods before the eyes of the cattle in the gutters, that they might conceive among the rods.'); +INSERT INTO t1(docid,words) VALUES(1030042,'But when the cattle were feeble, he put them not in: so the feebler were Laban''s, and the stronger Jacob''s.'); +INSERT INTO t1(docid,words) VALUES(1030043,'And the man increased exceedingly, and had much cattle, and maidservants, and menservants, and camels, and asses.'); +INSERT INTO t1(docid,words) VALUES(1031001,'And he heard the words of Laban''s sons, saying, Jacob hath taken away all that was our father''s; and of that which was our father''s hath he gotten all this glory.'); +INSERT INTO t1(docid,words) VALUES(1031002,'And Jacob beheld the countenance of Laban, and, behold, it was not toward him as before.'); +INSERT INTO t1(docid,words) VALUES(1031003,'And the LORD said unto Jacob, Return unto the land of thy fathers, and to thy kindred; and I will be with thee.'); +INSERT INTO t1(docid,words) VALUES(1031004,'And Jacob sent and called Rachel and Leah to the field unto his flock,'); +INSERT INTO t1(docid,words) VALUES(1031005,'And said unto them, I see your father''s countenance, that it is not toward me as before; but the God of my father hath been with me.'); +INSERT INTO t1(docid,words) VALUES(1031006,'And ye know that with all my power I have served your father.'); +INSERT INTO t1(docid,words) VALUES(1031007,'And your father hath deceived me, and changed my wages ten times; but God suffered him not to hurt me.'); +INSERT INTO t1(docid,words) VALUES(1031008,'If he said thus, The speckled shall be thy wages; then all the cattle bare speckled: and if he said thus, The ringstraked shall be thy hire; then bare all the cattle ringstraked.'); +INSERT INTO t1(docid,words) VALUES(1031009,'Thus God hath taken away the cattle of your father, and given them to me.'); +INSERT INTO t1(docid,words) VALUES(1031010,'And it came to pass at the time that the cattle conceived, that I lifted up mine eyes, and saw in a dream, and, behold, the rams which leaped upon the cattle were ringstraked, speckled, and grisled.'); +INSERT INTO t1(docid,words) VALUES(1031011,'And the angel of God spake unto me in a dream, saying, Jacob: And I said, Here am I.'); +INSERT INTO t1(docid,words) VALUES(1031012,'And he said, Lift up now thine eyes, and see, all the rams which leap upon the cattle are ringstraked, speckled, and grisled: for I have seen all that Laban doeth unto thee.'); +INSERT INTO t1(docid,words) VALUES(1031013,'I am the God of Bethel, where thou anointedst the pillar, and where thou vowedst a vow unto me: now arise, get thee out from this land, and return unto the land of thy kindred.'); +INSERT INTO t1(docid,words) VALUES(1031014,'And Rachel and Leah answered and said unto him, Is there yet any portion or inheritance for us in our father''s house?'); +INSERT INTO t1(docid,words) VALUES(1031015,'Are we not counted of him strangers? for he hath sold us, and hath quite devoured also our money.'); +INSERT INTO t1(docid,words) VALUES(1031016,'For all the riches which God hath taken from our father, that is ours, and our children''s: now then, whatsoever God hath said unto thee, do.'); +INSERT INTO t1(docid,words) VALUES(1031017,'Then Jacob rose up, and set his sons and his wives upon camels;'); +INSERT INTO t1(docid,words) VALUES(1031018,'And he carried away all his cattle, and all his goods which he had gotten, the cattle of his getting, which he had gotten in Padanaram, for to go to Isaac his father in the land of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1031019,'And Laban went to shear his sheep: and Rachel had stolen the images that were her father''s.'); +INSERT INTO t1(docid,words) VALUES(1031020,'And Jacob stole away unawares to Laban the Syrian, in that he told him not that he fled.'); +INSERT INTO t1(docid,words) VALUES(1031021,'So he fled with all that he had; and he rose up, and passed over the river, and set his face toward the mount Gilead.'); +INSERT INTO t1(docid,words) VALUES(1031022,'And it was told Laban on the third day that Jacob was fled.'); +INSERT INTO t1(docid,words) VALUES(1031023,'And he took his brethren with him, and pursued after him seven days'' journey; and they overtook him in the mount Gilead.'); +INSERT INTO t1(docid,words) VALUES(1031024,'And God came to Laban the Syrian in a dream by night, and said unto him, Take heed that thou speak not to Jacob either good or bad.'); +INSERT INTO t1(docid,words) VALUES(1031025,'Then Laban overtook Jacob. Now Jacob had pitched his tent in the mount: and Laban with his brethren pitched in the mount of Gilead.'); +INSERT INTO t1(docid,words) VALUES(1031026,'And Laban said to Jacob, What hast thou done, that thou hast stolen away unawares to me, and carried away my daughters, as captives taken with the sword?'); +INSERT INTO t1(docid,words) VALUES(1031027,'Wherefore didst thou flee away secretly, and steal away from me; and didst not tell me, that I might have sent thee away with mirth, and with songs, with tabret, and with harp?'); +INSERT INTO t1(docid,words) VALUES(1031028,'And hast not suffered me to kiss my sons and my daughters? thou hast now done foolishly in so doing.'); +INSERT INTO t1(docid,words) VALUES(1031029,'It is in the power of my hand to do you hurt: but the God of your father spake unto me yesternight, saying, Take thou heed that thou speak not to Jacob either good or bad.'); +INSERT INTO t1(docid,words) VALUES(1031030,'And now, though thou wouldest needs be gone, because thou sore longedst after thy father''s house, yet wherefore hast thou stolen my gods?'); +INSERT INTO t1(docid,words) VALUES(1031031,'And Jacob answered and said to Laban, Because I was afraid: for I said, Peradventure thou wouldest take by force thy daughters from me.'); +INSERT INTO t1(docid,words) VALUES(1031032,'With whomsoever thou findest thy gods, let him not live: before our brethren discern thou what is thine with me, and take it to thee. For Jacob knew not that Rachel had stolen them.'); +INSERT INTO t1(docid,words) VALUES(1031033,'And Laban went into Jacob''s tent, and into Leah''s tent, and into the two maidservants'' tents; but he found them not. Then went he out of Leah''s tent, and entered into Rachel''s tent.'); +INSERT INTO t1(docid,words) VALUES(1031034,'Now Rachel had taken the images, and put them in the camel''s furniture, and sat upon them. And Laban searched all the tent, but found them not.'); +INSERT INTO t1(docid,words) VALUES(1031035,'And she said to her father, Let it not displease my lord that I cannot rise up before thee; for the custom of women is upon me. And he searched but found not the images.'); +INSERT INTO t1(docid,words) VALUES(1031036,'And Jacob was wroth, and chode with Laban: and Jacob answered and said to Laban, What is my trespass? what is my sin, that thou hast so hotly pursued after me?'); +INSERT INTO t1(docid,words) VALUES(1031037,'Whereas thou hast searched all my stuff, what hast thou found of all thy household stuff? set it here before my brethren and thy brethren, that they may judge betwixt us both.'); +INSERT INTO t1(docid,words) VALUES(1031038,'This twenty years have I been with thee; thy ewes and thy she goats have not cast their young, and the rams of thy flock have I not eaten.'); +INSERT INTO t1(docid,words) VALUES(1031039,'That which was torn of beasts I brought not unto thee; I bare the loss of it; of my hand didst thou require it, whether stolen by day, or stolen by night.'); +INSERT INTO t1(docid,words) VALUES(1031040,'Thus I was; in the day the drought consumed me, and the frost by night; and my sleep departed from mine eyes.'); +INSERT INTO t1(docid,words) VALUES(1031041,'Thus have I been twenty years in thy house; I served thee fourteen years for thy two daughters, and six years for thy cattle: and thou hast changed my wages ten times.'); +INSERT INTO t1(docid,words) VALUES(1031042,'Except the God of my father, the God of Abraham, and the fear of Isaac, had been with me, surely thou hadst sent me away now empty. God hath seen mine affliction and the labour of my hands, and rebuked thee yesternight.'); +INSERT INTO t1(docid,words) VALUES(1031043,'And Laban answered and said unto Jacob, These daughters are my daughters, and these children are my children, and these cattle are my cattle, and all that thou seest is mine: and what can I do this day unto these my daughters, or unto their children which they have born?'); +INSERT INTO t1(docid,words) VALUES(1031044,'Now therefore come thou, let us make a covenant, I and thou; and let it be for a witness between me and thee.'); +INSERT INTO t1(docid,words) VALUES(1031045,'And Jacob took a stone, and set it up for a pillar.'); +INSERT INTO t1(docid,words) VALUES(1031046,'And Jacob said unto his brethren, Gather stones; and they took stones, and made an heap: and they did eat there upon the heap.'); +INSERT INTO t1(docid,words) VALUES(1031047,'And Laban called it Jegarsahadutha: but Jacob called it Galeed.'); +INSERT INTO t1(docid,words) VALUES(1031048,'And Laban said, This heap is a witness between me and thee this day. Therefore was the name of it called Galeed;'); +INSERT INTO t1(docid,words) VALUES(1031049,'And Mizpah; for he said, The LORD watch between me and thee, when we are absent one from another.'); +INSERT INTO t1(docid,words) VALUES(1031050,'If thou shalt afflict my daughters, or if thou shalt take other wives beside my daughters, no man is with us; see, God is witness betwixt me and thee.'); +INSERT INTO t1(docid,words) VALUES(1031051,'And Laban said to Jacob, Behold this heap, and behold this pillar, which I have cast betwixt me and thee:'); +INSERT INTO t1(docid,words) VALUES(1031052,'This heap be witness, and this pillar be witness, that I will not pass over this heap to thee, and that thou shalt not pass over this heap and this pillar unto me, for harm.'); +INSERT INTO t1(docid,words) VALUES(1031053,'The God of Abraham, and the God of Nahor, the God of their father, judge betwixt us. And Jacob sware by the fear of his father Isaac.'); +INSERT INTO t1(docid,words) VALUES(1031054,'Then Jacob offered sacrifice upon the mount, and called his brethren to eat bread: and they did eat bread, and tarried all night in the mount.'); +INSERT INTO t1(docid,words) VALUES(1031055,'And early in the morning Laban rose up, and kissed his sons and his daughters, and blessed them: and Laban departed, and returned unto his place.'); +INSERT INTO t1(docid,words) VALUES(1032001,'And Jacob went on his way, and the angels of God met him.'); +INSERT INTO t1(docid,words) VALUES(1032002,'And when Jacob saw them, he said, This is God''s host: and he called the name of that place Mahanaim.'); +INSERT INTO t1(docid,words) VALUES(1032003,'And Jacob sent messengers before him to Esau his brother unto the land of Seir, the country of Edom.'); +INSERT INTO t1(docid,words) VALUES(1032004,'And he commanded them, saying, Thus shall ye speak unto my lord Esau; Thy servant Jacob saith thus, I have sojourned with Laban, and stayed there until now:'); +INSERT INTO t1(docid,words) VALUES(1032005,'And I have oxen, and asses, flocks, and menservants, and womenservants: and I have sent to tell my lord, that I may find grace in thy sight.'); +INSERT INTO t1(docid,words) VALUES(1032006,'And the messengers returned to Jacob, saying, We came to thy brother Esau, and also he cometh to meet thee, and four hundred men with him.'); +INSERT INTO t1(docid,words) VALUES(1032007,'Then Jacob was greatly afraid and distressed: and he divided the people that was with him, and the flocks, and herds, and the camels, into two bands;'); +INSERT INTO t1(docid,words) VALUES(1032008,'And said, If Esau come to the one company, and smite it, then the other company which is left shall escape.'); +INSERT INTO t1(docid,words) VALUES(1032009,'And Jacob said, O God of my father Abraham, and God of my father Isaac, the LORD which saidst unto me, Return unto thy country, and to thy kindred, and I will deal well with thee:'); +INSERT INTO t1(docid,words) VALUES(1032010,'I am not worthy of the least of all the mercies, and of all the truth, which thou hast shewed unto thy servant; for with my staff I passed over this Jordan; and now I am become two bands.'); +INSERT INTO t1(docid,words) VALUES(1032011,'Deliver me, I pray thee, from the hand of my brother, from the hand of Esau: for I fear him, lest he will come and smite me, and the mother with the children.'); +INSERT INTO t1(docid,words) VALUES(1032012,'And thou saidst, I will surely do thee good, and make thy seed as the sand of the sea, which cannot be numbered for multitude.'); +INSERT INTO t1(docid,words) VALUES(1032013,'And he lodged there that same night; and took of that which came to his hand a present for Esau his brother;'); +INSERT INTO t1(docid,words) VALUES(1032014,'Two hundred she goats, and twenty he goats, two hundred ewes, and twenty rams,'); +INSERT INTO t1(docid,words) VALUES(1032015,'Thirty milch camels with their colts, forty kine, and ten bulls, twenty she asses, and ten foals.'); +INSERT INTO t1(docid,words) VALUES(1032016,'And he delivered them into the hand of his servants, every drove by themselves; and said unto his servants, Pass over before me, and put a space betwixt drove and drove.'); +INSERT INTO t1(docid,words) VALUES(1032017,'And he commanded the foremost, saying, When Esau my brother meeteth thee, and asketh thee, saying, Whose art thou? and whither goest thou? and whose are these before thee?'); +INSERT INTO t1(docid,words) VALUES(1032018,'Then thou shalt say, They be thy servant Jacob''s; it is a present sent unto my lord Esau: and, behold, also he is behind us.'); +INSERT INTO t1(docid,words) VALUES(1032019,'And so commanded he the second, and the third, and all that followed the droves, saying, On this manner shall ye speak unto Esau, when ye find him.'); +INSERT INTO t1(docid,words) VALUES(1032020,'And say ye moreover, Behold, thy servant Jacob is behind us. For he said, I will appease him with the present that goeth before me, and afterward I will see his face; peradventure he will accept of me.'); +INSERT INTO t1(docid,words) VALUES(1032021,'So went the present over before him: and himself lodged that night in the company.'); +INSERT INTO t1(docid,words) VALUES(1032022,'And he rose up that night, and took his two wives, and his two womenservants, and his eleven sons, and passed over the ford Jabbok.'); +INSERT INTO t1(docid,words) VALUES(1032023,'And he took them, and sent them over the brook, and sent over that he had.'); +INSERT INTO t1(docid,words) VALUES(1032024,'And Jacob was left alone; and there wrestled a man with him until the breaking of the day.'); +INSERT INTO t1(docid,words) VALUES(1032025,'And when he saw that he prevailed not against him, he touched the hollow of his thigh; and the hollow of Jacob''s thigh was out of joint, as he wrestled with him.'); +INSERT INTO t1(docid,words) VALUES(1032026,'And he said, Let me go, for the day breaketh. And he said, I will not let thee go, except thou bless me.'); +INSERT INTO t1(docid,words) VALUES(1032027,'And he said unto him, What is thy name? And he said, Jacob.'); +INSERT INTO t1(docid,words) VALUES(1032028,'And he said, Thy name shall be called no more Jacob, but Israel: for as a prince hast thou power with God and with men, and hast prevailed.'); +INSERT INTO t1(docid,words) VALUES(1032029,'And Jacob asked him, and said, Tell me, I pray thee, thy name. And he said, Wherefore is it that thou dost ask after my name? And he blessed him there.'); +INSERT INTO t1(docid,words) VALUES(1032030,'And Jacob called the name of the place Peniel: for I have seen God face to face, and my life is preserved.'); +INSERT INTO t1(docid,words) VALUES(1032031,'And as he passed over Penuel the sun rose upon him, and he halted upon his thigh.'); +INSERT INTO t1(docid,words) VALUES(1032032,'Therefore the children of Israel eat not of the sinew which shrank, which is upon the hollow of the thigh, unto this day: because he touched the hollow of Jacob''s thigh in the sinew that shrank.'); +INSERT INTO t1(docid,words) VALUES(1033001,'And Jacob lifted up his eyes, and looked, and, behold, Esau came, and with him four hundred men. And he divided the children unto Leah, and unto Rachel, and unto the two handmaids.'); +INSERT INTO t1(docid,words) VALUES(1033002,'And he put the handmaids and their children foremost, and Leah and her children after, and Rachel and Joseph hindermost.'); +INSERT INTO t1(docid,words) VALUES(1033003,'And he passed over before them, and bowed himself to the ground seven times, until he came near to his brother.'); +INSERT INTO t1(docid,words) VALUES(1033004,'And Esau ran to meet him, and embraced him, and fell on his neck, and kissed him: and they wept.'); +INSERT INTO t1(docid,words) VALUES(1033005,'And he lifted up his eyes, and saw the women and the children; and said, Who are those with thee? And he said, The children which God hath graciously given thy servant.'); +INSERT INTO t1(docid,words) VALUES(1033006,'Then the handmaidens came near, they and their children, and they bowed themselves.'); +INSERT INTO t1(docid,words) VALUES(1033007,'And Leah also with her children came near, and bowed themselves: and after came Joseph near and Rachel, and they bowed themselves.'); +INSERT INTO t1(docid,words) VALUES(1033008,'And he said, What meanest thou by all this drove which I met? And he said, These are to find grace in the sight of my lord.'); +INSERT INTO t1(docid,words) VALUES(1033009,'And Esau said, I have enough, my brother; keep that thou hast unto thyself.'); +INSERT INTO t1(docid,words) VALUES(1033010,'And Jacob said, Nay, I pray thee, if now I have found grace in thy sight, then receive my present at my hand: for therefore I have seen thy face, as though I had seen the face of God, and thou wast pleased with me.'); +INSERT INTO t1(docid,words) VALUES(1033011,'Take, I pray thee, my blessing that is brought to thee; because God hath dealt graciously with me, and because I have enough. And he urged him, and he took it.'); +INSERT INTO t1(docid,words) VALUES(1033012,'And he said, Let us take our journey, and let us go, and I will go before thee.'); +INSERT INTO t1(docid,words) VALUES(1033013,'And he said unto him, My lord knoweth that the children are tender, and the flocks and herds with young are with me: and if men should overdrive them one day, all the flock will die.'); +INSERT INTO t1(docid,words) VALUES(1033014,'Let my lord, I pray thee, pass over before his servant: and I will lead on softly, according as the cattle that goeth before me and the children be able to endure, until I come unto my lord unto Seir.'); +INSERT INTO t1(docid,words) VALUES(1033015,'And Esau said, Let me now leave with thee some of the folk that are with me. And he said, What needeth it? let me find grace in the sight of my lord.'); +INSERT INTO t1(docid,words) VALUES(1033016,'So Esau returned that day on his way unto Seir.'); +INSERT INTO t1(docid,words) VALUES(1033017,'And Jacob journeyed to Succoth, and built him an house, and made booths for his cattle: therefore the name of the place is called Succoth.'); +INSERT INTO t1(docid,words) VALUES(1033018,'And Jacob came to Shalem, a city of Shechem, which is in the land of Canaan, when he came from Padanaram; and pitched his tent before the city.'); +INSERT INTO t1(docid,words) VALUES(1033019,'And he bought a parcel of a field, where he had spread his tent, at the hand of the children of Hamor, Shechem''s father, for an hundred pieces of money.'); +INSERT INTO t1(docid,words) VALUES(1033020,'And he erected there an altar, and called it EleloheIsrael.'); +INSERT INTO t1(docid,words) VALUES(1034001,'And Dinah the daughter of Leah, which she bare unto Jacob, went out to see the daughters of the land.'); +INSERT INTO t1(docid,words) VALUES(1034002,'And when Shechem the son of Hamor the Hivite, prince of the country, saw her, he took her, and lay with her, and defiled her.'); +INSERT INTO t1(docid,words) VALUES(1034003,'And his soul clave unto Dinah the daughter of Jacob, and he loved the damsel, and spake kindly unto the damsel.'); +INSERT INTO t1(docid,words) VALUES(1034004,'And Shechem spake unto his father Hamor, saying, Get me this damsel to wife.'); +INSERT INTO t1(docid,words) VALUES(1034005,'And Jacob heard that he had defiled Dinah his daughter: now his sons were with his cattle in the field: and Jacob held his peace until they were come.'); +INSERT INTO t1(docid,words) VALUES(1034006,'And Hamor the father of Shechem went out unto Jacob to commune with him.'); +INSERT INTO t1(docid,words) VALUES(1034007,'And the sons of Jacob came out of the field when they heard it: and the men were grieved, and they were very wroth, because he had wrought folly in Israel in lying with Jacob''s daughter: which thing ought not to be done.'); +INSERT INTO t1(docid,words) VALUES(1034008,'And Hamor communed with them, saying, The soul of my son Shechem longeth for your daughter: I pray you give her him to wife.'); +INSERT INTO t1(docid,words) VALUES(1034009,'And make ye marriages with us, and give your daughters unto us, and take our daughters unto you.'); +INSERT INTO t1(docid,words) VALUES(1034010,'And ye shall dwell with us: and the land shall be before you; dwell and trade ye therein, and get you possessions therein.'); +INSERT INTO t1(docid,words) VALUES(1034011,'And Shechem said unto her father and unto her brethren, Let me find grace in your eyes, and what ye shall say unto me I will give.'); +INSERT INTO t1(docid,words) VALUES(1034012,'Ask me never so much dowry and gift, and I will give according as ye shall say unto me: but give me the damsel to wife.'); +INSERT INTO t1(docid,words) VALUES(1034013,'And the sons of Jacob answered Shechem and Hamor his father deceitfully, and said, because he had defiled Dinah their sister:'); +INSERT INTO t1(docid,words) VALUES(1034014,'And they said unto them, We cannot do this thing, to give our sister to one that is uncircumcised; for that were a reproach unto us:'); +INSERT INTO t1(docid,words) VALUES(1034015,'But in this will we consent unto you: If ye will be as we be, that every male of you be circumcised;'); +INSERT INTO t1(docid,words) VALUES(1034016,'Then will we give our daughters unto you, and we will take your daughters to us, and we will dwell with you, and we will become one people.'); +INSERT INTO t1(docid,words) VALUES(1034017,'But if ye will not hearken unto us, to be circumcised; then will we take our daughter, and we will be gone.'); +INSERT INTO t1(docid,words) VALUES(1034018,'And their words pleased Hamor, and Shechem Hamor''s son.'); +INSERT INTO t1(docid,words) VALUES(1034019,'And the young man deferred not to do the thing, because he had delight in Jacob''s daughter: and he was more honourable than all the house of his father.'); +INSERT INTO t1(docid,words) VALUES(1034020,'And Hamor and Shechem his son came unto the gate of their city, and communed with the men of their city, saying,'); +INSERT INTO t1(docid,words) VALUES(1034021,'These men are peaceable with us; therefore let them dwell in the land, and trade therein; for the land, behold, it is large enough for them; let us take their daughters to us for wives, and let us give them our daughters.'); +INSERT INTO t1(docid,words) VALUES(1034022,'Only herein will the men consent unto us for to dwell with us, to be one people, if every male among us be circumcised, as they are circumcised.'); +INSERT INTO t1(docid,words) VALUES(1034023,'Shall not their cattle and their substance and every beast of their''s be our''s? only let us consent unto them, and they will dwell with us.'); +INSERT INTO t1(docid,words) VALUES(1034024,'And unto Hamor and unto Shechem his son hearkened all that went out of the gate of his city; and every male was circumcised, all that went out of the gate of his city.'); +INSERT INTO t1(docid,words) VALUES(1034025,'And it came to pass on the third day, when they were sore, that two of the sons of Jacob, Simeon and Levi, Dinah''s brethren, took each man his sword, and came upon the city boldly, and slew all the males.'); +INSERT INTO t1(docid,words) VALUES(1034026,'And they slew Hamor and Shechem his son with the edge of the sword, and took Dinah out of Shechem''s house, and went out.'); +INSERT INTO t1(docid,words) VALUES(1034027,'The sons of Jacob came upon the slain, and spoiled the city, because they had defiled their sister.'); +INSERT INTO t1(docid,words) VALUES(1034028,'They took their sheep, and their oxen, and their asses, and that which was in the city, and that which was in the field,'); +INSERT INTO t1(docid,words) VALUES(1034029,'And all their wealth, and all their little ones, and their wives took they captive, and spoiled even all that was in the house.'); +INSERT INTO t1(docid,words) VALUES(1034030,'And Jacob said to Simeon and Levi, Ye have troubled me to make me to stink among the inhabitants of the land, among the Canaanites and the Perizzites: and I being few in number, they shall gather themselves together against me, and slay me; and I shall be destroyed, I and my house.'); +INSERT INTO t1(docid,words) VALUES(1034031,'And they said, Should he deal with our sister as with an harlot?'); +INSERT INTO t1(docid,words) VALUES(1035001,'And God said unto Jacob, Arise, go up to Bethel, and dwell there: and make there an altar unto God, that appeared unto thee when thou fleddest from the face of Esau thy brother.'); +INSERT INTO t1(docid,words) VALUES(1035002,'Then Jacob said unto his household, and to all that were with him, Put away the strange gods that are among you, and be clean, and change your garments:'); +INSERT INTO t1(docid,words) VALUES(1035003,'And let us arise, and go up to Bethel; and I will make there an altar unto God, who answered me in the day of my distress, and was with me in the way which I went.'); +INSERT INTO t1(docid,words) VALUES(1035004,'And they gave unto Jacob all the strange gods which were in their hand, and all their earrings which were in their ears; and Jacob hid them under the oak which was by Shechem.'); +INSERT INTO t1(docid,words) VALUES(1035005,'And they journeyed: and the terror of God was upon the cities that were round about them, and they did not pursue after the sons of Jacob.'); +INSERT INTO t1(docid,words) VALUES(1035006,'So Jacob came to Luz, which is in the land of Canaan, that is, Bethel, he and all the people that were with him.'); +INSERT INTO t1(docid,words) VALUES(1035007,'And he built there an altar, and called the place Elbethel: because there God appeared unto him, when he fled from the face of his brother.'); +INSERT INTO t1(docid,words) VALUES(1035008,'But Deborah Rebekah''s nurse died, and she was buried beneath Bethel under an oak: and the name of it was called Allonbachuth.'); +INSERT INTO t1(docid,words) VALUES(1035009,'And God appeared unto Jacob again, when he came out of Padanaram, and blessed him.'); +INSERT INTO t1(docid,words) VALUES(1035010,'And God said unto him, Thy name is Jacob: thy name shall not be called any more Jacob, but Israel shall be thy name: and he called his name Israel.'); +INSERT INTO t1(docid,words) VALUES(1035011,'And God said unto him, I am God Almighty: be fruitful and multiply; a nation and a company of nations shall be of thee, and kings shall come out of thy loins;'); +INSERT INTO t1(docid,words) VALUES(1035012,'And the land which I gave Abraham and Isaac, to thee I will give it, and to thy seed after thee will I give the land.'); +INSERT INTO t1(docid,words) VALUES(1035013,'And God went up from him in the place where he talked with him.'); +INSERT INTO t1(docid,words) VALUES(1035014,'And Jacob set up a pillar in the place where he talked with him, even a pillar of stone: and he poured a drink offering thereon, and he poured oil thereon.'); +INSERT INTO t1(docid,words) VALUES(1035015,'And Jacob called the name of the place where God spake with him, Bethel.'); +INSERT INTO t1(docid,words) VALUES(1035016,'And they journeyed from Bethel; and there was but a little way to come to Ephrath: and Rachel travailed, and she had hard labour.'); +INSERT INTO t1(docid,words) VALUES(1035017,'And it came to pass, when she was in hard labour, that the midwife said unto her, Fear not; thou shalt have this son also.'); +INSERT INTO t1(docid,words) VALUES(1035018,'And it came to pass, as her soul was in departing, (for she died) that she called his name Benoni: but his father called him Benjamin.'); +INSERT INTO t1(docid,words) VALUES(1035019,'And Rachel died, and was buried in the way to Ephrath, which is Bethlehem.'); +INSERT INTO t1(docid,words) VALUES(1035020,'And Jacob set a pillar upon her grave: that is the pillar of Rachel''s grave unto this day.'); +INSERT INTO t1(docid,words) VALUES(1035021,'And Israel journeyed, and spread his tent beyond the tower of Edar.'); +INSERT INTO t1(docid,words) VALUES(1035022,'And it came to pass, when Israel dwelt in that land, that Reuben went and lay with Bilhah his father''s concubine: and Israel heard it. Now the sons of Jacob were twelve:'); +INSERT INTO t1(docid,words) VALUES(1035023,'The sons of Leah; Reuben, Jacob''s firstborn, and Simeon, and Levi, and Judah, and Issachar, and Zebulun:'); +INSERT INTO t1(docid,words) VALUES(1035024,'The sons of Rachel; Joseph, and Benjamin:'); +INSERT INTO t1(docid,words) VALUES(1035025,'And the sons of Bilhah, Rachel''s handmaid; Dan, and Naphtali:'); +INSERT INTO t1(docid,words) VALUES(1035026,'And the sons of Zilpah, Leah''s handmaid: Gad, and Asher: these are the sons of Jacob, which were born to him in Padanaram.'); +INSERT INTO t1(docid,words) VALUES(1035027,'And Jacob came unto Isaac his father unto Mamre, unto the city of Arbah, which is Hebron, where Abraham and Isaac sojourned.'); +INSERT INTO t1(docid,words) VALUES(1035028,'And the days of Isaac were an hundred and fourscore years.'); +INSERT INTO t1(docid,words) VALUES(1035029,'And Isaac gave up the ghost, and died, and was gathered unto his people, being old and full of days: and his sons Esau and Jacob buried him.'); +INSERT INTO t1(docid,words) VALUES(1036001,'Now these are the generations of Esau, who is Edom.'); +INSERT INTO t1(docid,words) VALUES(1036002,'Esau took his wives of the daughters of Canaan; Adah the daughter of Elon the Hittite, and Aholibamah the daughter of Anah the daughter of Zibeon the Hivite;'); +INSERT INTO t1(docid,words) VALUES(1036003,'And Bashemath Ishmael''s daughter, sister of Nebajoth.'); +INSERT INTO t1(docid,words) VALUES(1036004,'And Adah bare to Esau Eliphaz; and Bashemath bare Reuel;'); +INSERT INTO t1(docid,words) VALUES(1036005,'And Aholibamah bare Jeush, and Jaalam, and Korah: these are the sons of Esau, which were born unto him in the land of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1036006,'And Esau took his wives, and his sons, and his daughters, and all the persons of his house, and his cattle, and all his beasts, and all his substance, which he had got in the land of Canaan; and went into the country from the face of his brother Jacob.'); +INSERT INTO t1(docid,words) VALUES(1036007,'For their riches were more than that they might dwell together; and the land wherein they were strangers could not bear them because of their cattle.'); +INSERT INTO t1(docid,words) VALUES(1036008,'Thus dwelt Esau in mount Seir: Esau is Edom.'); +INSERT INTO t1(docid,words) VALUES(1036009,'And these are the generations of Esau the father of the Edomites in mount Seir:'); +INSERT INTO t1(docid,words) VALUES(1036010,'These are the names of Esau''s sons; Eliphaz the son of Adah the wife of Esau, Reuel the son of Bashemath the wife of Esau.'); +INSERT INTO t1(docid,words) VALUES(1036011,'And the sons of Eliphaz were Teman, Omar, Zepho, and Gatam, and Kenaz.'); +INSERT INTO t1(docid,words) VALUES(1036012,'And Timna was concubine to Eliphaz Esau''s son; and she bare to Eliphaz Amalek: these were the sons of Adah Esau''s wife.'); +INSERT INTO t1(docid,words) VALUES(1036013,'And these are the sons of Reuel; Nahath, and Zerah, Shammah, and Mizzah: these were the sons of Bashemath Esau''s wife.'); +INSERT INTO t1(docid,words) VALUES(1036014,'And these were the sons of Aholibamah, the daughter of Anah the daughter of Zibeon, Esau''s wife: and she bare to Esau Jeush, and Jaalam, and Korah.'); +INSERT INTO t1(docid,words) VALUES(1036015,'These were dukes of the sons of Esau: the sons of Eliphaz the firstborn son of Esau; duke Teman, duke Omar, duke Zepho, duke Kenaz,'); +INSERT INTO t1(docid,words) VALUES(1036016,'Duke Korah, duke Gatam, and duke Amalek: these are the dukes that came of Eliphaz in the land of Edom; these were the sons of Adah.'); +INSERT INTO t1(docid,words) VALUES(1036017,'And these are the sons of Reuel Esau''s son; duke Nahath, duke Zerah, duke Shammah, duke Mizzah: these are the dukes that came of Reuel in the land of Edom; these are the sons of Bashemath Esau''s wife.'); +INSERT INTO t1(docid,words) VALUES(1036018,'And these are the sons of Aholibamah Esau''s wife; duke Jeush, duke Jaalam, duke Korah: these were the dukes that came of Aholibamah the daughter of Anah, Esau''s wife.'); +INSERT INTO t1(docid,words) VALUES(1036019,'These are the sons of Esau, who is Edom, and these are their dukes.'); +INSERT INTO t1(docid,words) VALUES(1036020,'These are the sons of Seir the Horite, who inhabited the land; Lotan, and Shobal, and Zibeon, and Anah,'); +INSERT INTO t1(docid,words) VALUES(1036021,'And Dishon, and Ezer, and Dishan: these are the dukes of the Horites, the children of Seir in the land of Edom.'); +INSERT INTO t1(docid,words) VALUES(1036022,'And the children of Lotan were Hori and Hemam; and Lotan''s sister was Timna.'); +INSERT INTO t1(docid,words) VALUES(1036023,'And the children of Shobal were these; Alvan, and Manahath, and Ebal, Shepho, and Onam.'); +INSERT INTO t1(docid,words) VALUES(1036024,'And these are the children of Zibeon; both Ajah, and Anah: this was that Anah that found the mules in the wilderness, as he fed the asses of Zibeon his father.'); +INSERT INTO t1(docid,words) VALUES(1036025,'And the children of Anah were these; Dishon, and Aholibamah the daughter of Anah.'); +INSERT INTO t1(docid,words) VALUES(1036026,'And these are the children of Dishon; Hemdan, and Eshban, and Ithran, and Cheran.'); +INSERT INTO t1(docid,words) VALUES(1036027,'The children of Ezer are these; Bilhan, and Zaavan, and Akan.'); +INSERT INTO t1(docid,words) VALUES(1036028,'The children of Dishan are these; Uz, and Aran.'); +INSERT INTO t1(docid,words) VALUES(1036029,'These are the dukes that came of the Horites; duke Lotan, duke Shobal, duke Zibeon, duke Anah,'); +INSERT INTO t1(docid,words) VALUES(1036030,'Duke Dishon, duke Ezer, duke Dishan: these are the dukes that came of Hori, among their dukes in the land of Seir.'); +INSERT INTO t1(docid,words) VALUES(1036031,'And these are the kings that reigned in the land of Edom, before there reigned any king over the children of Israel.'); +INSERT INTO t1(docid,words) VALUES(1036032,'And Bela the son of Beor reigned in Edom: and the name of his city was Dinhabah.'); +INSERT INTO t1(docid,words) VALUES(1036033,'And Bela died, and Jobab the son of Zerah of Bozrah reigned in his stead.'); +INSERT INTO t1(docid,words) VALUES(1036034,'And Jobab died, and Husham of the land of Temani reigned in his stead.'); +INSERT INTO t1(docid,words) VALUES(1036035,'And Husham died, and Hadad the son of Bedad, who smote Midian in the field of Moab, reigned in his stead: and the name of his city was Avith.'); +INSERT INTO t1(docid,words) VALUES(1036036,'And Hadad died, and Samlah of Masrekah reigned in his stead.'); +INSERT INTO t1(docid,words) VALUES(1036037,'And Samlah died, and Saul of Rehoboth by the river reigned in his stead.'); +INSERT INTO t1(docid,words) VALUES(1036038,'And Saul died, and Baalhanan the son of Achbor reigned in his stead.'); +INSERT INTO t1(docid,words) VALUES(1036039,'And Baalhanan the son of Achbor died, and Hadar reigned in his stead: and the name of his city was Pau; and his wife''s name was Mehetabel, the daughter of Matred, the daughter of Mezahab.'); +INSERT INTO t1(docid,words) VALUES(1036040,'And these are the names of the dukes that came of Esau, according to their families, after their places, by their names; duke Timnah, duke Alvah, duke Jetheth,'); +INSERT INTO t1(docid,words) VALUES(1036041,'Duke Aholibamah, duke Elah, duke Pinon,'); +INSERT INTO t1(docid,words) VALUES(1036042,'Duke Kenaz, duke Teman, duke Mibzar,'); +INSERT INTO t1(docid,words) VALUES(1036043,'Duke Magdiel, duke Iram: these be the dukes of Edom, according to their habitations in the land of their possession: he is Esau the father of the Edomites.'); +INSERT INTO t1(docid,words) VALUES(1037001,'And Jacob dwelt in the land wherein his father was a stranger, in the land of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1037002,'These are the generations of Jacob. Joseph, being seventeen years old, was feeding the flock with his brethren; and the lad was with the sons of Bilhah, and with the sons of Zilpah, his father''s wives: and Joseph brought unto his father their evil report.'); +INSERT INTO t1(docid,words) VALUES(1037003,'Now Israel loved Joseph more than all his children, because he was the son of his old age: and he made him a coat of many colours.'); +INSERT INTO t1(docid,words) VALUES(1037004,'And when his brethren saw that their father loved him more than all his brethren, they hated him, and could not speak peaceably unto him.'); +INSERT INTO t1(docid,words) VALUES(1037005,'And Joseph dreamed a dream, and he told it his brethren: and they hated him yet the more.'); +INSERT INTO t1(docid,words) VALUES(1037006,'And he said unto them, Hear, I pray you, this dream which I have dreamed:'); +INSERT INTO t1(docid,words) VALUES(1037007,'For, behold, we were binding sheaves in the field, and, lo, my sheaf arose, and also stood upright; and, behold, your sheaves stood round about, and made obeisance to my sheaf.'); +INSERT INTO t1(docid,words) VALUES(1037008,'And his brethren said to him, Shalt thou indeed reign over us? or shalt thou indeed have dominion over us? And they hated him yet the more for his dreams, and for his words.'); +INSERT INTO t1(docid,words) VALUES(1037009,'And he dreamed yet another dream, and told it his brethren, and said, Behold, I have dreamed a dream more; and, behold, the sun and the moon and the eleven stars made obeisance to me.'); +INSERT INTO t1(docid,words) VALUES(1037010,'And he told it to his father, and to his brethren: and his father rebuked him, and said unto him, What is this dream that thou hast dreamed? Shall I and thy mother and thy brethren indeed come to bow down ourselves to thee to the earth?'); +INSERT INTO t1(docid,words) VALUES(1037011,'And his brethren envied him; but his father observed the saying.'); +INSERT INTO t1(docid,words) VALUES(1037012,'And his brethren went to feed their father''s flock in Shechem.'); +INSERT INTO t1(docid,words) VALUES(1037013,'And Israel said unto Joseph, Do not thy brethren feed the flock in Shechem? come, and I will send thee unto them. And he said to him, Here am I.'); +INSERT INTO t1(docid,words) VALUES(1037014,'And he said to him, Go, I pray thee, see whether it be well with thy brethren, and well with the flocks; and bring me word again. So he sent him out of the vale of Hebron, and he came to Shechem.'); +INSERT INTO t1(docid,words) VALUES(1037015,'And a certain man found him, and, behold, he was wandering in the field: and the man asked him, saying, What seekest thou?'); +INSERT INTO t1(docid,words) VALUES(1037016,'And he said, I seek my brethren: tell me, I pray thee, where they feed their flocks.'); +INSERT INTO t1(docid,words) VALUES(1037017,'And the man said, They are departed hence; for I heard them say, Let us go to Dothan. And Joseph went after his brethren, and found them in Dothan.'); +INSERT INTO t1(docid,words) VALUES(1037018,'And when they saw him afar off, even before he came near unto them, they conspired against him to slay him.'); +INSERT INTO t1(docid,words) VALUES(1037019,'And they said one to another, Behold, this dreamer cometh.'); +INSERT INTO t1(docid,words) VALUES(1037020,'Come now therefore, and let us slay him, and cast him into some pit, and we will say, Some evil beast hath devoured him: and we shall see what will become of his dreams.'); +INSERT INTO t1(docid,words) VALUES(1037021,'And Reuben heard it, and he delivered him out of their hands; and said, Let us not kill him.'); +INSERT INTO t1(docid,words) VALUES(1037022,'And Reuben said unto them, Shed no blood, but cast him into this pit that is in the wilderness, and lay no hand upon him; that he might rid him out of their hands, to deliver him to his father again.'); +INSERT INTO t1(docid,words) VALUES(1037023,'And it came to pass, when Joseph was come unto his brethren, that they stript Joseph out of his coat, his coat of many colours that was on him;'); +INSERT INTO t1(docid,words) VALUES(1037024,'And they took him, and cast him into a pit: and the pit was empty, there was no water in it.'); +INSERT INTO t1(docid,words) VALUES(1037025,'And they sat down to eat bread: and they lifted up their eyes and looked, and, behold, a company of Ishmeelites came from Gilead with their camels bearing spicery and balm and myrrh, going to carry it down to Egypt.'); +INSERT INTO t1(docid,words) VALUES(1037026,'And Judah said unto his brethren, What profit is it if we slay our brother, and conceal his blood?'); +INSERT INTO t1(docid,words) VALUES(1037027,'Come, and let us sell him to the Ishmeelites, and let not our hand be upon him; for he is our brother and our flesh. And his brethren were content.'); +INSERT INTO t1(docid,words) VALUES(1037028,'Then there passed by Midianites merchantmen; and they drew and lifted up Joseph out of the pit, and sold Joseph to the Ishmeelites for twenty pieces of silver: and they brought Joseph into Egypt.'); +INSERT INTO t1(docid,words) VALUES(1037029,'And Reuben returned unto the pit; and, behold, Joseph was not in the pit; and he rent his clothes.'); +INSERT INTO t1(docid,words) VALUES(1037030,'And he returned unto his brethren, and said, The child is not; and I, whither shall I go?'); +INSERT INTO t1(docid,words) VALUES(1037031,'And they took Joseph''s coat, and killed a kid of the goats, and dipped the coat in the blood;'); +INSERT INTO t1(docid,words) VALUES(1037032,'And they sent the coat of many colours, and they brought it to their father; and said, This have we found: know now whether it be thy son''s coat or no.'); +INSERT INTO t1(docid,words) VALUES(1037033,'And he knew it, and said, It is my son''s coat; an evil beast hath devoured him; Joseph is without doubt rent in pieces.'); +INSERT INTO t1(docid,words) VALUES(1037034,'And Jacob rent his clothes, and put sackcloth upon his loins, and mourned for his son many days.'); +INSERT INTO t1(docid,words) VALUES(1037035,'And all his sons and all his daughters rose up to comfort him; but he refused to be comforted; and he said, For I will go down into the grave unto my son mourning. Thus his father wept for him.'); +INSERT INTO t1(docid,words) VALUES(1037036,'And the Midianites sold him into Egypt unto Potiphar, an officer of Pharaoh''s, and captain of the guard.'); +INSERT INTO t1(docid,words) VALUES(1038001,'And it came to pass at that time, that Judah went down from his brethren, and turned in to a certain Adullamite, whose name was Hirah.'); +INSERT INTO t1(docid,words) VALUES(1038002,'And Judah saw there a daughter of a certain Canaanite, whose name was Shuah; and he took her, and went in unto her.'); +INSERT INTO t1(docid,words) VALUES(1038003,'And she conceived, and bare a son; and he called his name Er.'); +INSERT INTO t1(docid,words) VALUES(1038004,'And she conceived again, and bare a son; and she called his name Onan.'); +INSERT INTO t1(docid,words) VALUES(1038005,'And she yet again conceived, and bare a son; and called his name Shelah: and he was at Chezib, when she bare him.'); +INSERT INTO t1(docid,words) VALUES(1038006,'And Judah took a wife for Er his firstborn, whose name was Tamar.'); +INSERT INTO t1(docid,words) VALUES(1038007,'And Er, Judah''s firstborn, was wicked in the sight of the LORD; and the LORD slew him.'); +INSERT INTO t1(docid,words) VALUES(1038008,'And Judah said unto Onan, Go in unto thy brother''s wife, and marry her, and raise up seed to thy brother.'); +INSERT INTO t1(docid,words) VALUES(1038009,'And Onan knew that the seed should not be his; and it came to pass, when he went in unto his brother''s wife, that he spilled it on the ground, lest that he should give seed to his brother.'); +INSERT INTO t1(docid,words) VALUES(1038010,'And the thing which he did displeased the LORD: wherefore he slew him also.'); +INSERT INTO t1(docid,words) VALUES(1038011,'Then said Judah to Tamar his daughter in law, Remain a widow at thy father''s house, till Shelah my son be grown: for he said, Lest peradventure he die also, as his brethren did. And Tamar went and dwelt in her father''s house.'); +INSERT INTO t1(docid,words) VALUES(1038012,'And in process of time the daughter of Shuah Judah''s wife died; and Judah was comforted, and went up unto his sheepshearers to Timnath, he and his friend Hirah the Adullamite.'); +INSERT INTO t1(docid,words) VALUES(1038013,'And it was told Tamar, saying, Behold thy father in law goeth up to Timnath to shear his sheep.'); +INSERT INTO t1(docid,words) VALUES(1038014,'And she put her widow''s garments off from her, and covered her with a vail, and wrapped herself, and sat in an open place, which is by the way to Timnath; for she saw that Shelah was grown, and she was not given unto him to wife.'); +INSERT INTO t1(docid,words) VALUES(1038015,'When Judah saw her, he thought her to be an harlot; because she had covered her face.'); +INSERT INTO t1(docid,words) VALUES(1038016,'And he turned unto her by the way, and said, Go to, I pray thee, let me come in unto thee; (for he knew not that she was his daughter in law.) And she said, What wilt thou give me, that thou mayest come in unto me?'); +INSERT INTO t1(docid,words) VALUES(1038017,'And he said, I will send thee a kid from the flock. And she said, Wilt thou give me a pledge, till thou send it?'); +INSERT INTO t1(docid,words) VALUES(1038018,'And he said, What pledge shall I give thee? And she said, Thy signet, and thy bracelets, and thy staff that is in thine hand. And he gave it her, and came in unto her, and she conceived by him.'); +INSERT INTO t1(docid,words) VALUES(1038019,'And she arose, and went away, and laid by her vail from her, and put on the garments of her widowhood.'); +INSERT INTO t1(docid,words) VALUES(1038020,'And Judah sent the kid by the hand of his friend the Adullamite, to receive his pledge from the woman''s hand: but he found her not.'); +INSERT INTO t1(docid,words) VALUES(1038021,'Then he asked the men of that place, saying, Where is the harlot, that was openly by the way side? And they said, There was no harlot in this place.'); +INSERT INTO t1(docid,words) VALUES(1038022,'And he returned to Judah, and said, I cannot find her; and also the men of the place said, that there was no harlot in this place.'); +INSERT INTO t1(docid,words) VALUES(1038023,'And Judah said, Let her take it to her, lest we be shamed: behold, I sent this kid, and thou hast not found her.'); +INSERT INTO t1(docid,words) VALUES(1038024,'And it came to pass about three months after, that it was told Judah, saying, Tamar thy daughter in law hath played the harlot; and also, behold, she is with child by whoredom. And Judah said, Bring her forth, and let her be burnt.'); +INSERT INTO t1(docid,words) VALUES(1038025,'When she was brought forth, she sent to her father in law, saying, By the man, whose these are, am I with child: and she said, Discern, I pray thee, whose are these, the signet, and bracelets, and staff.'); +INSERT INTO t1(docid,words) VALUES(1038026,'And Judah acknowledged them, and said, She hath been more righteous than I; because that I gave her not to Shelah my son. And he knew her again no more.'); +INSERT INTO t1(docid,words) VALUES(1038027,'And it came to pass in the time of her travail, that, behold, twins were in her womb.'); +INSERT INTO t1(docid,words) VALUES(1038028,'And it came to pass, when she travailed, that the one put out his hand: and the midwife took and bound upon his hand a scarlet thread, saying, This came out first.'); +INSERT INTO t1(docid,words) VALUES(1038029,'And it came to pass, as he drew back his hand, that, behold, his brother came out: and she said, How hast thou broken forth? this breach be upon thee: therefore his name was called Pharez.'); +INSERT INTO t1(docid,words) VALUES(1038030,'And afterward came out his brother, that had the scarlet thread upon his hand: and his name was called Zarah.'); +INSERT INTO t1(docid,words) VALUES(1039001,'And Joseph was brought down to Egypt; and Potiphar, an officer of Pharaoh, captain of the guard, an Egyptian, bought him of the hands of the Ishmeelites, which had brought him down thither.'); +INSERT INTO t1(docid,words) VALUES(1039002,'And the LORD was with Joseph, and he was a prosperous man; and he was in the house of his master the Egyptian.'); +INSERT INTO t1(docid,words) VALUES(1039003,'And his master saw that the LORD was with him, and that the LORD made all that he did to prosper in his hand.'); +INSERT INTO t1(docid,words) VALUES(1039004,'And Joseph found grace in his sight, and he served him: and he made him overseer over his house, and all that he had he put into his hand.'); +INSERT INTO t1(docid,words) VALUES(1039005,'And it came to pass from the time that he had made him overseer in his house, and over all that he had, that the LORD blessed the Egyptian''s house for Joseph''s sake; and the blessing of the LORD was upon all that he had in the house, and in the field.'); +INSERT INTO t1(docid,words) VALUES(1039006,'And he left all that he had in Joseph''s hand; and he knew not ought he had, save the bread which he did eat. And Joseph was a goodly person, and well favoured.'); +INSERT INTO t1(docid,words) VALUES(1039007,'And it came to pass after these things, that his master''s wife cast her eyes upon Joseph; and she said, Lie with me.'); +INSERT INTO t1(docid,words) VALUES(1039008,'But he refused, and said unto his master''s wife, Behold, my master wotteth not what is with me in the house, and he hath committed all that he hath to my hand;'); +INSERT INTO t1(docid,words) VALUES(1039009,'There is none greater in this house than I; neither hath he kept back any thing from me but thee, because thou art his wife: how then can I do this great wickedness, and sin against God?'); +INSERT INTO t1(docid,words) VALUES(1039010,'And it came to pass, as she spake to Joseph day by day, that he hearkened not unto her, to lie by her, or to be with her.'); +INSERT INTO t1(docid,words) VALUES(1039011,'And it came to pass about this time, that Joseph went into the house to do his business; and there was none of the men of the house there within.'); +INSERT INTO t1(docid,words) VALUES(1039012,'And she caught him by his garment, saying, Lie with me: and he left his garment in her hand, and fled, and got him out.'); +INSERT INTO t1(docid,words) VALUES(1039013,'And it came to pass, when she saw that he had left his garment in her hand, and was fled forth,'); +INSERT INTO t1(docid,words) VALUES(1039014,'That she called unto the men of her house, and spake unto them, saying, See, he hath brought in an Hebrew unto us to mock us; he came in unto me to lie with me, and I cried with a loud voice:'); +INSERT INTO t1(docid,words) VALUES(1039015,'And it came to pass, when he heard that I lifted up my voice and cried, that he left his garment with me, and fled, and got him out.'); +INSERT INTO t1(docid,words) VALUES(1039016,'And she laid up his garment by her, until his lord came home.'); +INSERT INTO t1(docid,words) VALUES(1039017,'And she spake unto him according to these words, saying, The Hebrew servant, which thou hast brought unto us, came in unto me to mock me:'); +INSERT INTO t1(docid,words) VALUES(1039018,'And it came to pass, as I lifted up my voice and cried, that he left his garment with me, and fled out.'); +INSERT INTO t1(docid,words) VALUES(1039019,'And it came to pass, when his master heard the words of his wife, which she spake unto him, saying, After this manner did thy servant to me; that his wrath was kindled.'); +INSERT INTO t1(docid,words) VALUES(1039020,'And Joseph''s master took him, and put him into the prison, a place where the king''s prisoners were bound: and he was there in the prison.'); +INSERT INTO t1(docid,words) VALUES(1039021,'But the LORD was with Joseph, and shewed him mercy, and gave him favour in the sight of the keeper of the prison.'); +INSERT INTO t1(docid,words) VALUES(1039022,'And the keeper of the prison committed to Joseph''s hand all the prisoners that were in the prison; and whatsoever they did there, he was the doer of it.'); +INSERT INTO t1(docid,words) VALUES(1039023,'The keeper of the prison looked not to any thing that was under his hand; because the LORD was with him, and that which he did, the LORD made it to prosper.'); +INSERT INTO t1(docid,words) VALUES(1040001,'And it came to pass after these things, that the butler of the king of Egypt and his baker had offended their lord the king of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1040002,'And Pharaoh was wroth against two of his officers, against the chief of the butlers, and against the chief of the bakers.'); +INSERT INTO t1(docid,words) VALUES(1040003,'And he put them in ward in the house of the captain of the guard, into the prison, the place where Joseph was bound.'); +INSERT INTO t1(docid,words) VALUES(1040004,'And the captain of the guard charged Joseph with them, and he served them: and they continued a season in ward.'); +INSERT INTO t1(docid,words) VALUES(1040005,'And they dreamed a dream both of them, each man his dream in one night, each man according to the interpretation of his dream, the butler and the baker of the king of Egypt, which were bound in the prison.'); +INSERT INTO t1(docid,words) VALUES(1040006,'And Joseph came in unto them in the morning, and looked upon them, and, behold, they were sad.'); +INSERT INTO t1(docid,words) VALUES(1040007,'And he asked Pharaoh''s officers that were with him in the ward of his lord''s house, saying, Wherefore look ye so sadly to day?'); +INSERT INTO t1(docid,words) VALUES(1040008,'And they said unto him, We have dreamed a dream, and there is no interpreter of it. And Joseph said unto them, Do not interpretations belong to God? tell me them, I pray you.'); +INSERT INTO t1(docid,words) VALUES(1040009,'And the chief butler told his dream to Joseph, and said to him, In my dream, behold, a vine was before me;'); +INSERT INTO t1(docid,words) VALUES(1040010,'And in the vine were three branches: and it was as though it budded, and her blossoms shot forth; and the clusters thereof brought forth ripe grapes:'); +INSERT INTO t1(docid,words) VALUES(1040011,'And Pharaoh''s cup was in my hand: and I took the grapes, and pressed them into Pharaoh''s cup, and I gave the cup into Pharaoh''s hand.'); +INSERT INTO t1(docid,words) VALUES(1040012,'And Joseph said unto him, This is the interpretation of it: The three branches are three days:'); +INSERT INTO t1(docid,words) VALUES(1040013,'Yet within three days shall Pharaoh lift up thine head, and restore thee unto thy place: and thou shalt deliver Pharaoh''s cup into his hand, after the former manner when thou wast his butler.'); +INSERT INTO t1(docid,words) VALUES(1040014,'But think on me when it shall be well with thee, and shew kindness, I pray thee, unto me, and make mention of me unto Pharaoh, and bring me out of this house:'); +INSERT INTO t1(docid,words) VALUES(1040015,'For indeed I was stolen away out of the land of the Hebrews: and here also have I done nothing that they should put me into the dungeon.'); +INSERT INTO t1(docid,words) VALUES(1040016,'When the chief baker saw that the interpretation was good, he said unto Joseph, I also was in my dream, and, behold, I had three white baskets on my head:'); +INSERT INTO t1(docid,words) VALUES(1040017,'And in the uppermost basket there was of all manner of bakemeats for Pharaoh; and the birds did eat them out of the basket upon my head.'); +INSERT INTO t1(docid,words) VALUES(1040018,'And Joseph answered and said, This is the interpretation thereof: The three baskets are three days:'); +INSERT INTO t1(docid,words) VALUES(1040019,'Yet within three days shall Pharaoh lift up thy head from off thee, and shall hang thee on a tree; and the birds shall eat thy flesh from off thee.'); +INSERT INTO t1(docid,words) VALUES(1040020,'And it came to pass the third day, which was Pharaoh''s birthday, that he made a feast unto all his servants: and he lifted up the head of the chief butler and of the chief baker among his servants.'); +INSERT INTO t1(docid,words) VALUES(1040021,'And he restored the chief butler unto his butlership again; and he gave the cup into Pharaoh''s hand:'); +INSERT INTO t1(docid,words) VALUES(1040022,'But he hanged the chief baker: as Joseph had interpreted to them.'); +INSERT INTO t1(docid,words) VALUES(1040023,'Yet did not the chief butler remember Joseph, but forgat him.'); +INSERT INTO t1(docid,words) VALUES(1041001,'And it came to pass at the end of two full years, that Pharaoh dreamed: and, behold, he stood by the river.'); +INSERT INTO t1(docid,words) VALUES(1041002,'And, behold, there came up out of the river seven well favoured kine and fatfleshed; and they fed in a meadow.'); +INSERT INTO t1(docid,words) VALUES(1041003,'And, behold, seven other kine came up after them out of the river, ill favoured and leanfleshed; and stood by the other kine upon the brink of the river.'); +INSERT INTO t1(docid,words) VALUES(1041004,'And the ill favoured and leanfleshed kine did eat up the seven well favoured and fat kine. So Pharaoh awoke.'); +INSERT INTO t1(docid,words) VALUES(1041005,'And he slept and dreamed the second time: and, behold, seven ears of corn came up upon one stalk, rank and good.'); +INSERT INTO t1(docid,words) VALUES(1041006,'And, behold, seven thin ears and blasted with the east wind sprung up after them.'); +INSERT INTO t1(docid,words) VALUES(1041007,'And the seven thin ears devoured the seven rank and full ears. And Pharaoh awoke, and, behold, it was a dream.'); +INSERT INTO t1(docid,words) VALUES(1041008,'And it came to pass in the morning that his spirit was troubled; and he sent and called for all the magicians of Egypt, and all the wise men thereof: and Pharaoh told them his dream; but there was none that could interpret them unto Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1041009,'Then spake the chief butler unto Pharaoh, saying, I do remember my faults this day:'); +INSERT INTO t1(docid,words) VALUES(1041010,'Pharaoh was wroth with his servants, and put me in ward in the captain of the guard''s house, both me and the chief baker:'); +INSERT INTO t1(docid,words) VALUES(1041011,'And we dreamed a dream in one night, I and he; we dreamed each man according to the interpretation of his dream.'); +INSERT INTO t1(docid,words) VALUES(1041012,'And there was there with us a young man, an Hebrew, servant to the captain of the guard; and we told him, and he interpreted to us our dreams; to each man according to his dream he did interpret.'); +INSERT INTO t1(docid,words) VALUES(1041013,'And it came to pass, as he interpreted to us, so it was; me he restored unto mine office, and him he hanged.'); +INSERT INTO t1(docid,words) VALUES(1041014,'Then Pharaoh sent and called Joseph, and they brought him hastily out of the dungeon: and he shaved himself, and changed his raiment, and came in unto Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1041015,'And Pharaoh said unto Joseph, I have dreamed a dream, and there is none that can interpret it: and I have heard say of thee, that thou canst understand a dream to interpret it.'); +INSERT INTO t1(docid,words) VALUES(1041016,'And Joseph answered Pharaoh, saying, It is not in me: God shall give Pharaoh an answer of peace.'); +INSERT INTO t1(docid,words) VALUES(1041017,'And Pharaoh said unto Joseph, In my dream, behold, I stood upon the bank of the river:'); +INSERT INTO t1(docid,words) VALUES(1041018,'And, behold, there came up out of the river seven kine, fatfleshed and well favoured; and they fed in a meadow:'); +INSERT INTO t1(docid,words) VALUES(1041019,'And, behold, seven other kine came up after them, poor and very ill favoured and leanfleshed, such as I never saw in all the land of Egypt for badness:'); +INSERT INTO t1(docid,words) VALUES(1041020,'And the lean and the ill favoured kine did eat up the first seven fat kine:'); +INSERT INTO t1(docid,words) VALUES(1041021,'And when they had eaten them up, it could not be known that they had eaten them; but they were still ill favoured, as at the beginning. So I awoke.'); +INSERT INTO t1(docid,words) VALUES(1041022,'And I saw in my dream, and, behold, seven ears came up in one stalk, full and good:'); +INSERT INTO t1(docid,words) VALUES(1041023,'And, behold, seven ears, withered, thin, and blasted with the east wind, sprung up after them:'); +INSERT INTO t1(docid,words) VALUES(1041024,'And the thin ears devoured the seven good ears: and I told this unto the magicians; but there was none that could declare it to me.'); +INSERT INTO t1(docid,words) VALUES(1041025,'And Joseph said unto Pharaoh, The dream of Pharaoh is one: God hath shewed Pharaoh what he is about to do.'); +INSERT INTO t1(docid,words) VALUES(1041026,'The seven good kine are seven years; and the seven good ears are seven years: the dream is one.'); +INSERT INTO t1(docid,words) VALUES(1041027,'And the seven thin and ill favoured kine that came up after them are seven years; and the seven empty ears blasted with the east wind shall be seven years of famine.'); +INSERT INTO t1(docid,words) VALUES(1041028,'This is the thing which I have spoken unto Pharaoh: What God is about to do he sheweth unto Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1041029,'Behold, there come seven years of great plenty throughout all the land of Egypt:'); +INSERT INTO t1(docid,words) VALUES(1041030,'And there shall arise after them seven years of famine; and all the plenty shall be forgotten in the land of Egypt; and the famine shall consume the land;'); +INSERT INTO t1(docid,words) VALUES(1041031,'And the plenty shall not be known in the land by reason of that famine following; for it shall be very grievous.'); +INSERT INTO t1(docid,words) VALUES(1041032,'And for that the dream was doubled unto Pharaoh twice; it is because the thing is established by God, and God will shortly bring it to pass.'); +INSERT INTO t1(docid,words) VALUES(1041033,'Now therefore let Pharaoh look out a man discreet and wise, and set him over the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041034,'Let Pharaoh do this, and let him appoint officers over the land, and take up the fifth part of the land of Egypt in the seven plenteous years.'); +INSERT INTO t1(docid,words) VALUES(1041035,'And let them gather all the food of those good years that come, and lay up corn under the hand of Pharaoh, and let them keep food in the cities.'); +INSERT INTO t1(docid,words) VALUES(1041036,'And that food shall be for store to the land against the seven years of famine, which shall be in the land of Egypt; that the land perish not through the famine.'); +INSERT INTO t1(docid,words) VALUES(1041037,'And the thing was good in the eyes of Pharaoh, and in the eyes of all his servants.'); +INSERT INTO t1(docid,words) VALUES(1041038,'And Pharaoh said unto his servants, Can we find such a one as this is, a man in whom the Spirit of God is?'); +INSERT INTO t1(docid,words) VALUES(1041039,'And Pharaoh said unto Joseph, Forasmuch as God hath shewed thee all this, there is none so discreet and wise as thou art:'); +INSERT INTO t1(docid,words) VALUES(1041040,'Thou shalt be over my house, and according unto thy word shall all my people be ruled: only in the throne will I be greater than thou.'); +INSERT INTO t1(docid,words) VALUES(1041041,'And Pharaoh said unto Joseph, See, I have set thee over all the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041042,'And Pharaoh took off his ring from his hand, and put it upon Joseph''s hand, and arrayed him in vestures of fine linen, and put a gold chain about his neck;'); +INSERT INTO t1(docid,words) VALUES(1041043,'And he made him to ride in the second chariot which he had; and they cried before him, Bow the knee: and he made him ruler over all the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041044,'And Pharaoh said unto Joseph, I am Pharaoh, and without thee shall no man lift up his hand or foot in all the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041045,'And Pharaoh called Joseph''s name Zaphnathpaaneah; and he gave him to wife Asenath the daughter of Potipherah priest of On. And Joseph went out over all the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041046,'And Joseph was thirty years old when he stood before Pharaoh king of Egypt. And Joseph went out from the presence of Pharaoh, and went throughout all the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041047,'And in the seven plenteous years the earth brought forth by handfuls.'); +INSERT INTO t1(docid,words) VALUES(1041048,'And he gathered up all the food of the seven years, which were in the land of Egypt, and laid up the food in the cities: the food of the field, which was round about every city, laid he up in the same.'); +INSERT INTO t1(docid,words) VALUES(1041049,'And Joseph gathered corn as the sand of the sea, very much, until he left numbering; for it was without number.'); +INSERT INTO t1(docid,words) VALUES(1041050,'And unto Joseph were born two sons before the years of famine came, which Asenath the daughter of Potipherah priest of On bare unto him.'); +INSERT INTO t1(docid,words) VALUES(1041051,'And Joseph called the name of the firstborn Manasseh: For God, said he, hath made me forget all my toil, and all my father''s house.'); +INSERT INTO t1(docid,words) VALUES(1041052,'And the name of the second called he Ephraim: For God hath caused me to be fruitful in the land of my affliction.'); +INSERT INTO t1(docid,words) VALUES(1041053,'And the seven years of plenteousness, that was in the land of Egypt, were ended.'); +INSERT INTO t1(docid,words) VALUES(1041054,'And the seven years of dearth began to come, according as Joseph had said: and the dearth was in all lands; but in all the land of Egypt there was bread.'); +INSERT INTO t1(docid,words) VALUES(1041055,'And when all the land of Egypt was famished, the people cried to Pharaoh for bread: and Pharaoh said unto all the Egyptians, Go unto Joseph; what he saith to you, do.'); +INSERT INTO t1(docid,words) VALUES(1041056,'And the famine was over all the face of the earth: and Joseph opened all the storehouses, and sold unto the Egyptians; and the famine waxed sore in the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1041057,'And all countries came into Egypt to Joseph for to buy corn; because that the famine was so sore in all lands.'); +INSERT INTO t1(docid,words) VALUES(1042001,'Now when Jacob saw that there was corn in Egypt, Jacob said unto his sons, Why do ye look one upon another?'); +INSERT INTO t1(docid,words) VALUES(1042002,'And he said, Behold, I have heard that there is corn in Egypt: get you down thither, and buy for us from thence; that we may live, and not die.'); +INSERT INTO t1(docid,words) VALUES(1042003,'And Joseph''s ten brethren went down to buy corn in Egypt.'); +INSERT INTO t1(docid,words) VALUES(1042004,'But Benjamin, Joseph''s brother, Jacob sent not with his brethren; for he said, Lest peradventure mischief befall him.'); +INSERT INTO t1(docid,words) VALUES(1042005,'And the sons of Israel came to buy corn among those that came: for the famine was in the land of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1042006,'And Joseph was the governor over the land, and he it was that sold to all the people of the land: and Joseph''s brethren came, and bowed down themselves before him with their faces to the earth.'); +INSERT INTO t1(docid,words) VALUES(1042007,'And Joseph saw his brethren, and he knew them, but made himself strange unto them, and spake roughly unto them; and he said unto them, Whence come ye? And they said, From the land of Canaan to buy food.'); +INSERT INTO t1(docid,words) VALUES(1042008,'And Joseph knew his brethren, but they knew not him.'); +INSERT INTO t1(docid,words) VALUES(1042009,'And Joseph remembered the dreams which he dreamed of them, and said unto them, Ye are spies; to see the nakedness of the land ye are come.'); +INSERT INTO t1(docid,words) VALUES(1042010,'And they said unto him, Nay, my lord, but to buy food are thy servants come.'); +INSERT INTO t1(docid,words) VALUES(1042011,'We are all one man''s sons; we are true men, thy servants are no spies.'); +INSERT INTO t1(docid,words) VALUES(1042012,'And he said unto them, Nay, but to see the nakedness of the land ye are come.'); +INSERT INTO t1(docid,words) VALUES(1042013,'And they said, Thy servants are twelve brethren, the sons of one man in the land of Canaan; and, behold, the youngest is this day with our father, and one is not.'); +INSERT INTO t1(docid,words) VALUES(1042014,'And Joseph said unto them, That is it that I spake unto you, saying, Ye are spies:'); +INSERT INTO t1(docid,words) VALUES(1042015,'Hereby ye shall be proved: By the life of Pharaoh ye shall not go forth hence, except your youngest brother come hither.'); +INSERT INTO t1(docid,words) VALUES(1042016,'Send one of you, and let him fetch your brother, and ye shall be kept in prison, that your words may be proved, whether there be any truth in you: or else by the life of Pharaoh surely ye are spies.'); +INSERT INTO t1(docid,words) VALUES(1042017,'And he put them all together into ward three days.'); +INSERT INTO t1(docid,words) VALUES(1042018,'And Joseph said unto them the third day, This do, and live; for I fear God:'); +INSERT INTO t1(docid,words) VALUES(1042019,'If ye be true men, let one of your brethren be bound in the house of your prison: go ye, carry corn for the famine of your houses:'); +INSERT INTO t1(docid,words) VALUES(1042020,'But bring your youngest brother unto me; so shall your words be verified, and ye shall not die. And they did so.'); +INSERT INTO t1(docid,words) VALUES(1042021,'And they said one to another, We are verily guilty concerning our brother, in that we saw the anguish of his soul, when he besought us, and we would not hear; therefore is this distress come upon us.'); +INSERT INTO t1(docid,words) VALUES(1042022,'And Reuben answered them, saying, Spake I not unto you, saying, Do not sin against the child; and ye would not hear? therefore, behold, also his blood is required.'); +INSERT INTO t1(docid,words) VALUES(1042023,'And they knew not that Joseph understood them; for he spake unto them by an interpreter.'); +INSERT INTO t1(docid,words) VALUES(1042024,'And he turned himself about from them, and wept; and returned to them again, and communed with them, and took from them Simeon, and bound him before their eyes.'); +INSERT INTO t1(docid,words) VALUES(1042025,'Then Joseph commanded to fill their sacks with corn, and to restore every man''s money into his sack, and to give them provision for the way: and thus did he unto them.'); +INSERT INTO t1(docid,words) VALUES(1042026,'And they laded their asses with the corn, and departed thence.'); +INSERT INTO t1(docid,words) VALUES(1042027,'And as one of them opened his sack to give his ass provender in the inn, he espied his money; for, behold, it was in his sack''s mouth.'); +INSERT INTO t1(docid,words) VALUES(1042028,'And he said unto his brethren, My money is restored; and, lo, it is even in my sack: and their heart failed them, and they were afraid, saying one to another, What is this that God hath done unto us?'); +INSERT INTO t1(docid,words) VALUES(1042029,'And they came unto Jacob their father unto the land of Canaan, and told him all that befell unto them; saying,'); +INSERT INTO t1(docid,words) VALUES(1042030,'The man, who is the lord of the land, spake roughly to us, and took us for spies of the country.'); +INSERT INTO t1(docid,words) VALUES(1042031,'And we said unto him, We are true men; we are no spies:'); +INSERT INTO t1(docid,words) VALUES(1042032,'We be twelve brethren, sons of our father; one is not, and the youngest is this day with our father in the land of Canaan.'); +INSERT INTO t1(docid,words) VALUES(1042033,'And the man, the lord of the country, said unto us, Hereby shall I know that ye are true men; leave one of your brethren here with me, and take food for the famine of your households, and be gone:'); +INSERT INTO t1(docid,words) VALUES(1042034,'And bring your youngest brother unto me: then shall I know that ye are no spies, but that ye are true men: so will I deliver you your brother, and ye shall traffick in the land.'); +INSERT INTO t1(docid,words) VALUES(1042035,'And it came to pass as they emptied their sacks, that, behold, every man''s bundle of money was in his sack: and when both they and their father saw the bundles of money, they were afraid.'); +INSERT INTO t1(docid,words) VALUES(1042036,'And Jacob their father said unto them, Me have ye bereaved of my children: Joseph is not, and Simeon is not, and ye will take Benjamin away: all these things are against me.'); +INSERT INTO t1(docid,words) VALUES(1042037,'And Reuben spake unto his father, saying, Slay my two sons, if I bring him not to thee: deliver him into my hand, and I will bring him to thee again.'); +INSERT INTO t1(docid,words) VALUES(1042038,'And he said, My son shall not go down with you; for his brother is dead, and he is left alone: if mischief befall him by the way in the which ye go, then shall ye bring down my gray hairs with sorrow to the grave.'); +INSERT INTO t1(docid,words) VALUES(1043001,'And the famine was sore in the land.'); +INSERT INTO t1(docid,words) VALUES(1043002,'And it came to pass, when they had eaten up the corn which they had brought out of Egypt, their father said unto them, Go again, buy us a little food.'); +INSERT INTO t1(docid,words) VALUES(1043003,'And Judah spake unto him, saying, The man did solemnly protest unto us, saying, Ye shall not see my face, except your brother be with you.'); +INSERT INTO t1(docid,words) VALUES(1043004,'If thou wilt send our brother with us, we will go down and buy thee food:'); +INSERT INTO t1(docid,words) VALUES(1043005,'But if thou wilt not send him, we will not go down: for the man said unto us, Ye shall not see my face, except your brother be with you.'); +INSERT INTO t1(docid,words) VALUES(1043006,'And Israel said, Wherefore dealt ye so ill with me, as to tell the man whether ye had yet a brother?'); +INSERT INTO t1(docid,words) VALUES(1043007,'And they said, The man asked us straitly of our state, and of our kindred, saying, Is your father yet alive? have ye another brother? and we told him according to the tenor of these words: could we certainly know that he would say, Bring your brother down?'); +INSERT INTO t1(docid,words) VALUES(1043008,'And Judah said unto Israel his father, Send the lad with me, and we will arise and go; that we may live, and not die, both we, and thou, and also our little ones.'); +INSERT INTO t1(docid,words) VALUES(1043009,'I will be surety for him; of my hand shalt thou require him: if I bring him not unto thee, and set him before thee, then let me bear the blame for ever:'); +INSERT INTO t1(docid,words) VALUES(1043010,'For except we had lingered, surely now we had returned this second time.'); +INSERT INTO t1(docid,words) VALUES(1043011,'And their father Israel said unto them, If it must be so now, do this; take of the best fruits in the land in your vessels, and carry down the man a present, a little balm, and a little honey, spices, and myrrh, nuts, and almonds:'); +INSERT INTO t1(docid,words) VALUES(1043012,'And take double money in your hand; and the money that was brought again in the mouth of your sacks, carry it again in your hand; peradventure it was an oversight:'); +INSERT INTO t1(docid,words) VALUES(1043013,'Take also your brother, and arise, go again unto the man:'); +INSERT INTO t1(docid,words) VALUES(1043014,'And God Almighty give you mercy before the man, that he may send away your other brother, and Benjamin. If I be bereaved of my children, I am bereaved.'); +INSERT INTO t1(docid,words) VALUES(1043015,'And the men took that present, and they took double money in their hand and Benjamin; and rose up, and went down to Egypt, and stood before Joseph.'); +INSERT INTO t1(docid,words) VALUES(1043016,'And when Joseph saw Benjamin with them, he said to the ruler of his house, Bring these men home, and slay, and make ready; for these men shall dine with me at noon.'); +INSERT INTO t1(docid,words) VALUES(1043017,'And the man did as Joseph bade; and the man brought the men into Joseph''s house.'); +INSERT INTO t1(docid,words) VALUES(1043018,'And the men were afraid, because they were brought into Joseph''s house; and they said, Because of the money that was returned in our sacks at the first time are we brought in; that he may seek occasion against us, and fall upon us, and take us for bondmen, and our asses.'); +INSERT INTO t1(docid,words) VALUES(1043019,'And they came near to the steward of Joseph''s house, and they communed with him at the door of the house,'); +INSERT INTO t1(docid,words) VALUES(1043020,'And said, O sir, we came indeed down at the first time to buy food:'); +INSERT INTO t1(docid,words) VALUES(1043021,'And it came to pass, when we came to the inn, that we opened our sacks, and, behold, every man''s money was in the mouth of his sack, our money in full weight: and we have brought it again in our hand.'); +INSERT INTO t1(docid,words) VALUES(1043022,'And other money have we brought down in our hands to buy food: we cannot tell who put our money in our sacks.'); +INSERT INTO t1(docid,words) VALUES(1043023,'And he said, Peace be to you, fear not: your God, and the God of your father, hath given you treasure in your sacks: I had your money. And he brought Simeon out unto them.'); +INSERT INTO t1(docid,words) VALUES(1043024,'And the man brought the men into Joseph''s house, and gave them water, and they washed their feet; and he gave their asses provender.'); +INSERT INTO t1(docid,words) VALUES(1043025,'And they made ready the present against Joseph came at noon: for they heard that they should eat bread there.'); +INSERT INTO t1(docid,words) VALUES(1043026,'And when Joseph came home, they brought him the present which was in their hand into the house, and bowed themselves to him to the earth.'); +INSERT INTO t1(docid,words) VALUES(1043027,'And he asked them of their welfare, and said, Is your father well, the old man of whom ye spake? Is he yet alive?'); +INSERT INTO t1(docid,words) VALUES(1043028,'And they answered, Thy servant our father is in good health, he is yet alive. And they bowed down their heads, and made obeisance.'); +INSERT INTO t1(docid,words) VALUES(1043029,'And he lifted up his eyes, and saw his brother Benjamin, his mother''s son, and said, Is this your younger brother, of whom ye spake unto me? And he said, God be gracious unto thee, my son.'); +INSERT INTO t1(docid,words) VALUES(1043030,'And Joseph made haste; for his bowels did yearn upon his brother: and he sought where to weep; and he entered into his chamber, and wept there.'); +INSERT INTO t1(docid,words) VALUES(1043031,'And he washed his face, and went out, and refrained himself, and said, Set on bread.'); +INSERT INTO t1(docid,words) VALUES(1043032,'And they set on for him by himself, and for them by themselves, and for the Egyptians, which did eat with him, by themselves: because the Egyptians might not eat bread with the Hebrews; for that is an abomination unto the Egyptians.'); +INSERT INTO t1(docid,words) VALUES(1043033,'And they sat before him, the firstborn according to his birthright, and the youngest according to his youth: and the men marvelled one at another.'); +INSERT INTO t1(docid,words) VALUES(1043034,'And he took and sent messes unto them from before him: but Benjamin''s mess was five times so much as any of their''s. And they drank, and were merry with him.'); +INSERT INTO t1(docid,words) VALUES(1044001,'And he commanded the steward of his house, saying, Fill the men''s sacks with food, as much as they can carry, and put every man''s money in his sack''s mouth.'); +INSERT INTO t1(docid,words) VALUES(1044002,'And put my cup, the silver cup, in the sack''s mouth of the youngest, and his corn money. And he did according to the word that Joseph had spoken.'); +INSERT INTO t1(docid,words) VALUES(1044003,'As soon as the morning was light, the men were sent away, they and their asses.'); +INSERT INTO t1(docid,words) VALUES(1044004,'And when they were gone out of the city, and not yet far off, Joseph said unto his steward, Up, follow after the men; and when thou dost overtake them, say unto them, Wherefore have ye rewarded evil for good?'); +INSERT INTO t1(docid,words) VALUES(1044005,'Is not this it in which my lord drinketh, and whereby indeed he divineth? ye have done evil in so doing.'); +INSERT INTO t1(docid,words) VALUES(1044006,'And he overtook them, and he spake unto them these same words.'); +INSERT INTO t1(docid,words) VALUES(1044007,'And they said unto him, Wherefore saith my lord these words? God forbid that thy servants should do according to this thing:'); +INSERT INTO t1(docid,words) VALUES(1044008,'Behold, the money, which we found in our sacks'' mouths, we brought again unto thee out of the land of Canaan: how then should we steal out of thy lord''s house silver or gold?'); +INSERT INTO t1(docid,words) VALUES(1044009,'With whomsoever of thy servants it be found, both let him die, and we also will be my lord''s bondmen.'); +INSERT INTO t1(docid,words) VALUES(1044010,'And he said, Now also let it be according unto your words: he with whom it is found shall be my servant; and ye shall be blameless.'); +INSERT INTO t1(docid,words) VALUES(1044011,'Then they speedily took down every man his sack to the ground, and opened every man his sack.'); +INSERT INTO t1(docid,words) VALUES(1044012,'And he searched, and began at the eldest, and left at the youngest: and the cup was found in Benjamin''s sack.'); +INSERT INTO t1(docid,words) VALUES(1044013,'Then they rent their clothes, and laded every man his ass, and returned to the city.'); +INSERT INTO t1(docid,words) VALUES(1044014,'And Judah and his brethren came to Joseph''s house; for he was yet there: and they fell before him on the ground.'); +INSERT INTO t1(docid,words) VALUES(1044015,'And Joseph said unto them, What deed is this that ye have done? wot ye not that such a man as I can certainly divine?'); +INSERT INTO t1(docid,words) VALUES(1044016,'And Judah said, What shall we say unto my lord? what shall we speak? or how shall we clear ourselves? God hath found out the iniquity of thy servants: behold, we are my lord''s servants, both we, and he also with whom the cup is found.'); +INSERT INTO t1(docid,words) VALUES(1044017,'And he said, God forbid that I should do so: but the man in whose hand the cup is found, he shall be my servant; and as for you, get you up in peace unto your father.'); +INSERT INTO t1(docid,words) VALUES(1044018,'Then Judah came near unto him, and said, Oh my lord, let thy servant, I pray thee, speak a word in my lord''s ears, and let not thine anger burn against thy servant: for thou art even as Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1044019,'My lord asked his servants, saying, Have ye a father, or a brother?'); +INSERT INTO t1(docid,words) VALUES(1044020,'And we said unto my lord, We have a father, an old man, and a child of his old age, a little one; and his brother is dead, and he alone is left of his mother, and his father loveth him.'); +INSERT INTO t1(docid,words) VALUES(1044021,'And thou saidst unto thy servants, Bring him down unto me, that I may set mine eyes upon him.'); +INSERT INTO t1(docid,words) VALUES(1044022,'And we said unto my lord, The lad cannot leave his father: for if he should leave his father, his father would die.'); +INSERT INTO t1(docid,words) VALUES(1044023,'And thou saidst unto thy servants, Except your youngest brother come down with you, ye shall see my face no more.'); +INSERT INTO t1(docid,words) VALUES(1044024,'And it came to pass when we came up unto thy servant my father, we told him the words of my lord.'); +INSERT INTO t1(docid,words) VALUES(1044025,'And our father said, Go again, and buy us a little food.'); +INSERT INTO t1(docid,words) VALUES(1044026,'And we said, We cannot go down: if our youngest brother be with us, then will we go down: for we may not see the man''s face, except our youngest brother be with us.'); +INSERT INTO t1(docid,words) VALUES(1044027,'And thy servant my father said unto us, Ye know that my wife bare me two sons:'); +INSERT INTO t1(docid,words) VALUES(1044028,'And the one went out from me, and I said, Surely he is torn in pieces; and I saw him not since:'); +INSERT INTO t1(docid,words) VALUES(1044029,'And if ye take this also from me, and mischief befall him, ye shall bring down my gray hairs with sorrow to the grave.'); +INSERT INTO t1(docid,words) VALUES(1044030,'Now therefore when I come to thy servant my father, and the lad be not with us; seeing that his life is bound up in the lad''s life;'); +INSERT INTO t1(docid,words) VALUES(1044031,'It shall come to pass, when he seeth that the lad is not with us, that he will die: and thy servants shall bring down the gray hairs of thy servant our father with sorrow to the grave.'); +INSERT INTO t1(docid,words) VALUES(1044032,'For thy servant became surety for the lad unto my father, saying, If I bring him not unto thee, then I shall bear the blame to my father for ever.'); +INSERT INTO t1(docid,words) VALUES(1044033,'Now therefore, I pray thee, let thy servant abide instead of the lad a bondman to my lord; and let the lad go up with his brethren.'); +INSERT INTO t1(docid,words) VALUES(1044034,'For how shall I go up to my father, and the lad be not with me? lest peradventure I see the evil that shall come on my father.'); +INSERT INTO t1(docid,words) VALUES(1045001,'Then Joseph could not refrain himself before all them that stood by him; and he cried, Cause every man to go out from me. And there stood no man with him, while Joseph made himself known unto his brethren.'); +INSERT INTO t1(docid,words) VALUES(1045002,'And he wept aloud: and the Egyptians and the house of Pharaoh heard.'); +INSERT INTO t1(docid,words) VALUES(1045003,'And Joseph said unto his brethren, I am Joseph; doth my father yet live? And his brethren could not answer him; for they were troubled at his presence.'); +INSERT INTO t1(docid,words) VALUES(1045004,'And Joseph said unto his brethren, Come near to me, I pray you. And they came near. And he said, I am Joseph your brother, whom ye sold into Egypt.'); +INSERT INTO t1(docid,words) VALUES(1045005,'Now therefore be not grieved, nor angry with yourselves, that ye sold me hither: for God did send me before you to preserve life.'); +INSERT INTO t1(docid,words) VALUES(1045006,'For these two years hath the famine been in the land: and yet there are five years, in the which there shall neither be earing nor harvest.'); +INSERT INTO t1(docid,words) VALUES(1045007,'And God sent me before you to preserve you a posterity in the earth, and to save your lives by a great deliverance.'); +INSERT INTO t1(docid,words) VALUES(1045008,'So now it was not you that sent me hither, but God: and he hath made me a father to Pharaoh, and lord of all his house, and a ruler throughout all the land of Egypt.'); +INSERT INTO t1(docid,words) VALUES(1045009,'Haste ye, and go up to my father, and say unto him, Thus saith thy son Joseph, God hath made me lord of all Egypt: come down unto me, tarry not:'); +INSERT INTO t1(docid,words) VALUES(1045010,'And thou shalt dwell in the land of Goshen, and thou shalt be near unto me, thou, and thy children, and thy children''s children, and thy flocks, and thy herds, and all that thou hast:'); +INSERT INTO t1(docid,words) VALUES(1045011,'And there will I nourish thee; for yet there are five years of famine; lest thou, and thy household, and all that thou hast, come to poverty.'); +INSERT INTO t1(docid,words) VALUES(1045012,'And, behold, your eyes see, and the eyes of my brother Benjamin, that it is my mouth that speaketh unto you.'); +INSERT INTO t1(docid,words) VALUES(1045013,'And ye shall tell my father of all my glory in Egypt, and of all that ye have seen; and ye shall haste and bring down my father hither.'); +INSERT INTO t1(docid,words) VALUES(1045014,'And he fell upon his brother Benjamin''s neck, and wept; and Benjamin wept upon his neck.'); +INSERT INTO t1(docid,words) VALUES(1045015,'Moreover he kissed all his brethren, and wept upon them: and after that his brethren talked with him.'); +INSERT INTO t1(docid,words) VALUES(1045016,'And the fame thereof was heard in Pharaoh''s house, saying, Joseph''s brethren are come: and it pleased Pharaoh well, and his servants.'); +INSERT INTO t1(docid,words) VALUES(1045017,'And Pharaoh said unto Joseph, Say unto thy brethren, This do ye; lade your beasts, and go, get you unto the land of Canaan;'); +INSERT INTO t1(docid,words) VALUES(1045018,'And take your father and your households, and come unto me: and I will give you the good of the land of Egypt, and ye shall eat the fat of the land.'); +INSERT INTO t1(docid,words) VALUES(1045019,'Now thou art commanded, this do ye; take you wagons out of the land of Egypt for your little ones, and for your wives, and bring your father, and come.'); +INSERT INTO t1(docid,words) VALUES(1045020,'Also regard not your stuff; for the good of all the land of Egypt is your''s.'); +INSERT INTO t1(docid,words) VALUES(1045021,'And the children of Israel did so: and Joseph gave them wagons, according to the commandment of Pharaoh, and gave them provision for the way.'); +INSERT INTO t1(docid,words) VALUES(1045022,'To all of them he gave each man changes of raiment; but to Benjamin he gave three hundred pieces of silver, and five changes of raiment.'); +INSERT INTO t1(docid,words) VALUES(1045023,'And to his father he sent after this manner; ten asses laden with the good things of Egypt, and ten she asses laden with corn and bread and meat for his father by the way.'); +INSERT INTO t1(docid,words) VALUES(1045024,'So he sent his brethren away, and they departed: and he said unto them, See that ye fall not out by the way.'); +INSERT INTO t1(docid,words) VALUES(1045025,'And they went up out of Egypt, and came into the land of Canaan unto Jacob their father,'); +INSERT INTO t1(docid,words) VALUES(1045026,'And told him, saying, Joseph is yet alive, and he is governor over all the land of Egypt. And Jacob''s heart fainted, for he believed them not.'); +INSERT INTO t1(docid,words) VALUES(1045027,'And they told him all the words of Joseph, which he had said unto them: and when he saw the wagons which Joseph had sent to carry him, the spirit of Jacob their father revived:'); +INSERT INTO t1(docid,words) VALUES(1045028,'And Israel said, It is enough; Joseph my son is yet alive: I will go and see him before I die.'); +INSERT INTO t1(docid,words) VALUES(1046001,'And Israel took his journey with all that he had, and came to Beersheba, and offered sacrifices unto the God of his father Isaac.'); +INSERT INTO t1(docid,words) VALUES(1046002,'And God spake unto Israel in the visions of the night, and said, Jacob, Jacob. And he said, Here am I.'); +INSERT INTO t1(docid,words) VALUES(1046003,'And he said, I am God, the God of thy father: fear not to go down into Egypt; for I will there make of thee a great nation:'); +INSERT INTO t1(docid,words) VALUES(1046004,'I will go down with thee into Egypt; and I will also surely bring thee up again: and Joseph shall put his hand upon thine eyes.'); +INSERT INTO t1(docid,words) VALUES(1046005,'And Jacob rose up from Beersheba: and the sons of Israel carried Jacob their father, and their little ones, and their wives, in the wagons which Pharaoh had sent to carry him.'); +INSERT INTO t1(docid,words) VALUES(1046006,'And they took their cattle, and their goods, which they had gotten in the land of Canaan, and came into Egypt, Jacob, and all his seed with him:'); +INSERT INTO t1(docid,words) VALUES(1046007,'His sons, and his sons'' sons with him, his daughters, and his sons'' daughters, and all his seed brought he with him into Egypt.'); +INSERT INTO t1(docid,words) VALUES(1046008,'And these are the names of the children of Israel, which came into Egypt, Jacob and his sons: Reuben, Jacob''s firstborn.'); +INSERT INTO t1(docid,words) VALUES(1046009,'And the sons of Reuben; Hanoch, and Phallu, and Hezron, and Carmi.'); +INSERT INTO t1(docid,words) VALUES(1046010,'And the sons of Simeon; Jemuel, and Jamin, and Ohad, and Jachin, and Zohar, and Shaul the son of a Canaanitish woman.'); +INSERT INTO t1(docid,words) VALUES(1046011,'And the sons of Levi; Gershon, Kohath, and Merari.'); +INSERT INTO t1(docid,words) VALUES(1046012,'And the sons of Judah; Er, and Onan, and Shelah, and Pharez, and Zarah: but Er and Onan died in the land of Canaan. And the sons of Pharez were Hezron and Hamul.'); +INSERT INTO t1(docid,words) VALUES(1046013,'And the sons of Issachar; Tola, and Phuvah, and Job, and Shimron.'); +INSERT INTO t1(docid,words) VALUES(1046014,'And the sons of Zebulun; Sered, and Elon, and Jahleel.'); +INSERT INTO t1(docid,words) VALUES(1046015,'These be the sons of Leah, which she bare unto Jacob in Padanaram, with his daughter Dinah: all the souls of his sons and his daughters were thirty and three.'); +INSERT INTO t1(docid,words) VALUES(1046016,'And the sons of Gad; Ziphion, and Haggi, Shuni, and Ezbon, Eri, and Arodi, and Areli.'); +INSERT INTO t1(docid,words) VALUES(1046017,'And the sons of Asher; Jimnah, and Ishuah, and Isui, and Beriah, and Serah their sister: and the sons of Beriah; Heber, and Malchiel.'); +INSERT INTO t1(docid,words) VALUES(1046018,'These are the sons of Zilpah, whom Laban gave to Leah his daughter, and these she bare unto Jacob, even sixteen souls.'); +INSERT INTO t1(docid,words) VALUES(1046019,'The sons of Rachel Jacob''s wife; Joseph, and Benjamin.'); +INSERT INTO t1(docid,words) VALUES(1046020,'And unto Joseph in the land of Egypt were born Manasseh and Ephraim, which Asenath the daughter of Potipherah priest of On bare unto him.'); +INSERT INTO t1(docid,words) VALUES(1046021,'And the sons of Benjamin were Belah, and Becher, and Ashbel, Gera, and Naaman, Ehi, and Rosh, Muppim, and Huppim, and Ard.'); +INSERT INTO t1(docid,words) VALUES(1046022,'These are the sons of Rachel, which were born to Jacob: all the souls were fourteen.'); +INSERT INTO t1(docid,words) VALUES(1046023,'And the sons of Dan; Hushim.'); +INSERT INTO t1(docid,words) VALUES(1046024,'And the sons of Naphtali; Jahzeel, and Guni, and Jezer, and Shillem.'); +INSERT INTO t1(docid,words) VALUES(1046025,'These are the sons of Bilhah, which Laban gave unto Rachel his daughter, and she bare these unto Jacob: all the souls were seven.'); +INSERT INTO t1(docid,words) VALUES(1046026,'All the souls that came with Jacob into Egypt, which came out of his loins, besides Jacob''s sons'' wives, all the souls were threescore and six;'); +INSERT INTO t1(docid,words) VALUES(1046027,'And the sons of Joseph, which were born him in Egypt, were two souls: all the souls of the house of Jacob, which came into Egypt, were threescore and ten.'); +INSERT INTO t1(docid,words) VALUES(1046028,'And he sent Judah before him unto Joseph, to direct his face unto Goshen; and they came into the land of Goshen.'); +INSERT INTO t1(docid,words) VALUES(1046029,'And Joseph made ready his chariot, and went up to meet Israel his father, to Goshen, and presented himself unto him; and he fell on his neck, and wept on his neck a good while.'); +INSERT INTO t1(docid,words) VALUES(1046030,'And Israel said unto Joseph, Now let me die, since I have seen thy face, because thou art yet alive.'); +INSERT INTO t1(docid,words) VALUES(1046031,'And Joseph said unto his brethren, and unto his father''s house, I will go up, and shew Pharaoh, and say unto him, My brethren, and my father''s house, which were in the land of Canaan, are come unto me;'); +INSERT INTO t1(docid,words) VALUES(1046032,'And the men are shepherds, for their trade hath been to feed cattle; and they have brought their flocks, and their herds, and all that they have.'); +INSERT INTO t1(docid,words) VALUES(1046033,'And it shall come to pass, when Pharaoh shall call you, and shall say, What is your occupation?'); +INSERT INTO t1(docid,words) VALUES(1046034,'That ye shall say, Thy servants'' trade hath been about cattle from our youth even until now, both we, and also our fathers: that ye may dwell in the land of Goshen; for every shepherd is an abomination unto the Egyptians.'); +INSERT INTO t1(docid,words) VALUES(1047001,'Then Joseph came and told Pharaoh, and said, My father and my brethren, and their flocks, and their herds, and all that they have, are come out of the land of Canaan; and, behold, they are in the land of Goshen.'); +INSERT INTO t1(docid,words) VALUES(1047002,'And he took some of his brethren, even five men, and presented them unto Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1047003,'And Pharaoh said unto his brethren, What is your occupation? And they said unto Pharaoh, Thy servants are shepherds, both we, and also our fathers.'); +INSERT INTO t1(docid,words) VALUES(1047004,'They said morever unto Pharaoh, For to sojourn in the land are we come; for thy servants have no pasture for their flocks; for the famine is sore in the land of Canaan: now therefore, we pray thee, let thy servants dwell in the land of Goshen.'); +INSERT INTO t1(docid,words) VALUES(1047005,'And Pharaoh spake unto Joseph, saying, Thy father and thy brethren are come unto thee:'); +INSERT INTO t1(docid,words) VALUES(1047006,'The land of Egypt is before thee; in the best of the land make thy father and brethren to dwell; in the land of Goshen let them dwell: and if thou knowest any men of activity among them, then make them rulers over my cattle.'); +INSERT INTO t1(docid,words) VALUES(1047007,'And Joseph brought in Jacob his father, and set him before Pharaoh: and Jacob blessed Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1047008,'And Pharaoh said unto Jacob, How old art thou?'); +INSERT INTO t1(docid,words) VALUES(1047009,'And Jacob said unto Pharaoh, The days of the years of my pilgrimage are an hundred and thirty years: few and evil have the days of the years of my life been, and have not attained unto the days of the years of the life of my fathers in the days of their pilgrimage.'); +INSERT INTO t1(docid,words) VALUES(1047010,'And Jacob blessed Pharaoh, and went out from before Pharaoh.'); +INSERT INTO t1(docid,words) VALUES(1047011,'And Joseph placed his father and his brethren, and gave them a possession in the land of Egypt, in the best of the land, in the land of Rameses, as Pharaoh had commanded.'); +INSERT INTO t1(docid,words) VALUES(1047012,'And Joseph nourished his father, and his brethren, and all his father''s household, with bread, according to their families.'); +INSERT INTO t1(docid,words) VALUES(1047013,'And there was no bread in all the land; for the famine was very sore, so that the land of Egypt and all the land of Canaan fainted by reason of the famine.'); +INSERT INTO t1(docid,words) VALUES(1047014,'And Joseph gathered up all the money that was found in the land of Egypt, and in the land of Canaan, for the corn which they bought: and Joseph brought the money into Pharaoh''s house.'); +INSERT INTO t1(docid,words) VALUES(1047015,'And when money failed in the land of Egypt, and in the land of Canaan, all the Egyptians came unto Joseph, and said, Give us bread: for why should we die in thy presence? for the money faileth.'); +INSERT INTO t1(docid,words) VALUES(1047016,'And Joseph said, Give your cattle; and I will give you for your cattle, if money fail.'); +INSERT INTO t1(docid,words) VALUES(1047017,'And they brought their cattle unto Joseph: and Joseph gave them bread in exchange for horses, and for the flocks, and for the cattle of the herds, and for the asses: and he fed them with bread for all their cattle for that year.'); +INSERT INTO t1(docid,words) VALUES(1047018,'When that year was ended, they came unto him the second year, and said unto him, We will not hide it from my lord, how that our money is spent; my lord also hath our herds of cattle; there is not ought left in the sight of my lord, but our bodies, and our lands:'); +INSERT INTO t1(docid,words) VALUES(1047019,'Wherefore shall we die before thine eyes, both we and our land? buy us and our land for bread, and we and our land will be servants unto Pharaoh: and give us seed, that we may live, and not die, that the land be not desolate.'); +INSERT INTO t1(docid,words) VALUES(1047020,'And Joseph bought all the land of Egypt for Pharaoh; for the Egyptians sold every man his field, because the famine prevailed over them: so the land became Pharaoh''s.'); +INSERT INTO t1(docid,words) VALUES(1047021,'And as for the people, he removed them to cities from one end of the borders of Egypt even to the other end thereof.'); +INSERT INTO t1(docid,words) VALUES(1047022,'Only the land of the priests bought he not; for the priests had a portion assigned them of Pharaoh, and did eat their portion which Pharaoh gave them: wherefore they sold not their lands.'); +INSERT INTO t1(docid,words) VALUES(1047023,'Then Joseph said unto the people, Behold, I have bought you this day and your land for Pharaoh: lo, here is seed for you, and ye shall sow the land.'); +INSERT INTO t1(docid,words) VALUES(1047024,'And it shall come to pass in the increase, that ye shall give the fifth part unto Pharaoh, and four parts shall be your own, for seed of the field, and for your food, and for them of your households, and for food for your little ones.'); +INSERT INTO t1(docid,words) VALUES(1047025,'And they said, Thou hast saved our lives: let us find grace in the sight of my lord, and we will be Pharaoh''s servants.'); +INSERT INTO t1(docid,words) VALUES(1047026,'And Joseph made it a law over the land of Egypt unto this day, that Pharaoh should have the fifth part, except the land of the priests only, which became not Pharaoh''s.'); +INSERT INTO t1(docid,words) VALUES(1047027,'And Israel dwelt in the land of Egypt, in the country of Goshen; and they had possessions therein, and grew, and multiplied exceedingly.'); +INSERT INTO t1(docid,words) VALUES(1047028,'And Jacob lived in the land of Egypt seventeen years: so the whole age of Jacob was an hundred forty and seven years.'); +INSERT INTO t1(docid,words) VALUES(1047029,'And the time drew nigh that Israel must die: and he called his son Joseph, and said unto him, If now I have found grace in thy sight, put, I pray thee, thy hand under my thigh, and deal kindly and truly with me; bury me not, I pray thee, in Egypt:'); +INSERT INTO t1(docid,words) VALUES(1047030,'But I will lie with my fathers, and thou shalt carry me out of Egypt, and bury me in their buryingplace. And he said, I will do as thou hast said.'); +INSERT INTO t1(docid,words) VALUES(1047031,'And he said, Swear unto me. And he sware unto him. And Israel bowed himself upon the bed''s head.'); +INSERT INTO t1(docid,words) VALUES(1048001,'And it came to pass after these things, that one told Joseph, Behold, thy father is sick: and he took with him his two sons, Manasseh and Ephraim.'); +INSERT INTO t1(docid,words) VALUES(1048002,'And one told Jacob, and said, Behold, thy son Joseph cometh unto thee: and Israel strengthened himself, and sat upon the bed.'); +INSERT INTO t1(docid,words) VALUES(1048003,'And Jacob said unto Joseph, God Almighty appeared unto me at Luz in the land of Canaan, and blessed me,'); +INSERT INTO t1(docid,words) VALUES(1048004,'And said unto me, Behold, I will make thee fruitful, and multiply thee, and I will make of thee a multitude of people; and will give this land to thy seed after thee for an everlasting possession.'); +INSERT INTO t1(docid,words) VALUES(1048005,'And now thy two sons, Ephraim and Manasseh, which were born unto thee in the land of Egypt before I came unto thee into Egypt, are mine; as Reuben and Simeon, they shall be mine.'); +INSERT INTO t1(docid,words) VALUES(1048006,'And thy issue, which thou begettest after them, shall be thine, and shall be called after the name of their brethren in their inheritance.'); +INSERT INTO t1(docid,words) VALUES(1048007,'And as for me, when I came from Padan, Rachel died by me in the land of Canaan in the way, when yet there was but a little way to come unto Ephrath: and I buried her there in the way of Ephrath; the same is Bethlehem.'); +INSERT INTO t1(docid,words) VALUES(1048008,'And Israel beheld Joseph''s sons, and said, Who are these?'); +INSERT INTO t1(docid,words) VALUES(1048009,'And Joseph said unto his father, They are my sons, whom God hath given me in this place. And he said, Bring them, I pray thee, unto me, and I will bless them.'); +INSERT INTO t1(docid,words) VALUES(1048010,'Now the eyes of Israel were dim for age, so that he could not see. And he brought them near unto him; and he kissed them, and embraced them.'); +INSERT INTO t1(docid,words) VALUES(1048011,'And Israel said unto Joseph, I had not thought to see thy face: and, lo, God hath shewed me also thy seed.'); +INSERT INTO t1(docid,words) VALUES(1048012,'And Joseph brought them out from between his knees, and he bowed himself with his face to the earth.'); +INSERT INTO t1(docid,words) VALUES(1048013,'And Joseph took them both, Ephraim in his right hand toward Israel''s left hand, and Manasseh in his left hand toward Israel''s right hand, and brought them near unto him.'); +INSERT INTO t1(docid,words) VALUES(1048014,'And Israel stretched out his right hand, and laid it upon Ephraim''s head, who was the younger, and his left hand upon Manasseh''s head, guiding his hands wittingly; for Manasseh was the firstborn.'); +INSERT INTO t1(docid,words) VALUES(1048015,'And he blessed Joseph, and said, God, before whom my fathers Abraham and Isaac did walk, the God which fed me all my life long unto this day,'); +INSERT INTO t1(docid,words) VALUES(1048016,'The Angel which redeemed me from all evil, bless the lads; and let my name be named on them, and the name of my fathers Abraham and Isaac; and let them grow into a multitude in the midst of the earth.'); +INSERT INTO t1(docid,words) VALUES(1048017,'And when Joseph saw that his father laid his right hand upon the head of Ephraim, it displeased him: and he held up his father''s hand, to remove it from Ephraim''s head unto Manasseh''s head.'); +INSERT INTO t1(docid,words) VALUES(1048018,'And Joseph said unto his father, Not so, my father: for this is the firstborn; put thy right hand upon his head.'); +INSERT INTO t1(docid,words) VALUES(1048019,'And his father refused, and said, I know it, my son, I know it: he also shall become a people, and he also shall be great: but truly his younger brother shall be greater than he, and his seed shall become a multitude of nations.'); +INSERT INTO t1(docid,words) VALUES(1048020,'And he blessed them that day, saying, In thee shall Israel bless, saying, God make thee as Ephraim and as Manasseh: and he set Ephraim before Manasseh.'); +INSERT INTO t1(docid,words) VALUES(1048021,'And Israel said unto Joseph, Behold, I die: but God shall be with you, and bring you again unto the land of your fathers.'); +INSERT INTO t1(docid,words) VALUES(1048022,'Moreover I have given to thee one portion above thy brethren, which I took out of the hand of the Amorite with my sword and with my bow.'); +INSERT INTO t1(docid,words) VALUES(1049001,'And Jacob called unto his sons, and said, Gather yourselves together, that I may tell you that which shall befall you in the last days.'); +INSERT INTO t1(docid,words) VALUES(1049002,'Gather yourselves together, and hear, ye sons of Jacob; and hearken unto Israel your father.'); +INSERT INTO t1(docid,words) VALUES(1049003,'Reuben, thou art my firstborn, my might, and the beginning of my strength, the excellency of dignity, and the excellency of power:'); +INSERT INTO t1(docid,words) VALUES(1049004,'Unstable as water, thou shalt not excel; because thou wentest up to thy father''s bed; then defiledst thou it: he went up to my couch.'); +INSERT INTO t1(docid,words) VALUES(1049005,'Simeon and Levi are brethren; instruments of cruelty are in their habitations.'); +INSERT INTO t1(docid,words) VALUES(1049006,'O my soul, come not thou into their secret; unto their assembly, mine honour, be not thou united: for in their anger they slew a man, and in their selfwill they digged down a wall.'); +INSERT INTO t1(docid,words) VALUES(1049007,'Cursed be their anger, for it was fierce; and their wrath, for it was cruel: I will divide them in Jacob, and scatter them in Israel.'); +INSERT INTO t1(docid,words) VALUES(1049008,'Judah, thou art he whom thy brethren shall praise: thy hand shall be in the neck of thine enemies; thy father''s children shall bow down before thee.'); +INSERT INTO t1(docid,words) VALUES(1049009,'Judah is a lion''s whelp: from the prey, my son, thou art gone up: he stooped down, he couched as a lion, and as an old lion; who shall rouse him up?'); +INSERT INTO t1(docid,words) VALUES(1049010,'The sceptre shall not depart from Judah, nor a lawgiver from between his feet, until Shiloh come; and unto him shall the gathering of the people be.'); +INSERT INTO t1(docid,words) VALUES(1049011,'Binding his foal unto the vine, and his ass''s colt unto the choice vine; he washed his garments in wine, and his clothes in the blood of grapes:'); +INSERT INTO t1(docid,words) VALUES(1049012,'His eyes shall be red with wine, and his teeth white with milk.'); +INSERT INTO t1(docid,words) VALUES(1049013,'Zebulun shall dwell at the haven of the sea; and he shall be for an haven of ships; and his border shall be unto Zidon.'); +INSERT INTO t1(docid,words) VALUES(1049014,'Issachar is a strong ass couching down between two burdens:'); +INSERT INTO t1(docid,words) VALUES(1049015,'And he saw that rest was good, and the land that it was pleasant; and bowed his shoulder to bear, and became a servant unto tribute.'); +INSERT INTO t1(docid,words) VALUES(1049016,'Dan shall judge his people, as one of the tribes of Israel.'); +INSERT INTO t1(docid,words) VALUES(1049017,'Dan shall be a serpent by the way, an adder in the path, that biteth the horse heels, so that his rider shall fall backward.'); +INSERT INTO t1(docid,words) VALUES(1049018,'I have waited for thy salvation, O LORD.'); +INSERT INTO t1(docid,words) VALUES(1049019,'Gad, a troop shall overcome him: but he shall overcome at the last.'); +INSERT INTO t1(docid,words) VALUES(1049020,'Out of Asher his bread shall be fat, and he shall yield royal dainties.'); +INSERT INTO t1(docid,words) VALUES(1049021,'Naphtali is a hind let loose: he giveth goodly words.'); +INSERT INTO t1(docid,words) VALUES(1049022,'Joseph is a fruitful bough, even a fruitful bough by a well; whose branches run over the wall:'); +INSERT INTO t1(docid,words) VALUES(1049023,'The archers have sorely grieved him, and shot at him, and hated him:'); +INSERT INTO t1(docid,words) VALUES(1049024,'But his bow abode in strength, and the arms of his hands were made strong by the hands of the mighty God of Jacob; (from thence is the shepherd, the stone of Israel:)'); +INSERT INTO t1(docid,words) VALUES(1049025,'Even by the God of thy father, who shall help thee; and by the Almighty, who shall bless thee with blessings of heaven above, blessings of the deep that lieth under, blessings of the breasts, and of the womb:'); +INSERT INTO t1(docid,words) VALUES(1049026,'The blessings of thy father have prevailed above the blessings of my progenitors unto the utmost bound of the everlasting hills: they shall be on the head of Joseph, and on the crown of the head of him that was separate from his brethren.'); +INSERT INTO t1(docid,words) VALUES(1049027,'Benjamin shall ravin as a wolf: in the morning he shall devour the prey, and at night he shall divide the spoil.'); +INSERT INTO t1(docid,words) VALUES(1049028,'All these are the twelve tribes of Israel: and this is it that their father spake unto them, and blessed them; every one according to his blessing he blessed them.'); +INSERT INTO t1(docid,words) VALUES(1049029,'And he charged them, and said unto them, I am to be gathered unto my people: bury me with my fathers in the cave that is in the field of Ephron the Hittite,'); +INSERT INTO t1(docid,words) VALUES(1049030,'In the cave that is in the field of Machpelah, which is before Mamre, in the land of Canaan, which Abraham bought with the field of Ephron the Hittite for a possession of a buryingplace.'); +INSERT INTO t1(docid,words) VALUES(1049031,'There they buried Abraham and Sarah his wife; there they buried Isaac and Rebekah his wife; and there I buried Leah.'); +INSERT INTO t1(docid,words) VALUES(1049032,'The purchase of the field and of the cave that is therein was from the children of Heth.'); +INSERT INTO t1(docid,words) VALUES(1049033,'And when Jacob had made an end of commanding his sons, he gathered up his feet into the bed, and yielded up the ghost, and was gathered unto his people.'); +INSERT INTO t1(docid,words) VALUES(1050001,'And Joseph fell upon his father''s face, and wept upon him, and kissed him.'); +INSERT INTO t1(docid,words) VALUES(1050002,'And Joseph commanded his servants the physicians to embalm his father: and the physicians embalmed Israel.'); +INSERT INTO t1(docid,words) VALUES(1050003,'And forty days were fulfilled for him; for so are fulfilled the days of those which are embalmed: and the Egyptians mourned for him threescore and ten days.'); +INSERT INTO t1(docid,words) VALUES(1050004,'And when the days of his mourning were past, Joseph spake unto the house of Pharaoh, saying, If now I have found grace in your eyes, speak, I pray you, in the ears of Pharaoh, saying,'); +INSERT INTO t1(docid,words) VALUES(1050005,'My father made me swear, saying, Lo, I die: in my grave which I have digged for me in the land of Canaan, there shalt thou bury me. Now therefore let me go up, I pray thee, and bury my father, and I will come again.'); +INSERT INTO t1(docid,words) VALUES(1050006,'And Pharaoh said, Go up, and bury thy father, according as he made thee swear.'); +INSERT INTO t1(docid,words) VALUES(1050007,'And Joseph went up to bury his father: and with him went up all the servants of Pharaoh, the elders of his house, and all the elders of the land of Egypt,'); +INSERT INTO t1(docid,words) VALUES(1050008,'And all the house of Joseph, and his brethren, and his father''s house: only their little ones, and their flocks, and their herds, they left in the land of Goshen.'); +INSERT INTO t1(docid,words) VALUES(1050009,'And there went up with him both chariots and horsemen: and it was a very great company.'); +INSERT INTO t1(docid,words) VALUES(1050010,'And they came to the threshingfloor of Atad, which is beyond Jordan, and there they mourned with a great and very sore lamentation: and he made a mourning for his father seven days.'); +INSERT INTO t1(docid,words) VALUES(1050011,'And when the inhabitants of the land, the Canaanites, saw the mourning in the floor of Atad, they said, This is a grievous mourning to the Egyptians: wherefore the name of it was called Abelmizraim, which is beyond Jordan.'); +INSERT INTO t1(docid,words) VALUES(1050012,'And his sons did unto him according as he commanded them:'); +INSERT INTO t1(docid,words) VALUES(1050013,'For his sons carried him into the land of Canaan, and buried him in the cave of the field of Machpelah, which Abraham bought with the field for a possession of a buryingplace of Ephron the Hittite, before Mamre.'); +INSERT INTO t1(docid,words) VALUES(1050014,'And Joseph returned into Egypt, he, and his brethren, and all that went up with him to bury his father, after he had buried his father.'); +INSERT INTO t1(docid,words) VALUES(1050015,'And when Joseph''s brethren saw that their father was dead, they said, Joseph will peradventure hate us, and will certainly requite us all the evil which we did unto him.'); +INSERT INTO t1(docid,words) VALUES(1050016,'And they sent a messenger unto Joseph, saying, Thy father did command before he died, saying,'); +INSERT INTO t1(docid,words) VALUES(1050017,'So shall ye say unto Joseph, Forgive, I pray thee now, the trespass of thy brethren, and their sin; for they did unto thee evil: and now, we pray thee, forgive the trespass of the servants of the God of thy father. And Joseph wept when they spake unto him.'); +INSERT INTO t1(docid,words) VALUES(1050018,'And his brethren also went and fell down before his face; and they said, Behold, we be thy servants.'); +INSERT INTO t1(docid,words) VALUES(1050019,'And Joseph said unto them, Fear not: for am I in the place of God?'); +INSERT INTO t1(docid,words) VALUES(1050020,'But as for you, ye thought evil against me; but God meant it unto good, to bring to pass, as it is this day, to save much people alive.'); +INSERT INTO t1(docid,words) VALUES(1050021,'Now therefore fear ye not: I will nourish you, and your little ones. And he comforted them, and spake kindly unto them.'); +INSERT INTO t1(docid,words) VALUES(1050022,'And Joseph dwelt in Egypt, he, and his father''s house: and Joseph lived an hundred and ten years.'); +INSERT INTO t1(docid,words) VALUES(1050023,'And Joseph saw Ephraim''s children of the third generation: the children also of Machir the son of Manasseh were brought up upon Joseph''s knees.'); +INSERT INTO t1(docid,words) VALUES(1050024,'And Joseph said unto his brethren, I die: and God will surely visit you, and bring you out of this land unto the land which he sware to Abraham, to Isaac, and to Jacob.'); +INSERT INTO t1(docid,words) VALUES(1050025,'And Joseph took an oath of the children of Israel, saying, God will surely visit you, and ye shall carry up my bones from hence.'); +INSERT INTO t1(docid,words) VALUES(1050026,'So Joseph died, being an hundred and ten years old: and they embalmed him, and he was put in a coffin in Egypt.'); +COMMIT; +} +} {} + + +do_test fts4aa-1.1 { + db eval { + SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize + } +} {} +do_test fts4aa-1.2 { + db eval { + SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1 + } +} {} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +do_test fts4aa-2.1 { + db eval { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'melchizedek'; + } +} {1014018 {1 1 1 1 1 1533 25 20}} +do_test fts4aa-2.2 { + db eval { + SELECT docid, mit(matchinfo(t1)) FROM t1 + WHERE t1 MATCH 'spake hebrew' + ORDER BY docid; + } +} {1039014 {2 1 1 40 40 1 6 6 1533 25 42} 1039017 {2 1 1 40 40 1 6 6 1533 25 26}} +do_test fts4aa-2.3 { + db eval { + SELECT docid, mit(matchinfo(t1)) FROM t1 + WHERE t1 MATCH 'laban overtook jacob' + ORDER BY docid; + } +} {1031025 {3 1 2 54 46 1 3 3 2 181 160 1533 25 24}} + +do_test fts4aa-9.1 { + db eval { + DELETE FROM t1 WHERE docid!=1050026; + SELECT hex(size) FROM t1_docsize; + SELECT hex(value) FROM t1_stat; + } +} {17 0117} + +do_test fts4aa-9.2 { + db eval { + SELECT docid FROM t1 EXCEPT SELECT docid FROM t1_docsize + } +} {} +do_test fts4aa-9.3 { + db eval { + SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1 + } +} {} +do_test fts4aa-9.4 { + db eval { + SELECT docid, mit(matchinfo(t1)) FROM t1 + WHERE t1 MATCH 'joseph died in egypt' + ORDER BY docid; + } +} {1050026 {4 1 1 1 1 1 1 1 2 2 1 1 1 1 1 23 23}} + +finish_test diff --git a/test/func.test b/test/func.test index 91ae0a59..eef05439 100644 --- a/test/func.test +++ b/test/func.test @@ -235,6 +235,85 @@ ifcapable floatingpoint { do_test func-4.16 { catchsql {SELECT round(b,2.0) FROM t1 ORDER BY b} } {0 {-2.0 1.23 2.0}} + # Verify some values reported on the mailing list. + # Some of these fail on MSVC builds with 64-bit + # long doubles, but not on GCC builds with 80-bit + # long doubles. + for {set i 1} {$i<999} {incr i} { + set x1 [expr 40222.5 + $i] + set x2 [expr 40223.0 + $i] + do_test func-4.17.$i { + execsql {SELECT round($x1);} + } $x2 + } + for {set i 1} {$i<999} {incr i} { + set x1 [expr 40222.05 + $i] + set x2 [expr 40222.10 + $i] + do_test func-4.18.$i { + execsql {SELECT round($x1,1);} + } $x2 + } + do_test func-4.20 { + execsql {SELECT round(40223.4999999999);} + } {40223.0} + do_test func-4.21 { + execsql {SELECT round(40224.4999999999);} + } {40224.0} + do_test func-4.22 { + execsql {SELECT round(40225.4999999999);} + } {40225.0} + for {set i 1} {$i<10} {incr i} { + do_test func-4.23.$i { + execsql {SELECT round(40223.4999999999,$i);} + } {40223.5} + do_test func-4.24.$i { + execsql {SELECT round(40224.4999999999,$i);} + } {40224.5} + do_test func-4.25.$i { + execsql {SELECT round(40225.4999999999,$i);} + } {40225.5} + } + for {set i 10} {$i<32} {incr i} { + do_test func-4.26.$i { + execsql {SELECT round(40223.4999999999,$i);} + } {40223.4999999999} + do_test func-4.27.$i { + execsql {SELECT round(40224.4999999999,$i);} + } {40224.4999999999} + do_test func-4.28.$i { + execsql {SELECT round(40225.4999999999,$i);} + } {40225.4999999999} + } + do_test func-4.29 { + execsql {SELECT round(1234567890.5);} + } {1234567891.0} + do_test func-4.30 { + execsql {SELECT round(12345678901.5);} + } {12345678902.0} + do_test func-4.31 { + execsql {SELECT round(123456789012.5);} + } {123456789013.0} + do_test func-4.32 { + execsql {SELECT round(1234567890123.5);} + } {1234567890124.0} + do_test func-4.33 { + execsql {SELECT round(12345678901234.5);} + } {12345678901235.0} + do_test func-4.34 { + execsql {SELECT round(1234567890123.35,1);} + } {1234567890123.4} + do_test func-4.35 { + execsql {SELECT round(1234567890123.445,2);} + } {1234567890123.45} + do_test func-4.36 { + execsql {SELECT round(99999999999994.5);} + } {99999999999995.0} + do_test func-4.37 { + execsql {SELECT round(9999999999999.55,1);} + } {9999999999999.6} + do_test func-4.38 { + execsql {SELECT round(9999999999999.555,2);} + } {9999999999999.56} } # Test the upper() and lower() functions diff --git a/test/fuzz2.test b/test/fuzz2.test index eb5eb83a..989b00f0 100644 --- a/test/fuzz2.test +++ b/test/fuzz2.test @@ -65,6 +65,9 @@ do_test fuzz2-2.11 { do_test fuzz2-2.12 { fuzzcatch {SELECT - #196} } {1} + +ifcapable {trigger} { # Only do the following tests if triggers are enabled + do_test fuzz2-3.0 { fuzzcatch {CREATE TRIGGER "AAAAAA" . "AAAAAA" AFTER UPDATE OF "AAAAAA" , "AAAAAA" ON "AAAAAA" . "AAAAAA" FOR EACH ROW BEGIN UPDATE AAAAAA SET "AAAAAA" = #162; END} } {1} @@ -74,6 +77,9 @@ do_test fuzz2-3.1 { do_test fuzz2-3.2 { fuzzcatch {CREATE TEMP TRIGGER IF NOT EXISTS AAAAAA . "AAAAAA" BEFORE UPDATE OF "AAAAAA" ON AAAAAA . "AAAAAA" BEGIN SELECT ALL * , #175 "AAAAAA" FROM "AAAAAA" . AAAAAA; END} } {1} + +} ;# End of ifcapable {trigger} + do_test fuzz2-4.0 { fuzzcatch {ATTACH DATABASE #168 AS whatever} } {1} diff --git a/test/insert4.test b/test/insert4.test index 7e876122..0b069e99 100644 --- a/test/insert4.test +++ b/test/insert4.test @@ -301,6 +301,29 @@ do_test insert4-6.4 { } {0} - +do_test insert4-6.5 { + execsql { + CREATE TABLE t6a(x CHECK( x<>'abc' )); + INSERT INTO t6a VALUES('ABC'); + SELECT * FROM t6a; + } +} {ABC} +do_test insert4-6.6 { + execsql { + CREATE TABLE t6b(x CHECK( x<>'abc' COLLATE nocase )); + } + catchsql { + INSERT INTO t6b SELECT * FROM t6a; + } +} {1 {constraint failed}} +do_test insert4-6.7 { + execsql { + DROP TABLE t6b; + CREATE TABLE t6b(x CHECK( x COLLATE nocase <>'abc' )); + } + catchsql { + INSERT INTO t6b SELECT * FROM t6a; + } +} {1 {constraint failed}} finish_test diff --git a/test/join6.test b/test/join6.test new file mode 100644 index 00000000..4f65dcb4 --- /dev/null +++ b/test/join6.test @@ -0,0 +1,158 @@ +# 2009 December 9 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for N-way joins (N>2) which make +# use of USING or NATURAL JOIN. For such joins, the USING and +# NATURAL JOIN processing needs to search all tables to the left +# of the join looking for a match. See ticket [f74beaabde] +# for additional information. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + + +# The problem as initially reported on the mailing list: +# +do_test join6-1.1 { + execsql { + CREATE TABLE t1(a); + CREATE TABLE t2(a); + CREATE TABLE t3(a,b); + INSERT INTO t1 VALUES(1); + INSERT INTO t3 VALUES(1,2); + + SELECT * FROM t1 LEFT JOIN t2 USING(a) LEFT JOIN t3 USING(a); + } +} {1 2} +do_test join6-1.2 { + execsql { + SELECT t1.a, t3.b + FROM t1 LEFT JOIN t2 ON t1.a=t2.a LEFT JOIN t3 ON t2.a=t3.a; + } +} {1 {}} +do_test join6-1.3 { + execsql { + SELECT t1.a, t3.b + FROM t1 LEFT JOIN t2 ON t1.a=t2.a LEFT JOIN t3 ON t1.a=t3.a; + } +} {1 2} + + +do_test join6-2.1 { + execsql { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + + CREATE TABLE t1(x,y); + CREATE TABLE t2(y,z); + CREATE TABLE t3(x,z); + + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(3,4); + + INSERT INTO t2 VALUES(2,3); + INSERT INTO t2 VALUES(4,5); + + INSERT INTO t3 VALUES(1,3); + INSERT INTO t3 VALUES(3,5); + + SELECT * FROM t1 JOIN t2 USING (y) JOIN t3 USING(x); + } +} {1 2 3 3 3 4 5 5} +do_test join6-2.2 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN t3; + } +} {1 2 3 3 4 5} + + +do_test join6-3.1 { + execsql { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + + CREATE TABLE t1(a,x,y); + INSERT INTO t1 VALUES(1,91,92); + INSERT INTO t1 VALUES(2,93,94); + + CREATE TABLE t2(b,y,z); + INSERT INTO t2 VALUES(3,92,93); + INSERT INTO t2 VALUES(4,94,95); + + CREATE TABLE t3(c,x,z); + INSERT INTO t3 VALUES(5,91,93); + INSERT INTO t3 VALUES(6,99,95); + + SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN t3; + } +} {1 91 92 3 93 5} +do_test join6-3.2 { + execsql { + SELECT * FROM t1 JOIN t2 NATURAL JOIN t3; + } +} {1 91 92 3 92 93 5} +do_test join6-3.3 { + execsql { + SELECT * FROM t1 JOIN t2 USING(y) NATURAL JOIN t3; + } +} {1 91 92 3 93 5} +do_test join6-3.4 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 JOIN t3 USING(x,z); + } +} {1 91 92 3 93 5} +do_test join6-3.5 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 JOIN t3 USING(x); + } +} {1 91 92 3 93 5 93} +do_test join6-3.6 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 JOIN t3 USING(z); + } +} {1 91 92 3 93 5 91 2 93 94 4 95 6 99} + +do_test join6-4.1 { + execsql { + SELECT * FROM + (SELECT 1 AS a, 91 AS x, 92 AS y UNION SELECT 2, 93, 94) + NATURAL JOIN t2 NATURAL JOIN t3 + } +} {1 91 92 3 93 5} +do_test join6-4.2 { + execsql { + SELECT * FROM t1 NATURAL JOIN + (SELECT 3 AS b, 92 AS y, 93 AS z UNION SELECT 4, 94, 95) + NATURAL JOIN t3 + } +} {1 91 92 3 93 5} +do_test join6-4.3 { + execsql { + SELECT * FROM t1 NATURAL JOIN t2 NATURAL JOIN + (SELECT 5 AS c, 91 AS x, 93 AS z UNION SELECT 6, 99, 95) + } +} {1 91 92 3 93 5} + + + + + + + + + + + +finish_test diff --git a/test/main.test b/test/main.test index 12262c1c..6248e11b 100644 --- a/test/main.test +++ b/test/main.test @@ -25,21 +25,40 @@ ifcapable {complete} { do_test main-1.1 { db complete {This is a test} } {0} -do_test main-1.2 { +do_test main-1.2.0 { db complete { } -} {1} -do_test main-1.3 { +} {0} +do_test main-1.2.1 { + db complete {} +} {0} +do_test main-1.3.0 { db complete { -- a comment ; } -} {1} -do_test main-1.4 { +} {0} +do_test main-1.3.1 { + db complete { + /* a comment ; */ + } +} {0} +do_test main-1.4.0 { db complete { -- a comment ; ; } } {1} +do_test main-1.4.1 { + db complete { + /* a comment ; */ + ; + } +} {1} +do_test main-1.4.2 { + db complete { + /* a comment ; */ ; + } +} {1} do_test main-1.5 { db complete {DROP TABLE 'xyz;} } {0} diff --git a/test/minmax3.test b/test/minmax3.test index 8e8164ac..c387b041 100644 --- a/test/minmax3.test +++ b/test/minmax3.test @@ -174,7 +174,7 @@ do_test minmax3-2.8 { execsql { SELECT min(b) FROM t2 WHERE a = 3 AND b<1; } } {{}} -do_test minmax3-2.1 { +do_test minmax3-3.1 { execsql { DROP TABLE t2; CREATE TABLE t2(a, b); @@ -192,26 +192,91 @@ do_test minmax3-2.1 { INSERT INTO t2 VALUES(3, 3); } } {} -do_test minmax3-2.2 { +do_test minmax3-3.2 { execsql { SELECT min(b) FROM t2 WHERE a = 1; } } {1} -do_test minmax3-2.3 { +do_test minmax3-3.3 { execsql { SELECT min(b) FROM t2 WHERE a = 1 AND b>1; } } {2} -do_test minmax3-2.4 { +do_test minmax3-3.4 { execsql { SELECT min(b) FROM t2 WHERE a = 1 AND b>-1; } } {1} -do_test minmax3-2.5 { +do_test minmax3-3.5 { execsql { SELECT min(b) FROM t2 WHERE a = 1; } } {1} -do_test minmax3-2.6 { +do_test minmax3-3.6 { execsql { SELECT min(b) FROM t2 WHERE a = 1 AND b<2; } } {1} -do_test minmax3-2.7 { +do_test minmax3-3.7 { execsql { SELECT min(b) FROM t2 WHERE a = 1 AND b<1; } } {{}} -do_test minmax3-2.8 { +do_test minmax3-3.8 { execsql { SELECT min(b) FROM t2 WHERE a = 3 AND b<1; } } {{}} +do_test minmax3-4.1 { + execsql { + CREATE TABLE t4(x); + INSERT INTO t4 VALUES('abc'); + INSERT INTO t4 VALUES('BCD'); + SELECT max(x) FROM t4; + } +} {abc} +do_test minmax3-4.2 { + execsql { + SELECT max(x COLLATE nocase) FROM t4; + } +} {BCD} +do_test minmax3-4.3 { + execsql { + SELECT max(x), max(x COLLATE nocase) FROM t4; + } +} {abc BCD} +do_test minmax3-4.4 { + execsql { + SELECT max(x COLLATE binary), max(x COLLATE nocase) FROM t4; + } +} {abc BCD} +do_test minmax3-4.5 { + execsql { + SELECT max(x COLLATE nocase), max(x COLLATE rtrim) FROM t4; + } +} {BCD abc} +do_test minmax3-4.6 { + execsql { + SELECT max(x COLLATE nocase), max(x) FROM t4; + } +} {BCD abc} +do_test minmax3-4.10 { + execsql { + SELECT min(x) FROM t4; + } +} {BCD} +do_test minmax3-4.11 { + execsql { + SELECT min(x COLLATE nocase) FROM t4; + } +} {abc} +do_test minmax3-4.12 { + execsql { + SELECT min(x), min(x COLLATE nocase) FROM t4; + } +} {BCD abc} +do_test minmax3-4.13 { + execsql { + SELECT min(x COLLATE binary), min(x COLLATE nocase) FROM t4; + } +} {BCD abc} +do_test minmax3-4.14 { + execsql { + SELECT min(x COLLATE nocase), min(x COLLATE rtrim) FROM t4; + } +} {abc BCD} +do_test minmax3-4.15 { + execsql { + SELECT min(x COLLATE nocase), min(x) FROM t4; + } +} {abc BCD} + + finish_test diff --git a/test/pagesize.test b/test/pagesize.test index b676df94..af2c017b 100644 --- a/test/pagesize.test +++ b/test/pagesize.test @@ -188,6 +188,32 @@ foreach PGSZ {512 2048 4096 8192} { ifcapable {vacuum} {execsql VACUUM} } {} integrity_check pagesize-2.$PGSZ.17 + + db close + file delete -force test.db + sqlite3 db test.db + do_test pagesize-2.$PGSZ.30 { + execsql " + CREATE TABLE t1(x); + PRAGMA temp.page_size=$PGSZ; + CREATE TEMP TABLE t2(y); + PRAGMA main.page_size; + PRAGMA temp.page_size; + " + } [list 1024 $PGSZ] + + db close + file delete -force test.db + sqlite3 db test.db + do_test pagesize-2.$PGSZ.40 { + execsql " + PRAGMA page_size=$PGSZ; + CREATE TABLE t1(x); + CREATE TEMP TABLE t2(y); + PRAGMA main.page_size; + PRAGMA temp.page_size; + " + } [list $PGSZ $PGSZ] } finish_test diff --git a/test/permutations.test b/test/permutations.test index 2dec2996..c4a92b78 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -472,6 +472,9 @@ run_tests "inmemory_journal" -description { # the number of fsync() calls. pager.test exclusive.test jrnlmode.test sync.test misc1.test journal1.test conflict.test crash8.test tkt3457.test io.test + + # Exclude stmt.test, which expects sub-journals to use temporary files. + stmt.test } ifcapable mem3 { diff --git a/test/safety.test b/test/safety.test deleted file mode 100644 index 9cca57c1..00000000 --- a/test/safety.test +++ /dev/null @@ -1,93 +0,0 @@ -# 2005 January 11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the sqlite3SafetyOn and sqlite3SafetyOff -# functions. Those routines are not strictly necessary - they are -# designed to detect misuse of the library. -# -# $Id: safety.test,v 1.4 2008/03/18 13:46:53 drh Exp $ - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -ifcapable !debug { - puts "Skipping safety tests since SQLITE_DEBUG is off" - finish_test - return -} - -# Return the UTF-8 representation of the supplied UTF-16 string $str. -proc utf8 {str} { - # If $str ends in two 0x00 0x00 bytes, knock these off before - # converting to UTF-8 using TCL. - binary scan $str \c* vals - if {[lindex $vals end]==0 && [lindex $vals end-1]==0} { - set str [binary format \c* [lrange $vals 0 end-2]] - } - - set r [encoding convertfrom unicode $str] - return $r -} - - -do_test safety-1.1 { - set DB [sqlite3_connection_pointer db] - db eval {CREATE TABLE t1(a)} - sqlite_set_magic $DB SQLITE_MAGIC_BUSY - catchsql { - SELECT name FROM sqlite_master; - } -} {1 {library routine called out of sequence}} -do_test safety-1.2 { - sqlite_set_magic $DB SQLITE_MAGIC_OPEN - catchsql { - SELECT name FROM sqlite_master - } -} {0 t1} - -do_test safety-2.1 { - proc safety_on {} "sqlite_set_magic $DB SQLITE_MAGIC_BUSY" - db function safety_on safety_on - catchsql { - SELECT safety_on(), name FROM sqlite_master - } -} {1 {library routine called out of sequence}} -ifcapable {utf16} { - do_test safety-2.1.1 { - utf8 [sqlite3_errmsg16 db] - } {library routine called out of sequence} -} -do_test safety-2.2 { - catchsql { - SELECT 'hello' - } -} {1 {library routine called out of sequence}} -do_test safety-2.3 { - sqlite3_close $DB -} {SQLITE_MISUSE} -do_test safety-2.4 { - sqlite_set_magic $DB SQLITE_MAGIC_OPEN - execsql { - SELECT name FROM sqlite_master - } -} {t1} - -do_test safety-3.1 { - set rc [catch { - db eval {SELECT name FROM sqlite_master} { - sqlite_set_magic $DB SQLITE_MAGIC_BUSY - } - } msg] - lappend rc $msg -} {1 {library routine called out of sequence}} -sqlite_set_magic $DB SQLITE_MAGIC_OPEN - -finish_test diff --git a/test/schema.test b/test/schema.test index eeb21028..afca39ed 100644 --- a/test/schema.test +++ b/test/schema.test @@ -364,6 +364,8 @@ do_test schema-12.1 { list [sqlite3_step $::STMT] [sqlite3_finalize $::STMT] } {SQLITE_ERROR SQLITE_SCHEMA} +ifcapable {auth} { + do_test schema-13.1 { set S [sqlite3_prepare_v2 db "SELECT * FROM sqlite_master" -1 dummy] db function hello hello @@ -374,14 +376,16 @@ do_test schema-13.1 { return SQLITE_OK } sqlite3_step $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} do_test schema-13.2 { sqlite3_step $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} do_test schema-13.3 { sqlite3_finalize $S -} {SQLITE_SCHEMA} +} {SQLITE_AUTH} + +} finish_test diff --git a/test/securedel.test b/test/securedel.test new file mode 100644 index 00000000..c965069e --- /dev/null +++ b/test/securedel.test @@ -0,0 +1,73 @@ +# 2010 January 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Tests for the secure_delete pragma. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +unset -nocomplain DEFAULT_SECDEL +set DEFAULT_SECDEL 0 +ifcapable secure_delete { + set DEFAULT_SECDEL 1 +} + + +do_test securedel-1.0 { + db eval {PRAGMA secure_delete;} +} $DEFAULT_SECDEL + +file delete -force test2.db test2.db-journal +do_test securedel-1.1 { + db eval { + ATTACH 'test2.db' AS db2; + PRAGMA main.secure_delete=ON; + PRAGMA db2.secure_delete; + } +} [list 1 $DEFAULT_SECDEL] +do_test securedel-1.2 { + db eval { + PRAGMA main.secure_delete=OFF; + PRAGMA db2.secure_delete; + } +} [list 0 $DEFAULT_SECDEL] +do_test securedel-1.3 { + db eval { + PRAGMA secure_delete=OFF; + PRAGMA db2.secure_delete; + } +} {0 0} +do_test securedel-1.4 { +breakpoint + db eval { + PRAGMA secure_delete=ON; + PRAGMA db2.secure_delete; + } +} {1 1} + +do_test securedel-2.1 { + db eval { + DETACH db2; + ATTACH 'test2.db' AS db2; + PRAGMA db2.secure_delete; + } +} 1 +do_test securedel-2.2 { + db eval { + DETACH db2; + PRAGMA main.secure_delete=OFF; + ATTACH 'test2.db' AS db2; + PRAGMA db2.secure_delete; + } +} {0 0} + +finish_test diff --git a/test/select7.test b/test/select7.test index 3837c882..e8fc4400 100644 --- a/test/select7.test +++ b/test/select7.test @@ -156,4 +156,47 @@ ifcapable compound { } } +# This block of tests verifies that bug aa92c76cd4 is fixed. +# +do_test select7-7.1 { + execsql { + CREATE TABLE t3(a REAL); + INSERT INTO t3 VALUES(44.0); + INSERT INTO t3 VALUES(56.0); + } +} {} +do_test select7-7.2 { + execsql { + pragma vdbe_trace = 0; + SELECT (CASE WHEN a=0 THEN 0 ELSE (a + 25) / 50 END) AS categ, count(*) + FROM t3 GROUP BY categ + } +} {1.38 1 1.62 1} +do_test select7-7.3 { + execsql { + CREATE TABLE t4(a REAL); + INSERT INTO t4 VALUES( 2.0 ); + INSERT INTO t4 VALUES( 3.0 ); + } +} {} +do_test select7-7.4 { + execsql { + SELECT (CASE WHEN a=0 THEN 'zero' ELSE a/2 END) AS t FROM t4 GROUP BY t; + } +} {1.0 1.5} +do_test select7-7.5 { + execsql { SELECT a=0, typeof(a) FROM t4 } +} {0 real 0 real} +do_test select7-7.6 { + execsql { SELECT a=0, typeof(a) FROM t4 GROUP BY a } +} {0 real 0 real} + +do_test select7-7.7 { + execsql { + CREATE TABLE t5(a TEXT, b INT); + INSERT INTO t5 VALUES(123, 456); + SELECT typeof(a), a FROM t5 GROUP BY a HAVING a