src: use our custom icu_tokenizer
This commit is contained in:
Fedor Indutnyy
parent
0807c512f3
commit
ac7338ad39
9
deps/common.gypi
vendored
9
deps/common.gypi
vendored
@ -13,6 +13,15 @@
|
||||
},
|
||||
},
|
||||
'conditions': [
|
||||
['target_arch == "x64"', {
|
||||
'variables': {
|
||||
'rust_arch%': 'x86_64',
|
||||
}
|
||||
}, {
|
||||
'variables': {
|
||||
'rust_arch%': 'aarch64',
|
||||
}
|
||||
}],
|
||||
['OS == "win"', {
|
||||
'defines': ['WIN32'],
|
||||
'variables': {
|
||||
|
||||
2
deps/download.js
vendored
2
deps/download.js
vendored
@ -6,7 +6,7 @@ const { Transform } = require('stream');
|
||||
const { pipeline } = require('stream/promises');
|
||||
|
||||
const BASE_URI = `https://build-artifacts.signal.org/desktop`;
|
||||
const HASH = '4587e88a4219cb76fd595a5b73f82bfd919b235d75626f40783525f4f58a7503';
|
||||
const HASH = '01d2f46ae275286ed552cfc977239e7dba1ca2cd572f1d0c057a29d87b1f9c79';
|
||||
const SQLCIPHER_VERSION = '4.5.2';
|
||||
const OPENSSL_VERSION = '3.0.7';
|
||||
const URL = `${BASE_URI}/sqlcipher-${SQLCIPHER_VERSION}--${OPENSSL_VERSION}-` +
|
||||
|
||||
10
deps/sqlite3.gyp
vendored
10
deps/sqlite3.gyp
vendored
@ -23,6 +23,7 @@
|
||||
'conditions': [
|
||||
['OS == "win"', {
|
||||
'outputs': [
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-pc-windows-msvc/fts5_tokenizer.lib',
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/libssl.lib',
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/libcrypto.lib',
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/ossl_static.pdb',
|
||||
@ -62,6 +63,7 @@
|
||||
'include_dirs': [
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/',
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/openssl-include',
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/include',
|
||||
],
|
||||
},
|
||||
'cflags': ['-std=c99', '-w'],
|
||||
@ -80,10 +82,12 @@
|
||||
'-llibcrypto.lib',
|
||||
'-llibssl.lib',
|
||||
'-lws2_32.lib',
|
||||
'-lcrypt32.lib'
|
||||
'-lcrypt32.lib',
|
||||
'-lfts5_tokenizer.lib',
|
||||
],
|
||||
'library_dirs': [
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)'
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-pc-windows-msvc',
|
||||
]
|
||||
}
|
||||
},
|
||||
@ -91,7 +95,8 @@
|
||||
'link_settings': {
|
||||
'libraries': [
|
||||
# This statically links libcrypto, whereas -lcrypto would dynamically link it
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/OpenSSL-mac-<(target_arch)/libcrypto.a'
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/OpenSSL-mac-<(target_arch)/libcrypto.a',
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-apple-darwin/libfts5_tokenizer.a',
|
||||
]
|
||||
}
|
||||
},
|
||||
@ -100,6 +105,7 @@
|
||||
'libraries': [
|
||||
# This statically links libcrypto, whereas -lcrypto would dynamically link it
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/OpenSSL-linux-<(target_arch)/libcrypto.a'
|
||||
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-unknown-linux-gnu/libfts5_tokenizer.a',
|
||||
]
|
||||
}
|
||||
}],
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
ctx->Exit();
|
||||
return proto->StrictEquals(baseProto) || proto->StrictEquals(v8::Null(isolate));
|
||||
}
|
||||
#line 105 "./src/better_sqlite3.lzz"
|
||||
#line 108 "./src/better_sqlite3.lzz"
|
||||
NODE_MODULE_INIT(/* exports, context */) {
|
||||
v8::Isolate* isolate = context->GetIsolate();
|
||||
v8::HandleScope scope(isolate);
|
||||
@ -467,8 +467,34 @@ Database::Database (v8::Isolate * isolate, Addon * addon, sqlite3 * db_handle, v
|
||||
addon->dbs.insert(this);
|
||||
}
|
||||
#line 150 "./src/objects/database.lzz"
|
||||
void Database::JS_new (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
fts5_api * Database::GetFTS5API ()
|
||||
#line 150 "./src/objects/database.lzz"
|
||||
{
|
||||
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt = nullptr;
|
||||
|
||||
rc = sqlite3_prepare(db_handle, "SELECT fts5(?1)", -1, &pStmt, 0);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db_handle);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
fts5_api *fts5 = nullptr;
|
||||
sqlite3_bind_pointer(pStmt, 1, (void*)&fts5, "fts5_api_ptr", nullptr);
|
||||
sqlite3_step(pStmt);
|
||||
rc = sqlite3_finalize(pStmt);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db_handle);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
assert(fts5 != nullptr);
|
||||
return fts5;
|
||||
}
|
||||
#line 174 "./src/objects/database.lzz"
|
||||
void Database::JS_new (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 174 "./src/objects/database.lzz"
|
||||
{
|
||||
assert(info.IsConstructCall());
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > filename = ( info [ 0 ] . As < v8 :: String > ( ) ) ;
|
||||
@ -518,11 +544,21 @@ void Database::JS_new (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
SetFrozen(isolate, ctx, info.This(), addon->cs.readonly, v8::Boolean::New(isolate, readonly));
|
||||
SetFrozen(isolate, ctx, info.This(), addon->cs.name, filenameGiven);
|
||||
|
||||
|
||||
fts5_api* fts5 = db->GetFTS5API();
|
||||
|
||||
if (fts5 == nullptr) {
|
||||
return;
|
||||
}
|
||||
ICUTokenizerModule* icu = new ICUTokenizerModule();
|
||||
fts5->xCreateTokenizer(fts5, "icu_tokenizer", icu, icu->get_api_object(),
|
||||
&ICUTokenizerModule::xDestroy);
|
||||
|
||||
info.GetReturnValue().Set(info.This());
|
||||
}
|
||||
#line 202 "./src/objects/database.lzz"
|
||||
#line 236 "./src/objects/database.lzz"
|
||||
void Database::JS_prepare (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 202 "./src/objects/database.lzz"
|
||||
#line 236 "./src/objects/database.lzz"
|
||||
{
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > source = ( info [ 0 ] . As < v8 :: String > ( ) ) ;
|
||||
if ( info . Length ( ) <= ( 1 ) || ! info [ 1 ] -> IsObject ( ) ) return ThrowTypeError ( "Expected " "second" " argument to be " "an object" ) ; v8 :: Local < v8 :: Object > database = ( info [ 1 ] . As < v8 :: Object > ( ) ) ;
|
||||
@ -538,9 +574,9 @@ void Database::JS_prepare (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
addon->privileged_info = NULL;
|
||||
if (!maybeStatement.IsEmpty()) info.GetReturnValue().Set(maybeStatement.ToLocalChecked());
|
||||
}
|
||||
#line 218 "./src/objects/database.lzz"
|
||||
#line 252 "./src/objects/database.lzz"
|
||||
void Database::JS_exec (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 218 "./src/objects/database.lzz"
|
||||
#line 252 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > source = ( info [ 0 ] . As < v8 :: String > ( ) ) ;
|
||||
@ -580,9 +616,9 @@ void Database::JS_exec (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
db->ThrowDatabaseError();
|
||||
}
|
||||
}
|
||||
#line 258 "./src/objects/database.lzz"
|
||||
#line 292 "./src/objects/database.lzz"
|
||||
void Database::JS_backup (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 258 "./src/objects/database.lzz"
|
||||
#line 292 "./src/objects/database.lzz"
|
||||
{
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsObject ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "an object" ) ; v8 :: Local < v8 :: Object > database = ( info [ 0 ] . As < v8 :: Object > ( ) ) ;
|
||||
if ( info . Length ( ) <= ( 1 ) || ! info [ 1 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "second" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > attachedName = ( info [ 1 ] . As < v8 :: String > ( ) ) ;
|
||||
@ -600,9 +636,9 @@ void Database::JS_backup (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
addon->privileged_info = NULL;
|
||||
if (!maybeBackup.IsEmpty()) info.GetReturnValue().Set(maybeBackup.ToLocalChecked());
|
||||
}
|
||||
#line 276 "./src/objects/database.lzz"
|
||||
#line 310 "./src/objects/database.lzz"
|
||||
void Database::JS_serialize (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 276 "./src/objects/database.lzz"
|
||||
#line 310 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > attachedName = ( info [ 0 ] . As < v8 :: String > ( ) ) ;
|
||||
@ -624,9 +660,9 @@ void Database::JS_serialize (v8::FunctionCallbackInfo <v8 :: Value> const & info
|
||||
node::Buffer::New(isolate, reinterpret_cast<char*>(data), length, FreeSerialization, NULL).ToLocalChecked()
|
||||
);
|
||||
}
|
||||
#line 298 "./src/objects/database.lzz"
|
||||
#line 332 "./src/objects/database.lzz"
|
||||
void Database::JS_function (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 298 "./src/objects/database.lzz"
|
||||
#line 332 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsFunction ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a function" ) ; v8 :: Local < v8 :: Function > fn = ( info [ 0 ] . As < v8 :: Function > ( ) ) ;
|
||||
@ -650,9 +686,9 @@ void Database::JS_function (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
db->ThrowDatabaseError();
|
||||
}
|
||||
}
|
||||
#line 322 "./src/objects/database.lzz"
|
||||
#line 356 "./src/objects/database.lzz"
|
||||
void Database::JS_aggregate (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 322 "./src/objects/database.lzz"
|
||||
#line 356 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if ( info . Length ( ) <= ( 0 ) ) return ThrowTypeError ( "Expected a " "first" " argument" ) ; v8 :: Local < v8 :: Value > start = info [ 0 ] ;
|
||||
@ -681,9 +717,9 @@ void Database::JS_aggregate (v8::FunctionCallbackInfo <v8 :: Value> const & info
|
||||
db->ThrowDatabaseError();
|
||||
}
|
||||
}
|
||||
#line 351 "./src/objects/database.lzz"
|
||||
#line 385 "./src/objects/database.lzz"
|
||||
void Database::JS_table (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 351 "./src/objects/database.lzz"
|
||||
#line 385 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsFunction ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a function" ) ; v8 :: Local < v8 :: Function > factory = ( info [ 0 ] . As < v8 :: Function > ( ) ) ;
|
||||
@ -703,9 +739,9 @@ void Database::JS_table (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
}
|
||||
db->busy = false;
|
||||
}
|
||||
#line 371 "./src/objects/database.lzz"
|
||||
#line 405 "./src/objects/database.lzz"
|
||||
void Database::JS_loadExtension (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 371 "./src/objects/database.lzz"
|
||||
#line 405 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
v8::Local<v8::String> entryPoint;
|
||||
@ -727,9 +763,9 @@ void Database::JS_loadExtension (v8::FunctionCallbackInfo <v8 :: Value> const &
|
||||
}
|
||||
sqlite3_free(error);
|
||||
}
|
||||
#line 393 "./src/objects/database.lzz"
|
||||
#line 427 "./src/objects/database.lzz"
|
||||
void Database::JS_close (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 393 "./src/objects/database.lzz"
|
||||
#line 427 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if (db->open) {
|
||||
@ -739,77 +775,60 @@ void Database::JS_close (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
db->CloseHandles();
|
||||
}
|
||||
}
|
||||
#line 403 "./src/objects/database.lzz"
|
||||
#line 437 "./src/objects/database.lzz"
|
||||
void Database::JS_defaultSafeIntegers (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 403 "./src/objects/database.lzz"
|
||||
#line 437 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if (info.Length() == 0) db->safe_ints = true;
|
||||
else { if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsBoolean ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a boolean" ) ; db -> safe_ints = ( info [ 0 ] . As < v8 :: Boolean > ( ) ) -> Value ( ) ; }
|
||||
}
|
||||
#line 409 "./src/objects/database.lzz"
|
||||
#line 443 "./src/objects/database.lzz"
|
||||
void Database::JS_unsafeMode (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 409 "./src/objects/database.lzz"
|
||||
#line 443 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if (info.Length() == 0) db->unsafe_mode = true;
|
||||
else { if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsBoolean ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a boolean" ) ; db -> unsafe_mode = ( info [ 0 ] . As < v8 :: Boolean > ( ) ) -> Value ( ) ; }
|
||||
sqlite3_db_config(db->db_handle, SQLITE_DBCONFIG_DEFENSIVE, static_cast<int>(!db->unsafe_mode), NULL);
|
||||
}
|
||||
#line 416 "./src/objects/database.lzz"
|
||||
#line 450 "./src/objects/database.lzz"
|
||||
void Database::JS_createFTS5Tokenizer (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 416 "./src/objects/database.lzz"
|
||||
#line 450 "./src/objects/database.lzz"
|
||||
{
|
||||
Addon * addon = static_cast < Addon * > ( info . Data ( ) . As < v8 :: External > ( ) -> Value ( ) ) ;
|
||||
v8 :: Isolate * isolate = info . GetIsolate ( ) ;
|
||||
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsString ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a string" ) ; v8 :: Local < v8 :: String > name = ( info [ 0 ] . As < v8 :: String > ( ) ) ;
|
||||
if ( info . Length ( ) <= ( 1 ) || ! info [ 1 ] -> IsFunction ( ) ) return ThrowTypeError ( "Expected " "second" " argument to be " "a function" ) ; v8 :: Local < v8 :: Function > create_instance_fn = ( info [ 1 ] . As < v8 :: Function > ( ) ) ;
|
||||
|
||||
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt = nullptr;
|
||||
|
||||
rc = sqlite3_prepare(db->db_handle, "SELECT fts5(?1)", -1, &pStmt, 0);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db->db_handle);
|
||||
return;
|
||||
}
|
||||
|
||||
fts5_api *fts5 = nullptr;
|
||||
sqlite3_bind_pointer(pStmt, 1, (void*)&fts5, "fts5_api_ptr", nullptr);
|
||||
sqlite3_step(pStmt);
|
||||
rc = sqlite3_finalize(pStmt);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db->db_handle);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(fts5 != nullptr);
|
||||
|
||||
TokenizerModule* t = new TokenizerModule(isolate, create_instance_fn);
|
||||
|
||||
v8::String::Utf8Value utf8(isolate, name);
|
||||
fts5_api* fts5 = db->GetFTS5API();
|
||||
|
||||
if (fts5 == nullptr) {
|
||||
return;
|
||||
}
|
||||
fts5->xCreateTokenizer(fts5, *utf8, t, t->get_api_object(),
|
||||
&TokenizerModule::xDestroy);
|
||||
}
|
||||
#line 452 "./src/objects/database.lzz"
|
||||
#line 469 "./src/objects/database.lzz"
|
||||
void Database::JS_open (v8::Local <v8 :: String> _, v8::PropertyCallbackInfo <v8 :: Value> const & info)
|
||||
#line 452 "./src/objects/database.lzz"
|
||||
#line 469 "./src/objects/database.lzz"
|
||||
{
|
||||
info.GetReturnValue().Set( ObjectWrapForElectron22 :: Unwrap <Database>(info.This())->open);
|
||||
}
|
||||
#line 456 "./src/objects/database.lzz"
|
||||
#line 473 "./src/objects/database.lzz"
|
||||
void Database::JS_inTransaction (v8::Local <v8 :: String> _, v8::PropertyCallbackInfo <v8 :: Value> const & info)
|
||||
#line 456 "./src/objects/database.lzz"
|
||||
#line 473 "./src/objects/database.lzz"
|
||||
{
|
||||
Database* db = ObjectWrapForElectron22 :: Unwrap <Database>(info.This());
|
||||
info.GetReturnValue().Set(db->open && !static_cast<bool>(sqlite3_get_autocommit(db->db_handle)));
|
||||
}
|
||||
#line 461 "./src/objects/database.lzz"
|
||||
#line 478 "./src/objects/database.lzz"
|
||||
bool Database::Deserialize (v8::Local <v8::Object> buffer, Addon * addon, sqlite3 * db_handle, bool readonly)
|
||||
#line 461 "./src/objects/database.lzz"
|
||||
#line 478 "./src/objects/database.lzz"
|
||||
{
|
||||
size_t length = node::Buffer::Length(buffer);
|
||||
unsigned char* data = (unsigned char*)sqlite3_malloc64(length);
|
||||
@ -834,15 +853,15 @@ bool Database::Deserialize (v8::Local <v8::Object> buffer, Addon * addon, sqlite
|
||||
|
||||
return true;
|
||||
}
|
||||
#line 486 "./src/objects/database.lzz"
|
||||
#line 503 "./src/objects/database.lzz"
|
||||
void Database::FreeSerialization (char * data, void * _)
|
||||
#line 486 "./src/objects/database.lzz"
|
||||
#line 503 "./src/objects/database.lzz"
|
||||
{
|
||||
sqlite3_free(data);
|
||||
}
|
||||
#line 490 "./src/objects/database.lzz"
|
||||
#line 507 "./src/objects/database.lzz"
|
||||
int const Database::MAX_BUFFER_SIZE;
|
||||
#line 491 "./src/objects/database.lzz"
|
||||
#line 508 "./src/objects/database.lzz"
|
||||
int const Database::MAX_STRING_SIZE;
|
||||
#line 4 "./src/objects/statement.lzz"
|
||||
v8::Local <v8 :: Function> Statement::Init (v8::Isolate * isolate, v8::Local <v8 :: External> data)
|
||||
@ -1555,6 +1574,32 @@ fts5_tokenizer TokenizerModule::api_object = {
|
||||
&xDelete,
|
||||
&xTokenize,
|
||||
};
|
||||
#line 3 "./src/objects/icu-tokenizer.lzz"
|
||||
ICUTokenizerModule::ICUTokenizerModule ()
|
||||
#line 3 "./src/objects/icu-tokenizer.lzz"
|
||||
{}
|
||||
#line 5 "./src/objects/icu-tokenizer.lzz"
|
||||
void ICUTokenizerModule::xDestroy (void * pCtx)
|
||||
#line 5 "./src/objects/icu-tokenizer.lzz"
|
||||
{}
|
||||
#line 14 "./src/objects/icu-tokenizer.lzz"
|
||||
int ICUTokenizerModule::xCreate (void * pCtx, char const * * azArg, int nArg, Fts5Tokenizer * * ppOut)
|
||||
#line 15 "./src/objects/icu-tokenizer.lzz"
|
||||
{
|
||||
TokenizerModule* m = static_cast<TokenizerModule*>(pCtx);
|
||||
*ppOut = reinterpret_cast<Fts5Tokenizer*>(m);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
#line 21 "./src/objects/icu-tokenizer.lzz"
|
||||
void ICUTokenizerModule::xDelete (Fts5Tokenizer * tokenizer)
|
||||
#line 21 "./src/objects/icu-tokenizer.lzz"
|
||||
{}
|
||||
#line 25 "./src/objects/icu-tokenizer.lzz"
|
||||
fts5_tokenizer ICUTokenizerModule::api_object = {
|
||||
&xCreate,
|
||||
&xDelete,
|
||||
fts5_icu_tokenize,
|
||||
};
|
||||
#line 4 "./src/util/data-converter.lzz"
|
||||
void DataConverter::ThrowDataConversionError (sqlite3_context * invocation, bool isBigInt)
|
||||
#line 4 "./src/util/data-converter.lzz"
|
||||
@ -2331,32 +2376,32 @@ Binder::Result Binder::BindArgs (v8::FunctionCallbackInfo <v8 :: Value> const &
|
||||
|
||||
return { count, bound_object };
|
||||
}
|
||||
#line 38 "./src/better_sqlite3.lzz"
|
||||
#line 41 "./src/better_sqlite3.lzz"
|
||||
void Addon::JS_setErrorConstructor (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 38 "./src/better_sqlite3.lzz"
|
||||
#line 41 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsFunction ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a function" ) ; v8 :: Local < v8 :: Function > SqliteError = ( info [ 0 ] . As < v8 :: Function > ( ) ) ;
|
||||
static_cast < Addon * > ( info . Data ( ) . As < v8 :: External > ( ) -> Value ( ) ) ->SqliteError.Reset( info . GetIsolate ( ) , SqliteError);
|
||||
}
|
||||
#line 43 "./src/better_sqlite3.lzz"
|
||||
#line 46 "./src/better_sqlite3.lzz"
|
||||
void Addon::JS_setLogHandler (v8::FunctionCallbackInfo <v8 :: Value> const & info)
|
||||
#line 43 "./src/better_sqlite3.lzz"
|
||||
#line 46 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
if ( info . Length ( ) <= ( 0 ) || ! info [ 0 ] -> IsFunction ( ) ) return ThrowTypeError ( "Expected " "first" " argument to be " "a function" ) ; v8 :: Local < v8 :: Function > LogHandler = ( info [ 0 ] . As < v8 :: Function > ( ) ) ;
|
||||
static_cast < Addon * > ( info . Data ( ) . As < v8 :: External > ( ) -> Value ( ) ) ->LogHandler.Reset( info . GetIsolate ( ) , LogHandler);
|
||||
}
|
||||
#line 48 "./src/better_sqlite3.lzz"
|
||||
#line 51 "./src/better_sqlite3.lzz"
|
||||
void Addon::Cleanup (void * ptr)
|
||||
#line 48 "./src/better_sqlite3.lzz"
|
||||
#line 51 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
Addon* addon = static_cast<Addon*>(ptr);
|
||||
for (Database* db : addon->dbs) db->CloseHandles();
|
||||
addon->dbs.clear();
|
||||
delete addon;
|
||||
}
|
||||
#line 55 "./src/better_sqlite3.lzz"
|
||||
#line 58 "./src/better_sqlite3.lzz"
|
||||
void Addon::SqliteLog (void * pArg, int iErrCode, char const * zMsg)
|
||||
#line 55 "./src/better_sqlite3.lzz"
|
||||
#line 58 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
Addon* addon = static_cast<Addon*>(uv_key_get(&thread_key));
|
||||
if (addon->LogHandler.IsEmpty()) {
|
||||
@ -2371,9 +2416,9 @@ void Addon::SqliteLog (void * pArg, int iErrCode, char const * zMsg)
|
||||
};
|
||||
handler->Call(isolate->GetCurrentContext(), v8::Undefined(isolate), 2, arg).ToLocalChecked();
|
||||
}
|
||||
#line 70 "./src/better_sqlite3.lzz"
|
||||
#line 73 "./src/better_sqlite3.lzz"
|
||||
void Addon::InitLoggerOnce ()
|
||||
#line 70 "./src/better_sqlite3.lzz"
|
||||
#line 73 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
int err = uv_key_create(&thread_key);
|
||||
if (err != 0) {
|
||||
@ -2382,16 +2427,16 @@ void Addon::InitLoggerOnce ()
|
||||
sqlite3_initialize();
|
||||
sqlite3_config(SQLITE_CONFIG_LOG, Addon::SqliteLog, nullptr);
|
||||
}
|
||||
#line 79 "./src/better_sqlite3.lzz"
|
||||
Addon::Addon (v8::Isolate * isolate)
|
||||
#line 79 "./src/better_sqlite3.lzz"
|
||||
: privileged_info (NULL), next_id (0), cs (isolate)
|
||||
#line 82 "./src/better_sqlite3.lzz"
|
||||
Addon::Addon (v8::Isolate * isolate)
|
||||
#line 82 "./src/better_sqlite3.lzz"
|
||||
: privileged_info (NULL), next_id (0), cs (isolate)
|
||||
#line 85 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
static uv_once_t init_once = UV_ONCE_INIT;
|
||||
uv_once(&init_once, InitLoggerOnce);
|
||||
uv_key_set(&thread_key, this);
|
||||
}
|
||||
#line 101 "./src/better_sqlite3.lzz"
|
||||
#line 104 "./src/better_sqlite3.lzz"
|
||||
uv_key_t Addon::thread_key;
|
||||
#undef LZZ_INLINE
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
#include <node.h>
|
||||
#include <node_buffer.h>
|
||||
#include <uv.h>
|
||||
#include "fts5-tokenizer.h"
|
||||
#line 31 "./src/util/macros.lzz"
|
||||
template <class T> using CopyablePersistent = v8::Persistent<T, v8::CopyablePersistentTraits<T>>;
|
||||
#line 36 "./src/util/binder.lzz"
|
||||
@ -197,13 +198,15 @@ private:
|
||||
#line 72 "./src/util/bind-map.lzz"
|
||||
int length;
|
||||
};
|
||||
#line 21 "./src/better_sqlite3.lzz"
|
||||
struct Addon;
|
||||
#line 22 "./src/better_sqlite3.lzz"
|
||||
class Statement;
|
||||
struct Addon;
|
||||
#line 23 "./src/better_sqlite3.lzz"
|
||||
class TokenizerModule;
|
||||
class Statement;
|
||||
#line 24 "./src/better_sqlite3.lzz"
|
||||
class TokenizerModule;
|
||||
#line 25 "./src/better_sqlite3.lzz"
|
||||
class ICUTokenizerModule;
|
||||
#line 26 "./src/better_sqlite3.lzz"
|
||||
class Backup;
|
||||
#line 1 "./src/objects/database.lzz"
|
||||
class Database : public ObjectWrapForElectron22
|
||||
@ -287,66 +290,68 @@ private:
|
||||
#line 127 "./src/objects/database.lzz"
|
||||
explicit Database (v8::Isolate * isolate, Addon * addon, sqlite3 * db_handle, v8::Local <v8::Value> logger);
|
||||
#line 150 "./src/objects/database.lzz"
|
||||
fts5_api * GetFTS5API ();
|
||||
#line 174 "./src/objects/database.lzz"
|
||||
static void JS_new (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 202 "./src/objects/database.lzz"
|
||||
#line 236 "./src/objects/database.lzz"
|
||||
static void JS_prepare (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 218 "./src/objects/database.lzz"
|
||||
#line 252 "./src/objects/database.lzz"
|
||||
static void JS_exec (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 258 "./src/objects/database.lzz"
|
||||
#line 292 "./src/objects/database.lzz"
|
||||
static void JS_backup (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 276 "./src/objects/database.lzz"
|
||||
#line 310 "./src/objects/database.lzz"
|
||||
static void JS_serialize (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 298 "./src/objects/database.lzz"
|
||||
#line 332 "./src/objects/database.lzz"
|
||||
static void JS_function (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 322 "./src/objects/database.lzz"
|
||||
#line 356 "./src/objects/database.lzz"
|
||||
static void JS_aggregate (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 351 "./src/objects/database.lzz"
|
||||
#line 385 "./src/objects/database.lzz"
|
||||
static void JS_table (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 371 "./src/objects/database.lzz"
|
||||
#line 405 "./src/objects/database.lzz"
|
||||
static void JS_loadExtension (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 393 "./src/objects/database.lzz"
|
||||
#line 427 "./src/objects/database.lzz"
|
||||
static void JS_close (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 403 "./src/objects/database.lzz"
|
||||
#line 437 "./src/objects/database.lzz"
|
||||
static void JS_defaultSafeIntegers (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 409 "./src/objects/database.lzz"
|
||||
#line 443 "./src/objects/database.lzz"
|
||||
static void JS_unsafeMode (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 416 "./src/objects/database.lzz"
|
||||
#line 450 "./src/objects/database.lzz"
|
||||
static void JS_createFTS5Tokenizer (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 452 "./src/objects/database.lzz"
|
||||
#line 469 "./src/objects/database.lzz"
|
||||
static void JS_open (v8::Local <v8 :: String> _, v8::PropertyCallbackInfo <v8 :: Value> const & info);
|
||||
#line 456 "./src/objects/database.lzz"
|
||||
#line 473 "./src/objects/database.lzz"
|
||||
static void JS_inTransaction (v8::Local <v8 :: String> _, v8::PropertyCallbackInfo <v8 :: Value> const & info);
|
||||
#line 461 "./src/objects/database.lzz"
|
||||
#line 478 "./src/objects/database.lzz"
|
||||
static bool Deserialize (v8::Local <v8::Object> buffer, Addon * addon, sqlite3 * db_handle, bool readonly);
|
||||
#line 486 "./src/objects/database.lzz"
|
||||
static void FreeSerialization (char * data, void * _);
|
||||
#line 490 "./src/objects/database.lzz"
|
||||
static int const MAX_BUFFER_SIZE = node::Buffer::kMaxLength > INT_MAX ? INT_MAX : static_cast<int>(node::Buffer::kMaxLength);
|
||||
#line 491 "./src/objects/database.lzz"
|
||||
static int const MAX_STRING_SIZE = v8::String::kMaxLength > INT_MAX ? INT_MAX : static_cast<int>(v8::String::kMaxLength);
|
||||
#line 493 "./src/objects/database.lzz"
|
||||
sqlite3 * const db_handle;
|
||||
#line 494 "./src/objects/database.lzz"
|
||||
bool open;
|
||||
#line 495 "./src/objects/database.lzz"
|
||||
bool busy;
|
||||
#line 496 "./src/objects/database.lzz"
|
||||
bool safe_ints;
|
||||
#line 497 "./src/objects/database.lzz"
|
||||
bool unsafe_mode;
|
||||
#line 498 "./src/objects/database.lzz"
|
||||
bool was_js_error;
|
||||
#line 499 "./src/objects/database.lzz"
|
||||
bool const has_logger;
|
||||
#line 500 "./src/objects/database.lzz"
|
||||
unsigned short int iterators;
|
||||
#line 501 "./src/objects/database.lzz"
|
||||
Addon * const addon;
|
||||
#line 502 "./src/objects/database.lzz"
|
||||
CopyablePersistent <v8::Value> const logger;
|
||||
#line 503 "./src/objects/database.lzz"
|
||||
static void FreeSerialization (char * data, void * _);
|
||||
#line 507 "./src/objects/database.lzz"
|
||||
static int const MAX_BUFFER_SIZE = node::Buffer::kMaxLength > INT_MAX ? INT_MAX : static_cast<int>(node::Buffer::kMaxLength);
|
||||
#line 508 "./src/objects/database.lzz"
|
||||
static int const MAX_STRING_SIZE = v8::String::kMaxLength > INT_MAX ? INT_MAX : static_cast<int>(v8::String::kMaxLength);
|
||||
#line 510 "./src/objects/database.lzz"
|
||||
sqlite3 * const db_handle;
|
||||
#line 511 "./src/objects/database.lzz"
|
||||
bool open;
|
||||
#line 512 "./src/objects/database.lzz"
|
||||
bool busy;
|
||||
#line 513 "./src/objects/database.lzz"
|
||||
bool safe_ints;
|
||||
#line 514 "./src/objects/database.lzz"
|
||||
bool unsafe_mode;
|
||||
#line 515 "./src/objects/database.lzz"
|
||||
bool was_js_error;
|
||||
#line 516 "./src/objects/database.lzz"
|
||||
bool const has_logger;
|
||||
#line 517 "./src/objects/database.lzz"
|
||||
unsigned short int iterators;
|
||||
#line 518 "./src/objects/database.lzz"
|
||||
Addon * const addon;
|
||||
#line 519 "./src/objects/database.lzz"
|
||||
CopyablePersistent <v8::Value> const logger;
|
||||
#line 520 "./src/objects/database.lzz"
|
||||
std::set <Statement*, CompareStatement> stmts;
|
||||
#line 504 "./src/objects/database.lzz"
|
||||
#line 521 "./src/objects/database.lzz"
|
||||
std::set <Backup*, CompareBackup> backups;
|
||||
};
|
||||
#line 1 "./src/objects/statement.lzz"
|
||||
@ -559,6 +564,26 @@ private:
|
||||
#line 157 "./src/objects/tokenizer.lzz"
|
||||
CopyablePersistent <v8::Function> const create_instance_fn;
|
||||
};
|
||||
#line 1 "./src/objects/icu-tokenizer.lzz"
|
||||
class ICUTokenizerModule
|
||||
{
|
||||
#line 2 "./src/objects/icu-tokenizer.lzz"
|
||||
public:
|
||||
#line 3 "./src/objects/icu-tokenizer.lzz"
|
||||
ICUTokenizerModule ();
|
||||
#line 5 "./src/objects/icu-tokenizer.lzz"
|
||||
static void xDestroy (void * pCtx);
|
||||
#line 9 "./src/objects/icu-tokenizer.lzz"
|
||||
fts5_tokenizer * get_api_object ();
|
||||
#line 13 "./src/objects/icu-tokenizer.lzz"
|
||||
private:
|
||||
#line 14 "./src/objects/icu-tokenizer.lzz"
|
||||
static int xCreate (void * pCtx, char const * * azArg, int nArg, Fts5Tokenizer * * ppOut);
|
||||
#line 21 "./src/objects/icu-tokenizer.lzz"
|
||||
static void xDelete (Fts5Tokenizer * tokenizer);
|
||||
#line 25 "./src/objects/icu-tokenizer.lzz"
|
||||
static fts5_tokenizer api_object;
|
||||
};
|
||||
#line 1 "./src/util/data-converter.lzz"
|
||||
class DataConverter
|
||||
{
|
||||
@ -865,42 +890,42 @@ private:
|
||||
#line 203 "./src/util/binder.lzz"
|
||||
bool success;
|
||||
};
|
||||
#line 37 "./src/better_sqlite3.lzz"
|
||||
#line 40 "./src/better_sqlite3.lzz"
|
||||
struct Addon
|
||||
{
|
||||
#line 38 "./src/better_sqlite3.lzz"
|
||||
#line 41 "./src/better_sqlite3.lzz"
|
||||
static void JS_setErrorConstructor (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 43 "./src/better_sqlite3.lzz"
|
||||
#line 46 "./src/better_sqlite3.lzz"
|
||||
static void JS_setLogHandler (v8::FunctionCallbackInfo <v8 :: Value> const & info);
|
||||
#line 48 "./src/better_sqlite3.lzz"
|
||||
#line 51 "./src/better_sqlite3.lzz"
|
||||
static void Cleanup (void * ptr);
|
||||
#line 55 "./src/better_sqlite3.lzz"
|
||||
#line 58 "./src/better_sqlite3.lzz"
|
||||
static void SqliteLog (void * pArg, int iErrCode, char const * zMsg);
|
||||
#line 70 "./src/better_sqlite3.lzz"
|
||||
#line 73 "./src/better_sqlite3.lzz"
|
||||
static void InitLoggerOnce ();
|
||||
#line 79 "./src/better_sqlite3.lzz"
|
||||
#line 82 "./src/better_sqlite3.lzz"
|
||||
explicit Addon (v8::Isolate * isolate);
|
||||
#line 88 "./src/better_sqlite3.lzz"
|
||||
#line 91 "./src/better_sqlite3.lzz"
|
||||
sqlite3_uint64 NextId ();
|
||||
#line 92 "./src/better_sqlite3.lzz"
|
||||
CopyablePersistent <v8::Function> Statement;
|
||||
#line 93 "./src/better_sqlite3.lzz"
|
||||
CopyablePersistent <v8::Function> StatementIterator;
|
||||
#line 94 "./src/better_sqlite3.lzz"
|
||||
CopyablePersistent <v8::Function> Backup;
|
||||
#line 95 "./src/better_sqlite3.lzz"
|
||||
CopyablePersistent <v8::Function> SqliteError;
|
||||
CopyablePersistent <v8::Function> Statement;
|
||||
#line 96 "./src/better_sqlite3.lzz"
|
||||
CopyablePersistent <v8::Function> LogHandler;
|
||||
CopyablePersistent <v8::Function> StatementIterator;
|
||||
#line 97 "./src/better_sqlite3.lzz"
|
||||
v8::FunctionCallbackInfo <v8 :: Value> const * privileged_info;
|
||||
CopyablePersistent <v8::Function> Backup;
|
||||
#line 98 "./src/better_sqlite3.lzz"
|
||||
sqlite3_uint64 next_id;
|
||||
CopyablePersistent <v8::Function> SqliteError;
|
||||
#line 99 "./src/better_sqlite3.lzz"
|
||||
CS cs;
|
||||
CopyablePersistent <v8::Function> LogHandler;
|
||||
#line 100 "./src/better_sqlite3.lzz"
|
||||
std::set <Database*, Database::CompareDatabase> dbs;
|
||||
v8::FunctionCallbackInfo <v8 :: Value> const * privileged_info;
|
||||
#line 101 "./src/better_sqlite3.lzz"
|
||||
sqlite3_uint64 next_id;
|
||||
#line 102 "./src/better_sqlite3.lzz"
|
||||
CS cs;
|
||||
#line 103 "./src/better_sqlite3.lzz"
|
||||
std::set <Database*, Database::CompareDatabase> dbs;
|
||||
#line 104 "./src/better_sqlite3.lzz"
|
||||
static uv_key_t thread_key;
|
||||
};
|
||||
#line 47 "./src/util/object_wrap.lzz"
|
||||
@ -1104,6 +1129,12 @@ LZZ_INLINE fts5_tokenizer * TokenizerModule::get_api_object ()
|
||||
{
|
||||
return &api_object;
|
||||
}
|
||||
#line 9 "./src/objects/icu-tokenizer.lzz"
|
||||
LZZ_INLINE fts5_tokenizer * ICUTokenizerModule::get_api_object ()
|
||||
#line 9 "./src/objects/icu-tokenizer.lzz"
|
||||
{
|
||||
return &api_object;
|
||||
}
|
||||
#line 39 "./src/util/custom-aggregate.lzz"
|
||||
LZZ_INLINE void CustomAggregate::xStepBase (sqlite3_context * invocation, int argc, sqlite3_value * * argv, CopyablePersistent <v8::Function> const CustomAggregate::* ptrtm)
|
||||
#line 39 "./src/util/custom-aggregate.lzz"
|
||||
@ -1181,9 +1212,9 @@ LZZ_INLINE CustomTable::VTab * CustomTable::Cursor::GetVTab ()
|
||||
{
|
||||
return VTab::Upcast(base.pVtab);
|
||||
}
|
||||
#line 88 "./src/better_sqlite3.lzz"
|
||||
#line 91 "./src/better_sqlite3.lzz"
|
||||
LZZ_INLINE sqlite3_uint64 Addon::NextId ()
|
||||
#line 88 "./src/better_sqlite3.lzz"
|
||||
#line 91 "./src/better_sqlite3.lzz"
|
||||
{
|
||||
return next_id++;
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include <node.h>
|
||||
#include <node_buffer.h>
|
||||
#include <uv.h>
|
||||
#include "fts5-tokenizer.h"
|
||||
#end
|
||||
|
||||
#insert "util/object_wrap.lzz"
|
||||
@ -21,12 +22,14 @@
|
||||
struct Addon;
|
||||
class Statement;
|
||||
class TokenizerModule;
|
||||
class ICUTokenizerModule;
|
||||
class Backup;
|
||||
#insert "objects/database.lzz"
|
||||
#insert "objects/statement.lzz"
|
||||
#insert "objects/statement-iterator.lzz"
|
||||
#insert "objects/backup.lzz"
|
||||
#insert "objects/tokenizer.lzz"
|
||||
#insert "objects/icu-tokenizer.lzz"
|
||||
#insert "util/data-converter.lzz"
|
||||
#insert "util/custom-function.lzz"
|
||||
#insert "util/custom-aggregate.lzz"
|
||||
|
||||
@ -147,6 +147,30 @@ private:
|
||||
addon->dbs.insert(this);
|
||||
}
|
||||
|
||||
fts5_api* GetFTS5API() {
|
||||
// Get fts5_api object
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt = nullptr;
|
||||
|
||||
rc = sqlite3_prepare(db_handle, "SELECT fts5(?1)", -1, &pStmt, 0);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db_handle);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
fts5_api *fts5 = nullptr;
|
||||
sqlite3_bind_pointer(pStmt, 1, (void*)&fts5, "fts5_api_ptr", nullptr);
|
||||
sqlite3_step(pStmt);
|
||||
rc = sqlite3_finalize(pStmt);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db_handle);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
assert(fts5 != nullptr);
|
||||
return fts5;
|
||||
}
|
||||
|
||||
NODE_METHOD(JS_new) {
|
||||
assert(info.IsConstructCall());
|
||||
REQUIRE_ARGUMENT_STRING(first, v8::Local<v8::String> filename);
|
||||
@ -196,6 +220,16 @@ private:
|
||||
SetFrozen(isolate, ctx, info.This(), addon->cs.readonly, v8::Boolean::New(isolate, readonly));
|
||||
SetFrozen(isolate, ctx, info.This(), addon->cs.name, filenameGiven);
|
||||
|
||||
// Add ICU tokenizer
|
||||
fts5_api* fts5 = db->GetFTS5API();
|
||||
// Already threw an exception
|
||||
if (fts5 == nullptr) {
|
||||
return;
|
||||
}
|
||||
ICUTokenizerModule* icu = new ICUTokenizerModule();
|
||||
fts5->xCreateTokenizer(fts5, "icu_tokenizer", icu, icu->get_api_object(),
|
||||
&ICUTokenizerModule::xDestroy);
|
||||
|
||||
info.GetReturnValue().Set(info.This());
|
||||
}
|
||||
|
||||
@ -414,37 +448,20 @@ private:
|
||||
}
|
||||
|
||||
NODE_METHOD(JS_createFTS5Tokenizer) {
|
||||
UseAddon;
|
||||
UseIsolate;
|
||||
|
||||
Database* db = Unwrap<Database>(info.This());
|
||||
REQUIRE_ARGUMENT_STRING(first, v8::Local<v8::String> name);
|
||||
REQUIRE_ARGUMENT_FUNCTION(second, v8::Local<v8::Function> create_instance_fn);
|
||||
|
||||
// Get fts5_api object
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt = nullptr;
|
||||
|
||||
rc = sqlite3_prepare(db->db_handle, "SELECT fts5(?1)", -1, &pStmt, 0);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db->db_handle);
|
||||
return;
|
||||
}
|
||||
|
||||
fts5_api *fts5 = nullptr;
|
||||
sqlite3_bind_pointer(pStmt, 1, (void*)&fts5, "fts5_api_ptr", nullptr);
|
||||
sqlite3_step(pStmt);
|
||||
rc = sqlite3_finalize(pStmt);
|
||||
if (rc != SQLITE_OK) {
|
||||
ThrowSqliteError(addon, db->db_handle);
|
||||
return;
|
||||
}
|
||||
|
||||
assert(fts5 != nullptr);
|
||||
|
||||
TokenizerModule* t = new TokenizerModule(isolate, create_instance_fn);
|
||||
|
||||
v8::String::Utf8Value utf8(isolate, name);
|
||||
fts5_api* fts5 = db->GetFTS5API();
|
||||
// Already threw an exception
|
||||
if (fts5 == nullptr) {
|
||||
return;
|
||||
}
|
||||
fts5->xCreateTokenizer(fts5, *utf8, t, t->get_api_object(),
|
||||
&TokenizerModule::xDestroy);
|
||||
}
|
||||
|
||||
30
src/objects/icu-tokenizer.lzz
Normal file
30
src/objects/icu-tokenizer.lzz
Normal file
@ -0,0 +1,30 @@
|
||||
class ICUTokenizerModule {
|
||||
public:
|
||||
ICUTokenizerModule() {}
|
||||
|
||||
static void xDestroy(void* pCtx) {
|
||||
// No-op
|
||||
}
|
||||
|
||||
inline fts5_tokenizer* get_api_object() {
|
||||
return &api_object;
|
||||
}
|
||||
|
||||
private:
|
||||
static int xCreate(
|
||||
void* pCtx, const char** azArg, int nArg, Fts5Tokenizer** ppOut) {
|
||||
TokenizerModule* m = static_cast<TokenizerModule*>(pCtx);
|
||||
*ppOut = reinterpret_cast<Fts5Tokenizer*>(m);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
static void xDelete(Fts5Tokenizer* tokenizer) {
|
||||
// No-op
|
||||
}
|
||||
|
||||
static fts5_tokenizer api_object = {
|
||||
&xCreate,
|
||||
&xDelete,
|
||||
fts5_icu_tokenize,
|
||||
};
|
||||
};
|
||||
58
test/38.database.icu-tokenizer.js
Normal file
58
test/38.database.icu-tokenizer.js
Normal file
@ -0,0 +1,58 @@
|
||||
'use strict';
|
||||
const Database = require('../.');
|
||||
|
||||
const segmenter = new Intl.Segmenter([], {
|
||||
granularity: 'word',
|
||||
});
|
||||
|
||||
const DIACRITICS = /[\u0300-\u036f]/g;
|
||||
|
||||
function removeDiacritics(str) {
|
||||
return str.normalize('NFD').replace(DIACRITICS, '');
|
||||
}
|
||||
|
||||
describe('Database#serialize()', function () {
|
||||
beforeEach(function () {
|
||||
this.db = new Database(':memory:');
|
||||
|
||||
this.db.prepare("CREATE VIRTUAL TABLE fts USING fts5(content, tokenize='icu_tokenizer')").run();
|
||||
this.insertStmt = this.db.prepare("INSERT INTO fts (content) VALUES (?)");
|
||||
this.lookupStmt = this.db.prepare(
|
||||
"SELECT snippet(fts, -1, '[', ']', '...', 20) " +
|
||||
"FROM fts " +
|
||||
"WHERE content MATCH $query").pluck();
|
||||
});
|
||||
afterEach(function () {
|
||||
this.db.close();
|
||||
});
|
||||
|
||||
it("should support CJK symbols at the start", function() {
|
||||
this.insertStmt.run("知识需要时间");
|
||||
const rows = this.lookupStmt.all({ query: "知*" });
|
||||
expect(rows).to.eql(["[知]识需要时间"]);
|
||||
});
|
||||
|
||||
it("should support CJK symbols in the middle", function() {
|
||||
this.insertStmt.run("知识需要时间");
|
||||
const rows = this.lookupStmt.all({ query: "需*" });
|
||||
expect(rows).to.eql(["知识[需]要时间"]);
|
||||
});
|
||||
|
||||
it("should support Korean symbols", function() {
|
||||
this.insertStmt.run("안녕 세상");
|
||||
const rows = this.lookupStmt.all({ query: "세*" });
|
||||
expect(rows).to.eql(["안녕 [세상]"]);
|
||||
});
|
||||
|
||||
it("should support normalization", function() {
|
||||
this.insertStmt.run("dïācrîtįcs");
|
||||
const rows = this.lookupStmt.all({ query: "diacritics*" });
|
||||
expect(rows).to.eql(["[dïācrîtįcs]"]);
|
||||
});
|
||||
|
||||
it("should support punctuation", function() {
|
||||
this.insertStmt.run("Hello!world! how are you?");
|
||||
const rows = this.lookupStmt.all({ query: "h*" });
|
||||
expect(rows).to.eql(["[Hello]!world! [how] are you?"]);
|
||||
});
|
||||
});
|
||||
@ -38,7 +38,7 @@ describe('Database#serialize()', function () {
|
||||
this.db.prepare("CREATE VIRTUAL TABLE fts USING fts5(content, tokenize='js arg1 arg2')").run();
|
||||
this.insertStmt = this.db.prepare("INSERT INTO fts (content) VALUES (?)");
|
||||
this.lookupStmt = this.db.prepare(
|
||||
"SELECT snippet(fts, -1, '[', ']', '...', 10) " +
|
||||
"SELECT snippet(fts, -1, '[', ']', '...', 20) " +
|
||||
"FROM fts " +
|
||||
"WHERE content MATCH $query").pluck();
|
||||
});
|
||||
@ -58,6 +58,12 @@ describe('Database#serialize()', function () {
|
||||
expect(rows).to.eql(["知识[需要]时间"]);
|
||||
});
|
||||
|
||||
it("should support Korean symbols", function() {
|
||||
this.insertStmt.run("안녕 세상");
|
||||
const rows = this.lookupStmt.all({ query: "세*" });
|
||||
expect(rows).to.eql(["안녕 [세상]"]);
|
||||
});
|
||||
|
||||
it("should support normalization", function() {
|
||||
this.insertStmt.run("dïācrîtįcs");
|
||||
const rows = this.lookupStmt.all({ query: "diacritics*" });
|
||||
Loading…
Reference in New Issue
Block a user