signalapp/better-sqlite3
signalapp/better-sqlite3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename tokenizer to signal_tokenizer

Browse Source
This commit is contained in:
Fedor Indutnyy 2023-01-26 13:56:15 -08:00
parent 583db4c9ee
commit 2e432b0aad
9 changed files with 56 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
deps/download.js vendored
View File

@ -6,11 +6,12 @@ const { Transform } = require('stream');
const { pipeline } = require('stream/promises');
const BASE_URI = `https://build-artifacts.signal.org/desktop`;
const HASH = '01d2f46ae275286ed552cfc977239e7dba1ca2cd572f1d0c057a29d87b1f9c79';
const HASH = '057008836632dfa5188f84eb073a3036688bacc73cfc64ad18af7a1f4e21b1c3';
const SQLCIPHER_VERSION = '4.5.2';
const OPENSSL_VERSION = '3.0.7';
const URL = `${BASE_URI}/sqlcipher-${SQLCIPHER_VERSION}--${OPENSSL_VERSION}-` +
`${HASH}.tar.gz`;
const TOKENIZER_VERSION = '0.1.0';
const TAG = [SQLCIPHER_VERSION, OPENSSL_VERSION, TOKENIZER_VERSION].join('--');
const URL = `${BASE_URI}/sqlcipher-${TAG}-${HASH}.tar.gz`;
const tmpFile = path.join(__dirname, 'unverified.tmp');
const finalFile = path.join(__dirname, 'sqlcipher.tar.gz');

12
deps/sqlite3.gyp vendored
View File

@ -25,7 +25,7 @@
'outputs': [
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/libssl.lib',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/libcrypto.lib',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-pc-windows-msvc/fts5_tokenizer.lib',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/signal-tokenizer/>(rust_arch)-pc-windows-msvc/signal_tokenizer.lib',
],
}],
],
@ -42,7 +42,7 @@
'files': [
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/libssl.lib',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/>(openssl_root)/libcrypto.lib',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-pc-windows-msvc/fts5_tokenizer.lib',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/signal-tokenizer/>(rust_arch)-pc-windows-msvc/signal_tokenizer.lib',
],
'destination': '<(PRODUCT_DIR)',
}],
@ -62,7 +62,7 @@
'include_dirs': [
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/openssl-include',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/include',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/signal-tokenizer/include',
],
},
'cflags': ['-std=c99', '-w'],
@ -82,7 +82,7 @@
'-llibssl.lib',
'-lws2_32.lib',
'-lcrypt32.lib',
'-lfts5_tokenizer.lib',
'-lsignal_tokenizer.lib',
],
'library_dirs': [
'<(PRODUCT_DIR)',
@ -94,7 +94,7 @@
'libraries': [
# This statically links libcrypto, whereas -lcrypto would dynamically link it
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/OpenSSL-mac-<(target_arch)/libcrypto.a',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-apple-darwin/libfts5_tokenizer.a',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/signal-tokenizer/>(rust_arch)-apple-darwin/libsignal_tokenizer.a',
]
}
},
@ -103,7 +103,7 @@
'libraries': [
# This statically links libcrypto, whereas -lcrypto would dynamically link it
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/OpenSSL-linux-<(target_arch)/libcrypto.a',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/fts5-tokenizer/>(rust_arch)-unknown-linux-gnu/libfts5_tokenizer.a',
'<(SHARED_INTERMEDIATE_DIR)/sqlite3/signal-tokenizer/>(rust_arch)-unknown-linux-gnu/libsignal_tokenizer.a',
]
}
}],

2
index.d.ts vendored
View File

@ -88,7 +88,7 @@ declare namespace BetterSqlite3 {
function(name: string, cb: (...params: any[]) => any): this;
function(name: string, options: Database.RegistrationOptions, cb: (...params: any[]) => any): this;
aggregate(name: string, options: Database.AggregateOptions): this;
loadExtension(path: string): this;
loadExtension(path: string, entryPoint?: string): this;
close(): this;
defaultSafeIntegers(toggleState?: boolean): this;
backup(destinationFile: string, options?: Database.BackupOptions): Promise<Database.BackupMetadata>;

38
src/better_sqlite3.cpp
View File

@ -550,9 +550,9 @@ void Database::JS_new (v8::FunctionCallbackInfo <v8 :: Value> const & info)
if (fts5 == nullptr) {
return;
}
ICUTokenizerModule* icu = new ICUTokenizerModule();
fts5->xCreateTokenizer(fts5, "icu_tokenizer", icu, icu->get_api_object(),
&ICUTokenizerModule::xDestroy);
SignalTokenizerModule* icu = new SignalTokenizerModule();
fts5->xCreateTokenizer(fts5, "signal_tokenizer", icu, icu->get_api_object(),
&SignalTokenizerModule::xDestroy);
info.GetReturnValue().Set(info.This());
}
@ -1574,31 +1574,31 @@ fts5_tokenizer TokenizerModule::api_object = {
&xDelete,
&xTokenize,
};
#line 3 "./src/objects/icu-tokenizer.lzz"
ICUTokenizerModule::ICUTokenizerModule ()
#line 3 "./src/objects/icu-tokenizer.lzz"
{}
#line 5 "./src/objects/icu-tokenizer.lzz"
void ICUTokenizerModule::xDestroy (void * pCtx)
#line 5 "./src/objects/icu-tokenizer.lzz"
#line 3 "./src/objects/signal-tokenizer.lzz"
SignalTokenizerModule::SignalTokenizerModule ()
#line 3 "./src/objects/signal-tokenizer.lzz"
{}
#line 5 "./src/objects/signal-tokenizer.lzz"
void SignalTokenizerModule::xDestroy (void * pCtx)
#line 5 "./src/objects/signal-tokenizer.lzz"
{}
#line 14 "./src/objects/icu-tokenizer.lzz"
int ICUTokenizerModule::xCreate (void * pCtx, char const * * azArg, int nArg, Fts5Tokenizer * * ppOut)
#line 15 "./src/objects/icu-tokenizer.lzz"
#line 14 "./src/objects/signal-tokenizer.lzz"
int SignalTokenizerModule::xCreate (void * pCtx, char const * * azArg, int nArg, Fts5Tokenizer * * ppOut)
#line 15 "./src/objects/signal-tokenizer.lzz"
{
TokenizerModule* m = static_cast<TokenizerModule*>(pCtx);
*ppOut = reinterpret_cast<Fts5Tokenizer*>(m);
return SQLITE_OK;
}
#line 21 "./src/objects/icu-tokenizer.lzz"
void ICUTokenizerModule::xDelete (Fts5Tokenizer * tokenizer)
#line 21 "./src/objects/icu-tokenizer.lzz"
#line 21 "./src/objects/signal-tokenizer.lzz"
void SignalTokenizerModule::xDelete (Fts5Tokenizer * tokenizer)
#line 21 "./src/objects/signal-tokenizer.lzz"
{}
#line 25 "./src/objects/icu-tokenizer.lzz"
fts5_tokenizer ICUTokenizerModule::api_object = {
#line 25 "./src/objects/signal-tokenizer.lzz"
fts5_tokenizer SignalTokenizerModule::api_object = {
&xCreate,
&xDelete,
fts5_icu_tokenize,
signal_fts5_tokenize,
};
#line 4 "./src/util/data-converter.lzz"
void DataConverter::ThrowDataConversionError (sqlite3_context * invocation, bool isBigInt)

32
src/better_sqlite3.hpp
View File

@ -16,7 +16,7 @@
#include <node.h>
#include <node_buffer.h>
#include <uv.h>
#include "fts5-tokenizer.h"
#include "signal-tokenizer.h"
#line 31 "./src/util/macros.lzz"
template <class T> using CopyablePersistent = v8::Persistent<T, v8::CopyablePersistentTraits<T>>;
#line 36 "./src/util/binder.lzz"
@ -205,7 +205,7 @@ class Statement;
#line 24 "./src/better_sqlite3.lzz"
class TokenizerModule;
#line 25 "./src/better_sqlite3.lzz"
class ICUTokenizerModule;
class SignalTokenizerModule;
#line 26 "./src/better_sqlite3.lzz"
class Backup;
#line 1 "./src/objects/database.lzz"
@ -564,24 +564,24 @@ private:
#line 157 "./src/objects/tokenizer.lzz"
CopyablePersistent <v8::Function> const create_instance_fn;
};
#line 1 "./src/objects/icu-tokenizer.lzz"
class ICUTokenizerModule
#line 1 "./src/objects/signal-tokenizer.lzz"
class SignalTokenizerModule
{
#line 2 "./src/objects/icu-tokenizer.lzz"
#line 2 "./src/objects/signal-tokenizer.lzz"
public:
#line 3 "./src/objects/icu-tokenizer.lzz"
ICUTokenizerModule ();
#line 5 "./src/objects/icu-tokenizer.lzz"
#line 3 "./src/objects/signal-tokenizer.lzz"
SignalTokenizerModule ();
#line 5 "./src/objects/signal-tokenizer.lzz"
static void xDestroy (void * pCtx);
#line 9 "./src/objects/icu-tokenizer.lzz"
#line 9 "./src/objects/signal-tokenizer.lzz"
fts5_tokenizer * get_api_object ();
#line 13 "./src/objects/icu-tokenizer.lzz"
#line 13 "./src/objects/signal-tokenizer.lzz"
private:
#line 14 "./src/objects/icu-tokenizer.lzz"
#line 14 "./src/objects/signal-tokenizer.lzz"
static int xCreate (void * pCtx, char const * * azArg, int nArg, Fts5Tokenizer * * ppOut);
#line 21 "./src/objects/icu-tokenizer.lzz"
#line 21 "./src/objects/signal-tokenizer.lzz"
static void xDelete (Fts5Tokenizer * tokenizer);
#line 25 "./src/objects/icu-tokenizer.lzz"
#line 25 "./src/objects/signal-tokenizer.lzz"
static fts5_tokenizer api_object;
};
#line 1 "./src/util/data-converter.lzz"
@ -1129,9 +1129,9 @@ LZZ_INLINE fts5_tokenizer * TokenizerModule::get_api_object ()
{
return &api_object;
}
#line 9 "./src/objects/icu-tokenizer.lzz"
LZZ_INLINE fts5_tokenizer * ICUTokenizerModule::get_api_object ()
#line 9 "./src/objects/icu-tokenizer.lzz"
#line 9 "./src/objects/signal-tokenizer.lzz"
LZZ_INLINE fts5_tokenizer * SignalTokenizerModule::get_api_object ()
#line 9 "./src/objects/signal-tokenizer.lzz"
{
return &api_object;
}

6
src/better_sqlite3.lzz
View File

@ -11,7 +11,7 @@
#include <node.h>
#include <node_buffer.h>
#include <uv.h>
#include "fts5-tokenizer.h"
#include "signal-tokenizer.h"
#end
#insert "util/object_wrap.lzz"
@ -22,14 +22,14 @@
struct Addon;
class Statement;
class TokenizerModule;
class ICUTokenizerModule;
class SignalTokenizerModule;
class Backup;
#insert "objects/database.lzz"
#insert "objects/statement.lzz"
#insert "objects/statement-iterator.lzz"
#insert "objects/backup.lzz"
#insert "objects/tokenizer.lzz"
#insert "objects/icu-tokenizer.lzz"
#insert "objects/signal-tokenizer.lzz"
#insert "util/data-converter.lzz"
#insert "util/custom-function.lzz"
#insert "util/custom-aggregate.lzz"

6
src/objects/database.lzz
View File

@ -226,9 +226,9 @@ private:
if (fts5 == nullptr) {
return;
}
ICUTokenizerModule* icu = new ICUTokenizerModule();
fts5->xCreateTokenizer(fts5, "icu_tokenizer", icu, icu->get_api_object(),
&ICUTokenizerModule::xDestroy);
SignalTokenizerModule* icu = new SignalTokenizerModule();
fts5->xCreateTokenizer(fts5, "signal_tokenizer", icu, icu->get_api_object(),
&SignalTokenizerModule::xDestroy);
info.GetReturnValue().Set(info.This());
}

6
src/objects/icu-tokenizer.lzz → src/objects/signal-tokenizer.lzz
View File

@ -1,6 +1,6 @@
class ICUTokenizerModule {
class SignalTokenizerModule {
public:
ICUTokenizerModule() {}
SignalTokenizerModule() {}
static void xDestroy(void* pCtx) {
// No-op
@ -25,6 +25,6 @@ private:
static fts5_tokenizer api_object = {
&xCreate,
&xDelete,
fts5_icu_tokenize,
signal_fts5_tokenize,
};
};

2
test/39.database.icu-tokenizer.js → test/39.database.signal-tokenizer.js
View File

@ -15,7 +15,7 @@ describe('Database#serialize()', function () {
beforeEach(function () {
this.db = new Database(':memory:');
this.db.prepare("CREATE VIRTUAL TABLE fts USING fts5(content, tokenize='icu_tokenizer')").run();
this.db.prepare("CREATE VIRTUAL TABLE fts USING fts5(content, tokenize='signal_tokenizer')").run();
this.insertStmt = this.db.prepare("INSERT INTO fts (content) VALUES (?)");
this.lookupStmt = this.db.prepare(
"SELECT snippet(fts, -1, '[', ']', '...', 20) " +