Mirrors/22120
1
0
Fork 0
mirror of https://github.com/c9fe/22120.git synced 2024-11-14 23:12:30 +01:00
Code Issues Releases Wiki Activity

"Added ability to export flexsearch FTS data"

Browse Source
This commit is contained in:
Cris Stringfellow 2021-12-16 13:57:29 +08:00
parent 93dfa1d409
commit b9ca04f9e9
2 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
archivist.js
View File

@ -81,7 +81,6 @@ const UNCACHED = {
body:UNCACHED_BODY, responseCode:UNCACHED_CODE, responseHeaders:UNCACHED_HEADERS
}
export default Archivist;
async function collect({chrome_port:port, mode} = {}) {
@ -284,13 +283,14 @@ async function collect({chrome_port:port, mode} = {}) {
}
async function indexURL({targetInfo:info = {}, sessionId, waitingForDebugger} = {}) {
console.log('Index URL called', info);
if ( Mode == 'serve' ) return;
if ( info.type != 'page' ) return;
if ( ! info.url || info.url == 'about:blank' ) return;
if ( info.url.startsWith('chrome') ) return;
if ( dontCache(info) ) return;
console.log('Index URL called', info);
if ( State.Indexing.has(info.targetId) ) return;
State.Indexing.add(info.targetId);
@ -670,31 +670,29 @@ async function saveFTS(path) {
clearTimeout(State.ftsIndexSaver);
const DEBUG = true;
if ( context == 'node' ) {
DEBUG && console.log("Writing FTS index to", path || FTS_INDEX_DIR());
const dir = path || FTS_INDEX_DIR();
if ( UpdatedKeys.size ) {
let writeCount = 0;
DEBUG && console.log(`${UpdatedKeys.size} keys updated since last write`);
Flex.export((key, data) => {
if ( UpdatedKeys.has(key) ) {
try {
Fs.writeFileSync(
/* haha .flx file extensionf or flexsearch index date file */
Path.resolve(dir, `${hash(key, HASH_OPTS)}.flx`),
Path.resolve(dir, key),
data
);
UpdatedKeys.delete(key);
writeCount++;
} catch(e) {
console.error('Error writing full text search index', e);
}
});
DEBUG && console.log("Wrote FTS index: ", writeCount, "files");
UpdatedKeys.clear();
} else {
DEBUG && console.log("No FTS keys updated, no writes needed this time.");
}
}
State.ftsIndexSaver = setTimeout(saveIndex, 31001);
State.ftsIndexSaver = setTimeout(saveFTS, 31001);
State.ftsSaveInProgress = false;
}

6
args.js
View File

@ -71,6 +71,12 @@ function updateBasePath(new_base_path) {
console.log(`Created!`);
}
if ( !fs.existsSync(fts_index_dir()) ) {
console.log(`FTS Index directory does not exist, creating...`);
fs.mkdirSync(fts_index_dir(), {recursive:true});
console.log(`Created!`);
}
console.log(`Base path updated to: ${BasePath}. Saving to preferences...`);
Pref.BasePath = BasePath;
savePref();