Mirrors/22120
1
0
Fork 0
mirror of https://github.com/c9fe/22120.git synced 2024-11-10 04:52:43 +01:00
Code Issues Releases Wiki Activity

"Adding search"

Browse Source
This commit is contained in:
Cris Stringfellow 2021-12-10 17:16:13 +08:00
parent e939adceb0
commit 9478a93729
3 changed files with 20 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
archivist.js
View File

@ -2,6 +2,7 @@ import hasha from 'hasha';
import {URL} from 'url';
import path from 'path';
import fs from 'fs';
import FlexSearch from 'flexsearch';
import args from './args.js';
import {APP_ROOT, context, sleep, DEBUG} from './common.js';
import {connect} from './protocol.js';
@ -14,6 +15,11 @@ import {BLOCKED_BODY, BLOCKED_CODE, BLOCKED_HEADERS} from './blockedResponse.js'
// that holds the serialized requests
// that are saved on disk
let Fs, Mode, Close;
const {Index, registerCharset, registerLanguage} = FlexSearch;
const FLEX_OPTS = {
context: true,
};
const Flex = new Index(FLEX_OPTS);
const Cache = new Map();
const State = {
Cache,
@ -234,6 +240,7 @@ async function collect({chrome_port:port, mode} = {}) {
(Text, {nodeValue}) => Text + nodeValue + ' ',
''
);
if ( false ) {
console.log({
page : {
@ -243,6 +250,7 @@ async function collect({chrome_port:port, mode} = {}) {
}
});
}
Flex.updateAsync(info.url, pageText);
}
console.log(`Indexed ${info.url} to ${info.title}`);

11
package-lock.json generated
View File

@ -13,6 +13,7 @@
"chrome-launcher": "latest",
"esm": "^3.2.25",
"express": "latest",
"flexsearch": "^0.7.21",
"hasha": "latest",
"node-fetch": "latest",
"ws": "latest"
@ -1899,6 +1900,11 @@
"node": ">=8"
}
},
"node_modules/flexsearch": {
"version": "0.7.21",
"resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.21.tgz",
"integrity": "sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg=="
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -10891,6 +10897,11 @@
"path-exists": "^4.0.0"
}
},
"flexsearch": {
"version": "0.7.21",
"resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.21.tgz",
"integrity": "sha512-W7cHV7Hrwjid6lWmy0IhsWDFQboWSng25U3VVywpHOTJnnAZNPScog67G+cVpeX9f7yDD21ih0WDrMMT+JoaYg=="
},
"forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",

1
package.json
View File

@ -35,6 +35,7 @@
"chrome-launcher": "latest",
"esm": "^3.2.25",
"express": "latest",
"flexsearch": "^0.7.21",
"hasha": "latest",
"node-fetch": "latest",
"ws": "latest"