Mirrors/uBlock
1
0
Fork 0
mirror of https://github.com/gorhill/uBlock.git synced 2024-11-23 10:52:43 +01:00
Code Issues Releases Wiki Activity

Fine-tune cache storage related-code

Browse Source 
Related discussion:
https://github.com/uBlockOrigin/uBlock-discussions/discussions/876

Related commit:
086766a924
This commit is contained in:
Raymond Hill 2024-02-27 15:04:05 -05:00
parent 4d88b5121c
commit 7590c0711d
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
4 changed files with 161 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/js/background.js
View File

@ -56,6 +56,7 @@ const hiddenSettingsDefault = {
blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001',
cacheStorageAPI: 'unset', cacheStorageAPI: 'unset',
cacheStorageCompression: true, cacheStorageCompression: true,
cacheStorageCompressionThreshold: 65536,
cacheStorageMultithread: 2, cacheStorageMultithread: 2,
cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
cloudStorageCompression: true, cloudStorageCompression: true,

84
src/js/cachestorage.js
View File

@ -58,6 +58,19 @@ const shouldCache = bin => {
return out; return out;
}; };
const missingKeys = (wanted, inbin, outbin) => {
inbin = inbin || {};
const found = Object.keys(inbin);
Object.assign(outbin, inbin);
if ( found.length === wanted.length ) { return; }
const missing = [];
for ( const key of wanted ) {
if ( outbin.hasOwnProperty(key) ) { continue; }
missing.push(key);
}
return missing;
};
/******************************************************************************* /*******************************************************************************
* *
* Extension storage * Extension storage
@ -68,11 +81,10 @@ const shouldCache = bin => {
const cacheStorage = (( ) => { const cacheStorage = (( ) => {
const LARGE = 65536;
const compress = async (key, data) => { const compress = async (key, data) => {
const isLarge = typeof data === 'string' && data.length >= LARGE;
const µbhs = µb.hiddenSettings; const µbhs = µb.hiddenSettings;
const isLarge = typeof data === 'string' &&
data.length >= µbhs.cacheStorageCompressionThreshold;
const after = await scuo.serializeAsync(data, { const after = await scuo.serializeAsync(data, {
compress: isLarge && µbhs.cacheStorageCompression, compress: isLarge && µbhs.cacheStorageCompression,
multithreaded: isLarge && µbhs.cacheStorageMultithread || 0, multithreaded: isLarge && µbhs.cacheStorageMultithread || 0,
@ -80,40 +92,39 @@ const cacheStorage = (( ) => {
return { key, data: after }; return { key, data: after };
}; };
const decompress = async (key, data) => { const decompress = async (bin, key) => {
if ( scuo.canDeserialize(data) === false ) { const data = bin[key];
return { key, data }; if ( scuo.isSerialized(data) === false ) { return; }
} const µbhs = µb.hiddenSettings;
const isLarge = data.length >= LARGE; const isLarge = data.length >= µbhs.cacheStorageCompressionThreshold;
const after = await scuo.deserializeAsync(data, { bin[key] = await scuo.deserializeAsync(data, {
multithreaded: isLarge && µb.hiddenSettings.cacheStorageMultithread || 0, multithreaded: isLarge && µbhs.cacheStorageMultithread || 0,
}); });
return { key, data: after };
}; };
return { return {
name: 'browser.storage.local', get(argbin) {
const outbin = {};
get(arg) { const wanted0 = keysFromGetArg(argbin);
const keys = arg; return cacheAPI.get(wanted0).then(bin => {
return cacheAPI.get(keysFromGetArg(arg)).then(bin => { const wanted1 = missingKeys(wanted0, bin, outbin);
if ( bin !== undefined ) { return bin; } if ( wanted1 === undefined ) { return; }
return extensionStorage.get(keys).catch(reason => { return extensionStorage.get(wanted1).then(bin => {
ubolog(reason); const wanted2 = missingKeys(wanted1, bin, outbin);
if ( wanted2 === undefined ) { return; }
if ( argbin instanceof Object === false ) { return; }
if ( Array.isArray(argbin) ) { return; }
for ( const key of wanted2 ) {
if ( argbin.hasOwnProperty(key) === false ) { continue; }
outbin[key] = argbin[key];
}
}); });
}).then(bin => { }).then(( ) => {
if ( bin instanceof Object === false ) { return bin; }
const promises = []; const promises = [];
for ( const key of Object.keys(bin) ) { for ( const key of Object.keys(outbin) ) {
promises.push(decompress(key, bin[key])); promises.push(decompress(outbin, key));
} }
return Promise.all(promises); return Promise.all(promises).then(( ) => outbin);
}).then(results => {
const bin = {};
for ( const { key, data } of results ) {
bin[key] = data;
}
return bin;
}).catch(reason => { }).catch(reason => {
ubolog(reason); ubolog(reason);
}); });
@ -183,8 +194,6 @@ const cacheStorage = (( ) => {
idbStorage.clear(); idbStorage.clear();
return Promise.all(toMigrate); return Promise.all(toMigrate);
}, },
error: undefined
}; };
})(); })();
@ -217,6 +226,7 @@ const cacheAPI = (( ) => {
} }
resolve(caches.open(STORAGE_NAME).catch(reason => { resolve(caches.open(STORAGE_NAME).catch(reason => {
ubolog(reason); ubolog(reason);
return null;
})); }));
}); });
@ -232,7 +242,7 @@ const cacheAPI = (( ) => {
const cache = await cacheStoragePromise; const cache = await cacheStoragePromise;
if ( cache === null ) { return; } if ( cache === null ) { return; }
return cache.match(keyToURL(key)).then(response => { return cache.match(keyToURL(key)).then(response => {
if ( response instanceof Response === false ) { return; } if ( response === undefined ) { return; }
return response.text(); return response.text();
}).then(text => { }).then(text => {
if ( text === undefined ) { return; } if ( text === undefined ) { return; }
@ -302,7 +312,7 @@ const cacheAPI = (( ) => {
} }
const responses = await Promise.all(toFetch); const responses = await Promise.all(toFetch);
for ( const response of responses ) { for ( const response of responses ) {
if ( response instanceof Object === false ) { continue; } if ( response === undefined ) { continue; }
const { key, text } = response; const { key, text } = response;
if ( typeof key !== 'string' ) { continue; } if ( typeof key !== 'string' ) { continue; }
if ( typeof text !== 'string' ) { continue; } if ( typeof text !== 'string' ) { continue; }
@ -321,7 +331,7 @@ const cacheAPI = (( ) => {
).catch(( ) => []); ).catch(( ) => []);
}, },
async set(keyvalStore) { set(keyvalStore) {
const keys = Object.keys(keyvalStore); const keys = Object.keys(keyvalStore);
if ( keys.length === 0 ) { return; } if ( keys.length === 0 ) { return; }
const promises = []; const promises = [];
@ -331,7 +341,7 @@ const cacheAPI = (( ) => {
return Promise.all(promises); return Promise.all(promises);
}, },
async remove(keys) { remove(keys) {
const toRemove = []; const toRemove = [];
if ( typeof keys === 'string' ) { if ( typeof keys === 'string' ) {
toRemove.push(removeOne(keys)); toRemove.push(removeOne(keys));
@ -343,7 +353,7 @@ const cacheAPI = (( ) => {
return Promise.all(toRemove); return Promise.all(toRemove);
}, },
async clear() { clear() {
return globalThis.caches.delete(STORAGE_NAME).catch(reason => { return globalThis.caches.delete(STORAGE_NAME).catch(reason => {
ubolog(reason); ubolog(reason);
}); });

2
src/js/messaging.js
View File

@ -948,7 +948,7 @@ const onMessage = function(request, sender, callback) {
case 'cloudPull': case 'cloudPull':
request.decode = encoded => { request.decode = encoded => {
if ( scuo.canDeserialize(encoded) ) { if ( scuo.isSerialized(encoded) ) {
return scuo.deserializeAsync(encoded, { thread: true }); return scuo.deserializeAsync(encoded, { thread: true });
} }
// Legacy decoding: needs to be kept around for the foreseeable future. // Legacy decoding: needs to be kept around for the foreseeable future.

135
src/js/scuo-serializer.js
View File

@ -1096,10 +1096,13 @@ export const deserialize = s => {
return data; return data;
}; };
export const canDeserialize = s => export const isSerialized = s =>
typeof s === 'string' && typeof s === 'string' &&
(s.startsWith(MAGICLZ4PREFIX) || s.startsWith(MAGICPREFIX)); (s.startsWith(MAGICLZ4PREFIX) || s.startsWith(MAGICPREFIX));
export const isCompressed = s =>
typeof s === 'string' && s.startsWith(MAGICLZ4PREFIX);
/******************************************************************************* /*******************************************************************************
* *
* Configuration * Configuration
@ -1137,10 +1140,78 @@ export const setConfig = config => {
* *
* */ * */
const THREAD_AREYOUREADY = 1;
const THREAD_IAMREADY = 2;
const THREAD_SERIALIZE = 3;
const THREAD_DESERIALIZE = 4;
class MainThread {
constructor() {
this.jobs = [];
this.workload = 0;
this.timer = undefined;
this.busy = false;
}
process() {
if ( this.jobs.length === 0 ) { return; }
const job = this.jobs.shift();
this.workload -= job.size;
const result = job.what === THREAD_SERIALIZE
? serialize(job.data, job.options)
: deserialize(job.data);
job.resolve(result);
this.processAsync();
if ( this.jobs.length === 0 ) {
this.busy = false;
}
}
processAsync() {
if ( this.timer !== undefined ) { return; }
if ( this.jobs.length === 0 ) { return; }
this.timer = globalThis.requestIdleCallback(deadline => {
this.timer = undefined;
globalThis.queueMicrotask(( ) => {
this.timer = undefined;
this.process();
});
this.busy = deadline.timeRemaining() === 0;
}, { timeout: 7 });
}
serialize(data, options) {
return new Promise(resolve => {
this.workload += 1;
this.jobs.push({ what: THREAD_SERIALIZE, data, options, size: 1, resolve });
this.processAsync();
});
}
deserialize(data, options) {
return new Promise(resolve => {
const size = data.length;
this.workload += size;
this.jobs.push({ what: THREAD_DESERIALIZE, data, options, size, resolve });
this.processAsync();
});
}
get queueSize() {
return this.jobs.length + 1;
}
get workSize() {
return this.busy ? Number.MAX_SAFE_INTEGER : this.workload * 2;
}
}
class Thread { class Thread {
constructor(gcer) { constructor(gcer) {
this.jobs = new Map(); this.jobs = new Map();
this.jobIdGenerator = 1; this.jobIdGenerator = 1;
this.workload = 0;
this.workerAccessTime = 0; this.workerAccessTime = 0;
this.workerTimer = undefined; this.workerTimer = undefined;
this.gcer = gcer; this.gcer = gcer;
@ -1151,7 +1222,7 @@ class Thread {
worker.onmessage = ev => { worker.onmessage = ev => {
const msg = ev.data; const msg = ev.data;
if ( isInstanceOf(msg, 'Object') === false ) { return; } if ( isInstanceOf(msg, 'Object') === false ) { return; }
if ( msg.what === 'ready!' ) { if ( msg.what === THREAD_IAMREADY ) {
worker.onmessage = ev => { this.onmessage(ev); }; worker.onmessage = ev => { this.onmessage(ev); };
worker.onerror = null; worker.onerror = null;
resolve(worker); resolve(worker);
@ -1161,7 +1232,10 @@ class Thread {
worker.onmessage = worker.onerror = null; worker.onmessage = worker.onerror = null;
resolve(null); resolve(null);
}; };
worker.postMessage({ what: 'ready?', config: currentConfig }); worker.postMessage({
what: THREAD_AREYOUREADY,
config: currentConfig,
});
} catch(ex) { } catch(ex) {
console.info(ex); console.info(ex);
worker.onmessage = worker.onerror = null; worker.onmessage = worker.onerror = null;
@ -1194,6 +1268,7 @@ class Thread {
if ( resolve === undefined ) { return; } if ( resolve === undefined ) { return; }
this.jobs.delete(job.id); this.jobs.delete(job.id);
resolve(job.result); resolve(job.result);
this.workload -= job.size;
if ( this.jobs.size !== 0 ) { return; } if ( this.jobs.size !== 0 ) { return; }
this.countdownWorker(); this.countdownWorker();
} }
@ -1208,7 +1283,8 @@ class Thread {
} }
const id = this.jobIdGenerator++; const id = this.jobIdGenerator++;
return new Promise(resolve => { return new Promise(resolve => {
const job = { what: 'serialize', id, data, options }; this.workload += 1;
const job = { what: THREAD_SERIALIZE, id, data, options, size: 1 };
this.jobs.set(job.id, resolve); this.jobs.set(job.id, resolve);
worker.postMessage(job); worker.postMessage(job);
}); });
@ -1224,25 +1300,36 @@ class Thread {
} }
const id = this.jobIdGenerator++; const id = this.jobIdGenerator++;
return new Promise(resolve => { return new Promise(resolve => {
const job = { what: 'deserialize', id, data, options }; const size = data.length;
this.workload += size;
const job = { what: THREAD_DESERIALIZE, id, data, options, size };
this.jobs.set(job.id, resolve); this.jobs.set(job.id, resolve);
worker.postMessage(job); worker.postMessage(job);
}); });
} }
get queueSize() {
return this.jobs.size;
}
get workSize() {
return this.workload;
}
} }
const threads = { const threads = {
pool: [], pool: [ new MainThread() ],
thread(maxPoolSize) { thread(maxPoolSize) {
for ( const thread of this.pool ) { const poolSize = this.pool.length;
if ( thread.jobs.size === 0 ) { return thread; } if ( poolSize !== 0 && poolSize >= maxPoolSize ) {
} if ( poolSize === 1 ) { return this.pool[0]; }
const len = this.pool.length; return this.pool.reduce((best, candidate) => {
if ( len !== 0 && len >= maxPoolSize ) { if ( candidate.queueSize === 0 ) { return candidate; }
if ( len === 1 ) { return this.pool[0]; } if ( best.queueSize === 0 ) { return best; }
return this.pool.reduce((best, candidate) => return candidate.workSize < best.workSize
candidate.jobs.size < best.jobs.size ? candidate : best ? candidate
); : best;
});
} }
const thread = new Thread(thread => { const thread = new Thread(thread => {
const pos = this.pool.indexOf(thread); const pos = this.pool.indexOf(thread);
@ -1267,6 +1354,7 @@ export async function serializeAsync(data, options = {}) {
} }
export async function deserializeAsync(data, options = {}) { export async function deserializeAsync(data, options = {}) {
if ( isSerialized(data) === false ) { return data; }
const maxThreadCount = options.multithreaded || 0; const maxThreadCount = options.multithreaded || 0;
if ( maxThreadCount === 0 ) { if ( maxThreadCount === 0 ) {
return deserialize(data, options); return deserialize(data, options);
@ -1288,16 +1376,17 @@ if ( isInstanceOf(globalThis, 'DedicatedWorkerGlobalScope') ) {
globalThis.onmessage = ev => { globalThis.onmessage = ev => {
const msg = ev.data; const msg = ev.data;
switch ( msg.what ) { switch ( msg.what ) {
case 'ready?': case THREAD_AREYOUREADY:
setConfig(msg.config); setConfig(msg.config);
globalThis.postMessage({ what: 'ready!' }); globalThis.postMessage({ what: THREAD_IAMREADY });
break; break;
case 'serialize': case THREAD_SERIALIZE:
case 'deserialize': { const result = serialize(msg.data, msg.options);
const result = msg.what === 'serialize' globalThis.postMessage({ id: msg.id, size: msg.size, result });
? serialize(msg.data, msg.options) break;
: deserialize(msg.data); case THREAD_DESERIALIZE: {
globalThis.postMessage({ id: msg.id, result }); const result = deserialize(msg.data);
globalThis.postMessage({ id: msg.id, size: msg.size, result });
break; break;
} }
} }