diff --git a/platform/chromium/webext.js b/platform/chromium/webext.js index 851b653e2..a0c24c228 100644 --- a/platform/chromium/webext.js +++ b/platform/chromium/webext.js @@ -156,6 +156,15 @@ if ( chrome.storage.sync instanceof Object ) { }; } +// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage/session +webext.storage.session = { + clear: ( ) => Promise.resolve(), + get: ( ) => Promise.resolve(), + getBytesInUse: ( ) => Promise.resolve(), + remove: ( ) => Promise.resolve(), + set: ( ) => Promise.resolve(), +}; + // https://bugs.chromium.org/p/chromium/issues/detail?id=608854 if ( chrome.tabs.removeCSS instanceof Function ) { webext.tabs.removeCSS = promisifyNoFail(chrome.tabs, 'removeCSS'); diff --git a/src/js/assets.js b/src/js/assets.js index 77e9b8167..749fb7802 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -838,7 +838,7 @@ async function assetCacheSetDetails(assetKey, details) { } } if ( modified ) { - saveAssetCacheRegistry(); + saveAssetCacheRegistry(3); } } diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js index e0fe98705..056cd8b68 100644 --- a/src/js/cachestorage.js +++ b/src/js/cachestorage.js @@ -58,17 +58,19 @@ const shouldCache = bin => { return out; }; -const missingKeys = (wanted, inbin, outbin) => { - inbin = inbin || {}; - const found = Object.keys(inbin); - Object.assign(outbin, inbin); - if ( found.length === wanted.length ) { return; } - const missing = []; - for ( const key of wanted ) { - if ( outbin.hasOwnProperty(key) ) { continue; } - missing.push(key); - } - return missing; +const exGet = (api, wanted, outbin) => { + return api.get(wanted).then(inbin => { + inbin = inbin || {}; + const found = Object.keys(inbin); + Object.assign(outbin, inbin); + if ( found.length === wanted.length ) { return; } + const missing = []; + for ( const key of wanted ) { + if ( outbin.hasOwnProperty(key) ) { continue; } + missing.push(key); + } + return missing; + }); }; /******************************************************************************* @@ -81,15 +83,15 @@ const missingKeys = (wanted, inbin, outbin) => { const cacheStorage = (( ) => { - const compress = async (key, data) => { + const compress = async (bin, key, data) => { const µbhs = µb.hiddenSettings; const isLarge = typeof data === 'string' && data.length >= µbhs.cacheStorageCompressionThreshold; const after = await scuo.serializeAsync(data, { compress: isLarge && µbhs.cacheStorageCompression, - multithreaded: isLarge && µbhs.cacheStorageMultithread || 2, + multithreaded: µbhs.cacheStorageMultithread, }); - return { key, data: after }; + bin[key] = after; }; const decompress = async (bin, key) => { @@ -98,27 +100,24 @@ const cacheStorage = (( ) => { const µbhs = µb.hiddenSettings; const isLarge = data.length >= µbhs.cacheStorageCompressionThreshold; bin[key] = await scuo.deserializeAsync(data, { - multithreaded: isLarge && µbhs.cacheStorageMultithread || 2, + multithreaded: isLarge && µbhs.cacheStorageMultithread || 1, }); }; return { get(argbin) { const outbin = {}; - const wanted0 = keysFromGetArg(argbin); - return cacheAPI.get(wanted0).then(bin => { - const wanted1 = missingKeys(wanted0, bin, outbin); - if ( wanted1 === undefined ) { return; } - return extensionStorage.get(wanted1).then(bin => { - const wanted2 = missingKeys(wanted1, bin, outbin); - if ( wanted2 === undefined ) { return; } - if ( argbin instanceof Object === false ) { return; } - if ( Array.isArray(argbin) ) { return; } - for ( const key of wanted2 ) { - if ( argbin.hasOwnProperty(key) === false ) { continue; } - outbin[key] = argbin[key]; - } - }); + return exGet(cacheAPI, keysFromGetArg(argbin), outbin).then(wanted => { + if ( wanted === undefined ) { return; } + return exGet(extensionStorage, wanted, outbin); + }).then(wanted => { + if ( wanted === undefined ) { return; } + if ( argbin instanceof Object === false ) { return; } + if ( Array.isArray(argbin) ) { return; } + for ( const key of wanted ) { + if ( argbin.hasOwnProperty(key) === false ) { continue; } + outbin[key] = argbin[key]; + } }).then(( ) => { const promises = []; for ( const key of Object.keys(outbin) ) { @@ -147,17 +146,14 @@ const cacheStorage = (( ) => { async set(keyvalStore) { const keys = Object.keys(keyvalStore); if ( keys.length === 0 ) { return; } + const bin = {}; const promises = []; for ( const key of keys ) { - promises.push(compress(key, keyvalStore[key])); + promises.push(compress(bin, key, keyvalStore[key])); } - const results = await Promise.all(promises); - const serializedStore = {}; - for ( const { key, data } of results ) { - serializedStore[key] = data; - } - cacheAPI.set(shouldCache(serializedStore)); - return extensionStorage.set(serializedStore).catch(reason => { + await Promise.all(promises); + cacheAPI.set(shouldCache(bin)); + return extensionStorage.set(bin).catch(reason => { ubolog(reason); }); }, @@ -361,6 +357,54 @@ const cacheAPI = (( ) => { }; })(); +/******************************************************************************* + * + * In-memory storage + * + * */ + +const memoryStorage = (( ) => { + + const sessionStorage = webext.storage.session; + + return { + get(...args) { + return sessionStorage.get(...args).catch(reason => { + ubolog(reason); + }); + }, + + async keys(regex) { + const results = await sessionStorage.get(null).catch(( ) => {}); + const keys = new Set(results[0]); + const bin = results[1] || {}; + for ( const key of Object.keys(bin) ) { + if ( regex && regex.test(key) === false ) { continue; } + keys.add(key); + } + return keys; + }, + + async set(...args) { + return sessionStorage.set(...args).catch(reason => { + ubolog(reason); + }); + }, + + remove(...args) { + return sessionStorage.remove(...args).catch(reason => { + ubolog(reason); + }); + }, + + clear(...args) { + return sessionStorage.clear(...args).catch(reason => { + ubolog(reason); + }); + }, + }; +})(); + /******************************************************************************* * * IndexedDB diff --git a/src/js/scuo-serializer.js b/src/js/scuo-serializer.js index f5f18fdd4..c0154fd9d 100644 --- a/src/js/scuo-serializer.js +++ b/src/js/scuo-serializer.js @@ -1148,10 +1148,11 @@ const THREAD_DESERIALIZE = 4; class MainThread { constructor() { + this.name = 'main'; this.jobs = []; this.workload = 0; this.timer = undefined; - this.busy = false; + this.busy = 2; } process() { @@ -1164,7 +1165,9 @@ class MainThread { job.resolve(result); this.processAsync(); if ( this.jobs.length === 0 ) { - this.busy = false; + this.busy = 2; + } else if ( this.busy > 2 ) { + this.busy -= 1; } } @@ -1174,11 +1177,12 @@ class MainThread { this.timer = globalThis.requestIdleCallback(deadline => { this.timer = undefined; globalThis.queueMicrotask(( ) => { - this.timer = undefined; this.process(); }); - this.busy = deadline.timeRemaining() === 0; - }, { timeout: 7 }); + if ( deadline.timeRemaining() === 0 ) { + this.busy += 1; + } + }, { timeout: 5 }); } serialize(data, options) { @@ -1199,16 +1203,17 @@ class MainThread { } get queueSize() { - return this.jobs.length + 1; + return this.jobs.length; } get workSize() { - return this.busy ? Number.MAX_SAFE_INTEGER : this.workload * 2; + return this.workload * this.busy; } } class Thread { constructor(gcer) { + this.name = 'worker'; this.jobs = new Map(); this.jobIdGenerator = 1; this.workload = 0; @@ -1263,7 +1268,10 @@ class Thread { } onmessage(ev) { - const job = ev.data; + this.ondone(ev.data); + } + + ondone(job) { const resolve = this.jobs.get(job.id); if ( resolve === undefined ) { return; } this.jobs.delete(job.id); @@ -1274,37 +1282,35 @@ class Thread { } async serialize(data, options) { - this.workerAccessTime = Date.now(); - const worker = await this.workerPromise; - if ( worker === null ) { - const result = serialize(data, options); - this.countdownWorker(); - return result; - } - const id = this.jobIdGenerator++; return new Promise(resolve => { + const id = this.jobIdGenerator++; this.workload += 1; - const job = { what: THREAD_SERIALIZE, id, data, options, size: 1 }; - this.jobs.set(job.id, resolve); - worker.postMessage(job); + this.jobs.set(id, resolve); + return this.workerPromise.then(worker => { + this.workerAccessTime = Date.now(); + if ( worker === null ) { + this.ondone({ id, result: serialize(data, options), size: 1 }); + } else { + worker.postMessage({ what: THREAD_SERIALIZE, id, data, options, size: 1 }); + } + }); }); } async deserialize(data, options) { - this.workerAccessTime = Date.now(); - const worker = await this.workerPromise; - if ( worker === null ) { - const result = deserialize(data, options); - this.countdownWorker(); - return result; - } - const id = this.jobIdGenerator++; return new Promise(resolve => { + const id = this.jobIdGenerator++; const size = data.length; this.workload += size; - const job = { what: THREAD_DESERIALIZE, id, data, options, size }; - this.jobs.set(job.id, resolve); - worker.postMessage(job); + this.jobs.set(id, resolve); + return this.workerPromise.then(worker => { + this.workerAccessTime = Date.now(); + if ( worker === null ) { + this.ondone({ id, result: deserialize(data, options), size }); + } else { + worker.postMessage({ what: THREAD_DESERIALIZE, id, data, options, size }); + } + }); }); } @@ -1323,12 +1329,11 @@ const threads = { const poolSize = this.pool.length; if ( poolSize !== 0 && poolSize >= maxPoolSize ) { if ( poolSize === 1 ) { return this.pool[0]; } - return this.pool.reduce((best, candidate) => { - if ( candidate.queueSize === 0 ) { return candidate; } - if ( best.queueSize === 0 ) { return best; } - return candidate.workSize < best.workSize - ? candidate - : best; + return this.pool.reduce((a, b) => { + //console.log(`${a.name}: q=${a.queueSize} w=${a.workSize} ${b.name}: q=${b.queueSize} w=${b.workSize}`); + if ( b.queueSize === 0 ) { return b; } + if ( a.queueSize === 0 ) { return a; } + return b.workSize < a.workSize ? b : a; }); } const thread = new Thread(thread => { @@ -1346,9 +1351,9 @@ export async function serializeAsync(data, options = {}) { if ( maxThreadCount === 0 ) { return serialize(data, options); } - const result = await threads - .thread(maxThreadCount) - .serialize(data, options); + const thread = threads.thread(maxThreadCount); + //console.log(`serializeAsync: thread=${thread.name} workload=${thread.workSize}`); + const result = await thread.serialize(data, options); if ( result !== undefined ) { return result; } return serialize(data, options); } @@ -1359,9 +1364,9 @@ export async function deserializeAsync(data, options = {}) { if ( maxThreadCount === 0 ) { return deserialize(data, options); } - const result = await threads - .thread(maxThreadCount) - .deserialize(data, options); + const thread = threads.thread(maxThreadCount); + //console.log(`deserializeAsync: thread=${thread.name} data=${data.length} workload=${thread.workSize}`); + const result = await thread.deserialize(data, options); if ( result !== undefined ) { return result; } return deserialize(data, options); }