diff --git a/platform/chromium/webext.js b/platform/chromium/webext.js index a0c24c228..851b653e2 100644 --- a/platform/chromium/webext.js +++ b/platform/chromium/webext.js @@ -156,15 +156,6 @@ if ( chrome.storage.sync instanceof Object ) { }; } -// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage/session -webext.storage.session = { - clear: ( ) => Promise.resolve(), - get: ( ) => Promise.resolve(), - getBytesInUse: ( ) => Promise.resolve(), - remove: ( ) => Promise.resolve(), - set: ( ) => Promise.resolve(), -}; - // https://bugs.chromium.org/p/chromium/issues/detail?id=608854 if ( chrome.tabs.removeCSS instanceof Function ) { webext.tabs.removeCSS = promisifyNoFail(chrome.tabs, 'removeCSS'); diff --git a/platform/common/vapi-background.js b/platform/common/vapi-background.js index 08cfd5872..83d04a9af 100644 --- a/platform/common/vapi-background.js +++ b/platform/common/vapi-background.js @@ -109,7 +109,7 @@ vAPI.generateSecret = (size = 1) => { * * */ -vAPI.sessionStorage = { +vAPI.sessionStorage = webext.storage.session || { get() { return Promise.resolve({}); }, @@ -122,7 +122,6 @@ vAPI.sessionStorage = { clear() { return Promise.resolve(); }, - implemented: false, }; /******************************************************************************* @@ -137,46 +136,21 @@ vAPI.sessionStorage = { vAPI.storage = { get(key, ...args) { - if ( vAPI.sessionStorage.implemented !== true ) { - return webext.storage.local.get(key, ...args).catch(reason => { - console.log(reason); - }); - } - return vAPI.sessionStorage.get(key, ...args).then(bin => { - const size = Object.keys(bin).length; - if ( size === 1 && typeof key === 'string' && bin[key] === null ) { - return {}; - } - if ( size !== 0 ) { return bin; } - return webext.storage.local.get(key, ...args).then(bin => { - if ( bin instanceof Object === false ) { return bin; } - // Mirror empty result as null value in order to prevent - // from falling back to storage.local when there is no need. - const tomirror = Object.assign({}, bin); - if ( typeof key === 'string' && Object.keys(bin).length === 0 ) { - Object.assign(tomirror, { [key]: null }); - } - vAPI.sessionStorage.set(tomirror); - return bin; - }).catch(reason => { - console.log(reason); - }); + return webext.storage.local.get(key, ...args).catch(reason => { + console.log(reason); }); }, set(...args) { - vAPI.sessionStorage.set(...args); return webext.storage.local.set(...args).catch(reason => { console.log(reason); }); }, remove(...args) { - vAPI.sessionStorage.remove(...args); return webext.storage.local.remove(...args).catch(reason => { console.log(reason); }); }, clear(...args) { - vAPI.sessionStorage.clear(...args); return webext.storage.local.clear(...args).catch(reason => { console.log(reason); }); diff --git a/src/js/background.js b/src/js/background.js index 123fe5778..79cae6eb1 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -146,7 +146,7 @@ if ( vAPI.webextFlavor.soup.has('firefox') ) { } const µBlock = { // jshint ignore:line - wakeupReason: '', + alarmQueue: [], userSettingsDefault, userSettings: Object.assign({}, userSettingsDefault), diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js index 056cd8b68..5dcaffbec 100644 --- a/src/js/cachestorage.js +++ b/src/js/cachestorage.js @@ -45,34 +45,6 @@ const keysFromGetArg = arg => { return Object.keys(arg); }; -// Cache API is subject to quota so we will use it only for what is key -// performance-wise -const shouldCache = bin => { - const out = {}; - for ( const key of Object.keys(bin) ) { - if ( key.startsWith('cache/') ) { - if ( /^cache\/(compiled|selfie)\//.test(key) === false ) { continue; } - } - out[key] = bin[key]; - } - return out; -}; - -const exGet = (api, wanted, outbin) => { - return api.get(wanted).then(inbin => { - inbin = inbin || {}; - const found = Object.keys(inbin); - Object.assign(outbin, inbin); - if ( found.length === wanted.length ) { return; } - const missing = []; - for ( const key of wanted ) { - if ( outbin.hasOwnProperty(key) ) { continue; } - missing.push(key); - } - return missing; - }); -}; - /******************************************************************************* * * Extension storage @@ -83,12 +55,26 @@ const exGet = (api, wanted, outbin) => { const cacheStorage = (( ) => { + const exGet = (api, wanted, outbin) => { + return api.get(wanted).then(inbin => { + inbin = inbin || {}; + const found = Object.keys(inbin); + Object.assign(outbin, inbin); + if ( found.length === wanted.length ) { return; } + const missing = []; + for ( const key of wanted ) { + if ( outbin.hasOwnProperty(key) ) { continue; } + missing.push(key); + } + return missing; + }); + }; + const compress = async (bin, key, data) => { const µbhs = µb.hiddenSettings; - const isLarge = typeof data === 'string' && - data.length >= µbhs.cacheStorageCompressionThreshold; const after = await scuo.serializeAsync(data, { - compress: isLarge && µbhs.cacheStorageCompression, + compress: µbhs.cacheStorageCompression, + compressThreshold: µbhs.cacheStorageCompressionThreshold, multithreaded: µbhs.cacheStorageMultithread, }); bin[key] = after; @@ -104,7 +90,7 @@ const cacheStorage = (( ) => { }); }; - return { + const api = { get(argbin) { const outbin = {}; return exGet(cacheAPI, keysFromGetArg(argbin), outbin).then(wanted => { @@ -152,7 +138,8 @@ const cacheStorage = (( ) => { promises.push(compress(bin, key, keyvalStore[key])); } await Promise.all(promises); - cacheAPI.set(shouldCache(bin)); + memoryStorage.set(bin); + cacheAPI.set(bin); return extensionStorage.set(bin).catch(reason => { ubolog(reason); }); @@ -191,16 +178,18 @@ const cacheStorage = (( ) => { return Promise.all(toMigrate); }, }; -})(); -// Not all platforms support getBytesInUse -if ( extensionStorage.getBytesInUse instanceof Function ) { - cacheStorage.getBytesInUse = function(...args) { - return extensionStorage.getBytesInUse(...args).catch(reason => { - ubolog(reason); - }); - }; -} + // Not all platforms support getBytesInUse + if ( extensionStorage.getBytesInUse instanceof Function ) { + api.getBytesInUse = function(...args) { + return extensionStorage.getBytesInUse(...args).catch(reason => { + ubolog(reason); + }); + }; + } + + return api; +})(); /******************************************************************************* * @@ -234,6 +223,19 @@ const cacheAPI = (( ) => { const urlToKey = url => decodeURIComponent(url.slice(urlPrefix.length)); + // Cache API is subject to quota so we will use it only for what is key + // performance-wise + const shouldCache = bin => { + const out = {}; + for ( const key of Object.keys(bin) ) { + if ( key.startsWith('cache/' ) ) { + if ( /^cache\/(compiled|selfie)\//.test(key) === false ) { continue; } + } + out[key] = bin[key]; + } + if ( Object.keys(out).length !== 0 ) { return out; } + }; + const getOne = async key => { const cache = await cacheStoragePromise; if ( cache === null ) { return; } @@ -327,12 +329,13 @@ const cacheAPI = (( ) => { ).catch(( ) => []); }, - set(keyvalStore) { - const keys = Object.keys(keyvalStore); - if ( keys.length === 0 ) { return; } + async set(...args) { + const bin = shouldCache(...args); + if ( bin === undefined ) { return; } + const keys = Object.keys(bin); const promises = []; for ( const key of keys ) { - promises.push(setOne(key, keyvalStore[key])); + promises.push(setOne(key, bin[key])); } return Promise.all(promises); }, @@ -363,30 +366,46 @@ const cacheAPI = (( ) => { * * */ -const memoryStorage = (( ) => { +const memoryStorage = (( ) => { // jshint ignore:line - const sessionStorage = webext.storage.session; + const sessionStorage = vAPI.sessionStorage; + + // This should help speed up loading from suspended state in Firefox for + // Android. + // 20240228 Observation: Slows down loading from suspended state in + // Firefox desktop. Could be different in Firefox for Android. + const shouldCache = bin => { + const out = {}; + for ( const key of Object.keys(bin) ) { + if ( key.startsWith('cache/compiled/') ) { continue; } + out[key] = bin[key]; + } + if ( Object.keys(out).length !== 0 ) { return out; } + }; return { get(...args) { - return sessionStorage.get(...args).catch(reason => { + return sessionStorage.get(...args).then(bin => { + return bin; + }).catch(reason => { ubolog(reason); }); }, async keys(regex) { - const results = await sessionStorage.get(null).catch(( ) => {}); - const keys = new Set(results[0]); - const bin = results[1] || {}; - for ( const key of Object.keys(bin) ) { + const bin = await sessionStorage.get(null).catch(( ) => {}); + const keys = []; + for ( const key of Object.keys(bin || {}) ) { if ( regex && regex.test(key) === false ) { continue; } - keys.add(key); + keys.push(key); } return keys; }, async set(...args) { - return sessionStorage.set(...args).catch(reason => { + const bin = shouldCache(...args); + if ( bin === undefined ) { return; } + return sessionStorage.set(bin).catch(reason => { ubolog(reason); }); }, @@ -522,7 +541,7 @@ const idbStorage = (( ) => { if ( entry.value instanceof Blob === false ) { return; } promises.push(decompress(keyvalStore, key, value)); }).catch(reason => { - ubolog(`cacheStorage.getAllFromDb() failed: ${reason}`); + ubolog(`idbStorage.getAllFromDb() failed: ${reason}`); callback(); }); }; diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 7d70e35ee..1edb37624 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -72,7 +72,7 @@ const warSecret = typeof vAPI === 'object' && vAPI !== null : ( ) => ''; const RESOURCES_SELFIE_VERSION = 7; -const RESOURCES_SELFIE_NAME = 'compiled/redirectEngine/resources'; +const RESOURCES_SELFIE_NAME = 'selfie/redirectEngine/resources'; /******************************************************************************/ /******************************************************************************/ diff --git a/src/js/scuo-serializer.js b/src/js/scuo-serializer.js index c0154fd9d..81b688547 100644 --- a/src/js/scuo-serializer.js +++ b/src/js/scuo-serializer.js @@ -281,6 +281,12 @@ const isInstanceOf = (o, s) => { ); }; +const shouldCompress = (s, options) => + options.compress === true && ( + options.compressThreshold === undefined || + options.compressThreshold <= s.length + ); + /******************************************************************************* * * A large Uint is always a positive integer (can be zero), assumed to be @@ -1051,10 +1057,9 @@ export const serialize = (data, options = {}) => { const s = writeBuffer.join(''); writeRefs.clear(); writeBuffer.length = 0; - if ( options.compress !== true ) { return s; } + if ( shouldCompress(s, options) === false ) { return s; } const lz4Util = new LZ4BlockJS(); - const encoder = new TextEncoder(); - const uint8ArrayBefore = encoder.encode(s); + const uint8ArrayBefore = textEncoder.encode(s); const uint8ArrayAfter = lz4Util.encode(uint8ArrayBefore, 0); const lz4 = { size: uint8ArrayBefore.length, @@ -1145,7 +1150,6 @@ const THREAD_IAMREADY = 2; const THREAD_SERIALIZE = 3; const THREAD_DESERIALIZE = 4; - class MainThread { constructor() { this.name = 'main'; diff --git a/src/js/start.js b/src/js/start.js index 877d909c4..caa27dfbf 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -76,6 +76,10 @@ vAPI.app.onShutdown = ( ) => { permanentSwitches.reset(); }; +vAPI.alarms.onAlarm.addListener(alarm => { + µb.alarmQueue.push(alarm.name); +}); + /******************************************************************************/ // This is called only once, when everything has been loaded in memory after @@ -160,6 +164,13 @@ const onVersionReady = async lastVersion => { await cacheStorage.migrate(µb.hiddenSettings.cacheStorageAPI); } + // Remove cache items with obsolete names + if ( lastVersionInt < vAPI.app.intFromVersion('1.56.1b5') ) { + io.remove(`compiled/${µb.pslAssetKey}`); + io.remove('compiled/redirectEngine/resources'); + io.remove('selfie/main'); + } + // Since built-in resources may have changed since last version, we // force a reload of all resources. redirectEngine.invalidateResourcesSelfie(io); @@ -436,7 +447,7 @@ let selfieIsValid = false; try { selfieIsValid = await µb.selfieManager.load(); if ( selfieIsValid === true ) { - ubolog(`Selfie ready ${Date.now()-vAPI.T0} ms after launch`); + ubolog(`Loaded filtering engine from selfie ${Date.now()-vAPI.T0} ms after launch`); } } catch (ex) { console.trace(ex); @@ -506,5 +517,16 @@ ubolog(`All ready ${µb.supportStats.allReadyAfter} after launch`); µb.isReadyResolve(); +// Process alarm queue +while ( µb.alarmQueue.length !== 0 ) { + const what = µb.alarmQueue.shift(); + ubolog(`Processing alarm event from suspended state: '${what}'`); + switch ( what ) { + case 'createSelfie': + µb.selfieManager.create(); + break; + } +} + // <<<<< end of async/await scope })(); diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 86d042248..01afc3a1b 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -4629,6 +4629,7 @@ FilterContainer.prototype.optimize = function(throttle = 0) { /******************************************************************************/ FilterContainer.prototype.toSelfie = function() { + this.optimize(0); bidiTrieOptimize(true); keyvalStore.setItem('SNFE.origHNTrieContainer.trieDetails', origHNTrieContainer.optimize() diff --git a/src/js/storage.js b/src/js/storage.js index 99116547e..7c29f1422 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1147,7 +1147,10 @@ onBroadcast(msg => { µb.loadRedirectResources = async function() { try { const success = await redirectEngine.resourcesFromSelfie(io); - if ( success === true ) { return true; } + if ( success === true ) { + ubolog('Loaded redirect/scriptlets resources from selfie'); + return true; + } const fetcher = (path, options = undefined) => { if ( path.startsWith('/web_accessible_resources/') ) { @@ -1220,8 +1223,11 @@ onBroadcast(msg => { } try { - const selfie = await io.fromCache(`compiled/${this.pslAssetKey}`); - if ( psl.fromSelfie(selfie) ) { return; } + const selfie = await io.fromCache(`selfie/${this.pslAssetKey}`); + if ( psl.fromSelfie(selfie) ) { + ubolog('Loaded PSL from selfie'); + return; + } } catch (reason) { ubolog(reason); } @@ -1235,7 +1241,8 @@ onBroadcast(msg => { µb.compilePublicSuffixList = function(content) { const psl = publicSuffixList; psl.parse(content, punycode.toASCII); - return io.toCache(`compiled/${this.pslAssetKey}`, psl.toSelfie()); + ubolog(`Loaded PSL from ${this.pslAssetKey}`); + return io.toCache(`selfie/${this.pslAssetKey}`, psl.toSelfie()); }; /******************************************************************************/ @@ -1255,7 +1262,7 @@ onBroadcast(msg => { if ( µb.inMemoryFilters.length !== 0 ) { return; } if ( Object.keys(µb.availableFilterLists).length === 0 ) { return; } await Promise.all([ - io.toCache('selfie/main', { + io.toCache('selfie/staticMain', { magic: µb.systemSettings.selfieMagic, availableFilterLists: µb.availableFilterLists, }), @@ -1268,11 +1275,11 @@ onBroadcast(msg => { ]); lz4Codec.relinquish(); µb.selfieIsInvalid = false; - ubolog(`Selfie was created`); + ubolog('Filtering engine selfie created'); }; const loadMain = async function() { - const selfie = await io.fromCache('selfie/main'); + const selfie = await io.fromCache('selfie/staticMain'); if ( selfie instanceof Object === false ) { return false; } if ( selfie.magic !== µb.systemSettings.selfieMagic ) { return false; } if ( selfie.availableFilterLists instanceof Object === false ) { return false; } @@ -1300,35 +1307,26 @@ onBroadcast(msg => { catch (reason) { ubolog(reason); } - ubolog('Selfie not available'); + ubolog('Filtering engine selfie not available'); destroy(); return false; }; const destroy = function(options = {}) { if ( µb.selfieIsInvalid === false ) { - io.remove(/^selfie\//, options); + io.remove(/^selfie\/static/, options); µb.selfieIsInvalid = true; - ubolog('Selfie marked for invalidation'); - } - if ( µb.wakeupReason === 'createSelfie' ) { - µb.wakeupReason = ''; - return createTimer.offon({ sec: 27 }); + ubolog('Filtering engine selfie marked for invalidation'); } vAPI.alarms.create('createSelfie', { - delayInMinutes: µb.hiddenSettings.selfieAfter + delayInMinutes: µb.hiddenSettings.selfieAfter + 0.5 }); createTimer.offon({ min: µb.hiddenSettings.selfieAfter }); }; const createTimer = vAPI.defer.create(create); - vAPI.alarms.onAlarm.addListener(alarm => { - if ( alarm.name !== 'createSelfie') { return; } - µb.wakeupReason = 'createSelfie'; - }); - - µb.selfieManager = { load, destroy }; + µb.selfieManager = { load, create, destroy }; } /******************************************************************************/