Mirrors/uBlock
1
0
Fork 0
mirror of https://github.com/gorhill/uBlock.git synced 2024-11-22 10:22:51 +01:00
Code Issues Releases Wiki Activity

Refactor selfie generation into a more flexible persistence mechanism

Browse Source 
The motivation is to address the higher peak memory usage at launch
time with 3rd-gen HNTrie when a selfie was present.

The selfie generation prior to this change was to collect all
filtering data into a single data structure, and then to serialize
that whole structure at once into storage (using JSON.stringify).

However, HNTrie serialization requires that a large UintArray32 be
converted into a plain JS array, which itslef would be indirectly
converted into a JSON string. This was the main reason why peak
memory usage would be higher at launch from selfie, since the JSON
string would need to be wholly unserialized into JS objects, which
themselves would need to be converted into more specialized data
structures (like that Uint32Array one).

The solution to lower peak memory usage at launch is to refactor
selfie generation to allow a more piecemeal approach: each filtering
component is given the ability to serialize itself rather than to be
forced to be embedded in the master selfie. With this approach, the
HNTrie buffer can now serialize to its own storage by converting the
buffer data directly into a string which can be directly sent to
storage. This avoiding expensive intermediate steps such as
converting into a JS array and then to a JSON string.

As part of the refactoring, there was also opportunistic code
upgrade to ES6 and Promise (eventually all of uBO's code will be
proper ES6).

Additionally, the polyfill to bring getBytesInUse() to Firefox has
been revisited to replace the rather expensive previous
implementation with an implementation with virtually no overhead.
This commit is contained in:
Raymond Hill 2019-02-14 13:33:55 -05:00
parent 83a3767a16
commit ed7e34fb07
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
14 changed files with 588 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.jshintrc
View File

@ -7,6 +7,7 @@
"browser": false, // global variable in Firefox, Edge "browser": false, // global variable in Firefox, Edge
"chrome": false, // global variable in Chromium, Chrome, Opera "chrome": false, // global variable in Chromium, Chrome, Opera
"Components": false, // global variable in Firefox "Components": false, // global variable in Firefox
"log": false,
"safari": false, "safari": false,
"self": false, "self": false,
"vAPI": false, "vAPI": false,

4
platform/chromium/vapi-common.js
View File

@ -30,6 +30,10 @@
/******************************************************************************/ /******************************************************************************/
vAPI.T0 = Date.now();
/******************************************************************************/
vAPI.setTimeout = vAPI.setTimeout || self.setTimeout.bind(self); vAPI.setTimeout = vAPI.setTimeout || self.setTimeout.bind(self);
/******************************************************************************/ /******************************************************************************/

1
src/background.html
View File

@ -5,6 +5,7 @@
<title>uBlock Origin</title> <title>uBlock Origin</title>
</head> </head>
<body> <body>
<script src="js/console.js"></script>
<script src="lib/lz4/lz4-block-codec-any.js"></script> <script src="lib/lz4/lz4-block-codec-any.js"></script>
<script src="lib/punycode.js"></script> <script src="lib/punycode.js"></script>
<script src="lib/publicsuffixlist/publicsuffixlist.js"></script> <script src="lib/publicsuffixlist/publicsuffixlist.js"></script>

125
src/js/assets.js
View File

@ -449,10 +449,9 @@ const assetCacheRegistryStartTime = Date.now();
let assetCacheRegistryPromise; let assetCacheRegistryPromise;
let assetCacheRegistry = {}; let assetCacheRegistry = {};
const getAssetCacheRegistry = function(callback) { const getAssetCacheRegistry = function() {
if ( assetCacheRegistryPromise === undefined ) { if ( assetCacheRegistryPromise === undefined ) {
assetCacheRegistryPromise = new Promise(resolve => { assetCacheRegistryPromise = new Promise(resolve => {
// start of executor
µBlock.cacheStorage.get('assetCacheRegistry', bin => { µBlock.cacheStorage.get('assetCacheRegistry', bin => {
if ( if (
bin instanceof Object && bin instanceof Object &&
@ -462,13 +461,10 @@ const getAssetCacheRegistry = function(callback) {
} }
resolve(); resolve();
}); });
// end of executor
}); });
} }
assetCacheRegistryPromise.then(( ) => { return assetCacheRegistryPromise.then(( ) => assetCacheRegistry);
callback(assetCacheRegistry);
});
}; };
const saveAssetCacheRegistry = (function() { const saveAssetCacheRegistry = (function() {
@ -513,11 +509,9 @@ const assetCacheRead = function(assetKey, callback) {
reportBack(bin[internalKey]); reportBack(bin[internalKey]);
}; };
let onReady = function() { getAssetCacheRegistry().then(( ) => {
µBlock.cacheStorage.get(internalKey, onAssetRead); µBlock.cacheStorage.get(internalKey, onAssetRead);
}; });
getAssetCacheRegistry(onReady);
}; };
const assetCacheWrite = function(assetKey, details, callback) { const assetCacheWrite = function(assetKey, details, callback) {
@ -542,7 +536,18 @@ const assetCacheWrite = function(assetKey, details, callback) {
if ( details instanceof Object && typeof details.url === 'string' ) { if ( details instanceof Object && typeof details.url === 'string' ) {
entry.remoteURL = details.url; entry.remoteURL = details.url;
} }
µBlock.cacheStorage.set({ assetCacheRegistry, [internalKey]: content }); µBlock.cacheStorage.set(
{ [internalKey]: content },
details => {
if (
details instanceof Object &&
typeof details.bytesInUse === 'number'
) {
entry.byteLength = details.bytesInUse;
}
saveAssetCacheRegistry(true);
}
);
const result = { assetKey, content }; const result = { assetKey, content };
if ( typeof callback === 'function' ) { if ( typeof callback === 'function' ) {
callback(result); callback(result);
@ -550,14 +555,16 @@ const assetCacheWrite = function(assetKey, details, callback) {
// https://github.com/uBlockOrigin/uBlock-issues/issues/248 // https://github.com/uBlockOrigin/uBlock-issues/issues/248
fireNotification('after-asset-updated', result); fireNotification('after-asset-updated', result);
}; };
getAssetCacheRegistry(onReady);
getAssetCacheRegistry().then(( ) => {
µBlock.cacheStorage.get(internalKey, onReady);
});
}; };
const assetCacheRemove = function(pattern, callback) { const assetCacheRemove = function(pattern, callback) {
const onReady = function() { getAssetCacheRegistry().then(cacheDict => {
const cacheDict = assetCacheRegistry, const removedEntries = [];
removedEntries = [], const removedContent = [];
removedContent = [];
for ( const assetKey in cacheDict ) { for ( const assetKey in cacheDict ) {
if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
continue; continue;
@ -582,14 +589,15 @@ const assetCacheRemove = function(pattern, callback) {
{ assetKey: removedEntries[i] } { assetKey: removedEntries[i] }
); );
} }
}; });
getAssetCacheRegistry(onReady);
}; };
const assetCacheMarkAsDirty = function(pattern, exclude, callback) { const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
const onReady = function() { if ( typeof exclude === 'function' ) {
const cacheDict = assetCacheRegistry; callback = exclude;
exclude = undefined;
}
getAssetCacheRegistry().then(cacheDict => {
let mustSave = false; let mustSave = false;
for ( const assetKey in cacheDict ) { for ( const assetKey in cacheDict ) {
if ( pattern instanceof RegExp ) { if ( pattern instanceof RegExp ) {
@ -617,12 +625,7 @@ const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
if ( typeof callback === 'function' ) { if ( typeof callback === 'function' ) {
callback(); callback();
} }
}; });
if ( typeof exclude === 'function' ) {
callback = exclude;
exclude = undefined;
}
getAssetCacheRegistry(onReady);
}; };
/******************************************************************************/ /******************************************************************************/
@ -642,12 +645,12 @@ const stringIsNotEmpty = function(s) {
**/ **/
var readUserAsset = function(assetKey, callback) { const readUserAsset = function(assetKey, callback) {
var reportBack = function(content) { const reportBack = function(content) {
callback({ assetKey: assetKey, content: content }); callback({ assetKey: assetKey, content: content });
}; };
var onLoaded = function(bin) { const onLoaded = function(bin) {
if ( !bin ) { return reportBack(''); } if ( !bin ) { return reportBack(''); }
var content = ''; var content = '';
if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) { if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
@ -671,7 +674,7 @@ var readUserAsset = function(assetKey, callback) {
} }
return reportBack(content); return reportBack(content);
}; };
var toRead = assetKey; let toRead = assetKey;
if ( assetKey === µBlock.userFiltersPath ) { if ( assetKey === µBlock.userFiltersPath ) {
toRead = [ toRead = [
assetKey, assetKey,
@ -682,7 +685,7 @@ var readUserAsset = function(assetKey, callback) {
vAPI.storage.get(toRead, onLoaded); vAPI.storage.get(toRead, onLoaded);
}; };
var saveUserAsset = function(assetKey, content, callback) { const saveUserAsset = function(assetKey, content, callback) {
var bin = {}; var bin = {};
bin[assetKey] = content; bin[assetKey] = content;
// TODO(seamless migration): // TODO(seamless migration):
@ -711,27 +714,33 @@ api.get = function(assetKey, options, callback) {
callback = noopfunc; callback = noopfunc;
} }
return new Promise(resolve => {
// start of executor
if ( assetKey === µBlock.userFiltersPath ) { if ( assetKey === µBlock.userFiltersPath ) {
readUserAsset(assetKey, callback); readUserAsset(assetKey, details => {
callback(details);
resolve(details);
});
return; return;
} }
var assetDetails = {}, let assetDetails = {},
contentURLs, contentURLs,
contentURL; contentURL;
var reportBack = function(content, err) { const reportBack = function(content, err) {
var details = { assetKey: assetKey, content: content }; const details = { assetKey: assetKey, content: content };
if ( err ) { if ( err ) {
details.error = assetDetails.lastError = err; details.error = assetDetails.lastError = err;
} else { } else {
assetDetails.lastError = undefined; assetDetails.lastError = undefined;
} }
callback(details); callback(details);
resolve(details);
}; };
var onContentNotLoaded = function() { const onContentNotLoaded = function() {
var isExternal; let isExternal;
while ( (contentURL = contentURLs.shift()) ) { while ( (contentURL = contentURLs.shift()) ) {
isExternal = reIsExternalPath.test(contentURL); isExternal = reIsExternalPath.test(contentURL);
if ( isExternal === false || assetDetails.hasLocalURL !== true ) { if ( isExternal === false || assetDetails.hasLocalURL !== true ) {
@ -748,7 +757,7 @@ api.get = function(assetKey, options, callback) {
} }
}; };
var onContentLoaded = function(details) { const onContentLoaded = function(details) {
if ( stringIsNotEmpty(details.content) === false ) { if ( stringIsNotEmpty(details.content) === false ) {
onContentNotLoaded(); onContentNotLoaded();
return; return;
@ -762,7 +771,7 @@ api.get = function(assetKey, options, callback) {
reportBack(details.content); reportBack(details.content);
}; };
var onCachedContentLoaded = function(details) { const onCachedContentLoaded = function(details) {
if ( details.content !== '' ) { if ( details.content !== '' ) {
return reportBack(details.content); return reportBack(details.content);
} }
@ -780,11 +789,13 @@ api.get = function(assetKey, options, callback) {
}; };
assetCacheRead(assetKey, onCachedContentLoaded); assetCacheRead(assetKey, onCachedContentLoaded);
// end of executor
});
}; };
/******************************************************************************/ /******************************************************************************/
var getRemote = function(assetKey, callback) { const getRemote = function(assetKey, callback) {
var assetDetails = {}, var assetDetails = {},
contentURLs, contentURLs,
contentURL; contentURL;
@ -852,10 +863,19 @@ var getRemote = function(assetKey, callback) {
/******************************************************************************/ /******************************************************************************/
api.put = function(assetKey, content, callback) { api.put = function(assetKey, content, callback) {
if ( reIsUserAsset.test(assetKey) ) { return new Promise(resolve => {
return saveUserAsset(assetKey, content, callback); const onDone = function(details) {
if ( typeof callback === 'function' ) {
callback(details);
} }
assetCacheWrite(assetKey, content, callback); resolve(details);
};
if ( reIsUserAsset.test(assetKey) ) {
saveUserAsset(assetKey, content, onDone);
} else {
assetCacheWrite(assetKey, content, onDone);
}
});
}; };
/******************************************************************************/ /******************************************************************************/
@ -895,7 +915,7 @@ api.metadata = function(callback) {
if ( cacheRegistryReady ) { onReady(); } if ( cacheRegistryReady ) { onReady(); }
}); });
getAssetCacheRegistry(function() { getAssetCacheRegistry().then(( ) => {
cacheRegistryReady = true; cacheRegistryReady = true;
if ( assetRegistryReady ) { onReady(); } if ( assetRegistryReady ) { onReady(); }
}); });
@ -903,6 +923,19 @@ api.metadata = function(callback) {
/******************************************************************************/ /******************************************************************************/
api.getBytesInUse = function() {
return getAssetCacheRegistry().then(cacheDict => {
let bytesUsed = 0;
for ( const assetKey in cacheDict ) {
if ( cacheDict.hasOwnProperty(assetKey) === false ) { continue; }
bytesUsed += cacheDict[assetKey].byteLength || 0;
}
return bytesUsed;
});
};
/******************************************************************************/
api.purge = assetCacheMarkAsDirty; api.purge = assetCacheMarkAsDirty;
api.remove = function(pattern, callback) { api.remove = function(pattern, callback) {
@ -1013,7 +1046,7 @@ var updateNext = function() {
updateOne(); updateOne();
}); });
getAssetCacheRegistry(function(dict) { getAssetCacheRegistry().then(dict => {
cacheDict = dict; cacheDict = dict;
if ( !assetDict ) { return; } if ( !assetDict ) { return; }
updateOne(); updateOne();

14
src/js/background.js
View File

@ -46,6 +46,7 @@ const µBlock = (function() { // jshint ignore:line
cacheStorageAPI: 'unset', cacheStorageAPI: 'unset',
cacheStorageCompression: true, cacheStorageCompression: true,
cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
consoleLogLevel: 'unset',
debugScriptlets: false, debugScriptlets: false,
disableWebAssembly: false, disableWebAssembly: false,
ignoreRedirectFilters: false, ignoreRedirectFilters: false,
@ -53,6 +54,7 @@ const µBlock = (function() { // jshint ignore:line
manualUpdateAssetFetchPeriod: 500, manualUpdateAssetFetchPeriod: 500,
popupFontSize: 'unset', popupFontSize: 'unset',
requestJournalProcessPeriod: 1000, requestJournalProcessPeriod: 1000,
selfieAfter: 11,
strictBlockingBypassDuration: 120, strictBlockingBypassDuration: 120,
suspendTabsUntilReady: false, suspendTabsUntilReady: false,
userResourcesLocation: 'unset' userResourcesLocation: 'unset'
@ -95,13 +97,13 @@ const µBlock = (function() { // jshint ignore:line
hiddenSettingsDefault: hiddenSettingsDefault, hiddenSettingsDefault: hiddenSettingsDefault,
hiddenSettings: (function() { hiddenSettings: (function() {
let out = Object.assign({}, hiddenSettingsDefault), const out = Object.assign({}, hiddenSettingsDefault),
json = vAPI.localStorage.getItem('immediateHiddenSettings'); json = vAPI.localStorage.getItem('immediateHiddenSettings');
if ( typeof json === 'string' ) { if ( typeof json === 'string' ) {
try { try {
let o = JSON.parse(json); const o = JSON.parse(json);
if ( o instanceof Object ) { if ( o instanceof Object ) {
for ( let k in o ) { for ( const k in o ) {
if ( out.hasOwnProperty(k) ) { if ( out.hasOwnProperty(k) ) {
out[k] = o[k]; out[k] = o[k];
} }
@ -111,8 +113,6 @@ const µBlock = (function() { // jshint ignore:line
catch(ex) { catch(ex) {
} }
} }
// Remove once 1.15.12+ is widespread.
vAPI.localStorage.removeItem('hiddenSettings');
return out; return out;
})(), })(),
@ -138,7 +138,7 @@ const µBlock = (function() { // jshint ignore:line
// Read-only // Read-only
systemSettings: { systemSettings: {
compiledMagic: 6, // Increase when compiled format changes compiledMagic: 6, // Increase when compiled format changes
selfieMagic: 7 // Increase when selfie format changes selfieMagic: 8 // Increase when selfie format changes
}, },
restoreBackupSettings: { restoreBackupSettings: {
@ -161,8 +161,6 @@ const µBlock = (function() { // jshint ignore:line
selectedFilterLists: [], selectedFilterLists: [],
availableFilterLists: {}, availableFilterLists: {},
selfieAfter: 17 * oneMinute,
pageStores: new Map(), pageStores: new Map(),
pageStoresToken: 0, pageStoresToken: 0,

36
src/js/cachestorage.js
View File

@ -326,17 +326,27 @@
if ( typeof callback !== 'function' ) { if ( typeof callback !== 'function' ) {
callback = noopfn; callback = noopfn;
} }
let keys = Object.keys(keyvalStore); const keys = Object.keys(keyvalStore);
if ( keys.length === 0 ) { return callback(); } if ( keys.length === 0 ) { return callback(); }
let promises = [ getDb() ]; const promises = [ getDb() ];
let entries = []; const entries = [];
let dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true; const dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true;
let handleEncodingResult = result => { let bytesInUse = 0;
const handleEncodingResult = result => {
if ( typeof result.data === 'string' ) {
bytesInUse += result.data.length;
} else if ( result.data instanceof Blob ) {
bytesInUse += result.data.size;
}
entries.push({ key: result.key, value: result.data }); entries.push({ key: result.key, value: result.data });
}; };
for ( let key of keys ) { for ( const key of keys ) {
let data = keyvalStore[key]; const data = keyvalStore[key];
if ( typeof data !== 'string' || dontCompress ) { const isString = typeof data === 'string';
if ( isString === false || dontCompress ) {
if ( isString ) {
bytesInUse += data.length;
}
entries.push({ key, value: data }); entries.push({ key, value: data });
continue; continue;
} }
@ -346,20 +356,20 @@
} }
Promise.all(promises).then(( ) => { Promise.all(promises).then(( ) => {
if ( !db ) { return callback(); } if ( !db ) { return callback(); }
let finish = ( ) => { const finish = ( ) => {
dbBytesInUse = undefined; dbBytesInUse = undefined;
if ( callback === undefined ) { return; } if ( callback === undefined ) { return; }
let cb = callback; let cb = callback;
callback = undefined; callback = undefined;
cb(); cb({ bytesInUse });
}; };
try { try {
let transaction = db.transaction(STORAGE_NAME, 'readwrite'); const transaction = db.transaction(STORAGE_NAME, 'readwrite');
transaction.oncomplete = transaction.oncomplete =
transaction.onerror = transaction.onerror =
transaction.onabort = finish; transaction.onabort = finish;
let table = transaction.objectStore(STORAGE_NAME); const table = transaction.objectStore(STORAGE_NAME);
for ( let entry of entries ) { for ( const entry of entries ) {
table.put(entry); table.put(entry);
} }
} catch (ex) { } catch (ex) {

34
src/js/console.js Normal file
View File

@ -0,0 +1,34 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2019-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
'use strict';
self.log = (function() {
const noopFunc = function() {};
const info = function(s) { console.log(`[uBO] ${s}`); };
return {
get verbosity( ) { return; },
set verbosity(level) {
this.info = console.info = level === 'info' ? info : noopFunc;
},
info,
};
})();

30
src/js/hntrie.js
View File

@ -355,7 +355,13 @@ HNTrieContainer.prototype = {
return trieRef; return trieRef;
}, },
serialize: function() { serialize: function(encoder) {
if ( encoder instanceof Object ) {
return encoder.encode(
this.buf32.buffer,
this.buf32[HNTRIE_CHAR1_SLOT]
);
}
return Array.from( return Array.from(
new Uint32Array( new Uint32Array(
this.buf32.buffer, this.buf32.buffer,
@ -365,23 +371,29 @@ HNTrieContainer.prototype = {
); );
}, },
unserialize: function(selfie) { unserialize: function(selfie, decoder) {
const len = (selfie.length << 2) + HNTRIE_PAGE_SIZE-1 & ~(HNTRIE_PAGE_SIZE-1); const shouldDecode = typeof selfie === 'string';
let byteLength = shouldDecode
? decoder.decodeSize(selfie)
: selfie.length << 2;
byteLength = byteLength + HNTRIE_PAGE_SIZE-1 & ~(HNTRIE_PAGE_SIZE-1);
if ( this.wasmMemory !== null ) { if ( this.wasmMemory !== null ) {
const pageCountBefore = this.buf.length >>> 16; const pageCountBefore = this.buf.length >>> 16;
const pageCountAfter = len >>> 16; const pageCountAfter = byteLength >>> 16;
if ( pageCountAfter > pageCountBefore ) { if ( pageCountAfter > pageCountBefore ) {
this.wasmMemory.grow(pageCountAfter - pageCountBefore); this.wasmMemory.grow(pageCountAfter - pageCountBefore);
this.buf = new Uint8Array(this.wasmMemory.buffer); this.buf = new Uint8Array(this.wasmMemory.buffer);
this.buf32 = new Uint32Array(this.buf.buffer); this.buf32 = new Uint32Array(this.buf.buffer);
} }
} else { } else if ( byteLength > this.buf.length ) {
if ( len > this.buf.length ) { this.buf = new Uint8Array(byteLength);
this.buf = new Uint8Array(len);
this.buf32 = new Uint32Array(this.buf.buffer); this.buf32 = new Uint32Array(this.buf.buffer);
} }
} if ( shouldDecode ) {
decoder.decode(selfie, this.buf.buffer);
} else {
this.buf32.set(selfie); this.buf32.set(selfie);
}
this.needle = ''; this.needle = '';
}, },
@ -684,6 +696,6 @@ HNTrieContainer.prototype.HNTrieRef.prototype = {
WebAssembly.compileStreaming WebAssembly.compileStreaming
).catch(reason => { ).catch(reason => {
HNTrieContainer.wasmModulePromise = null; HNTrieContainer.wasmModulePromise = null;
console.info(reason); log.info(reason);
}); });
})(); })();

87
src/js/redirect-engine.js
View File

@ -29,12 +29,12 @@
/******************************************************************************/ /******************************************************************************/
const warResolve = (function() { const warResolve = (function() {
var warPairs = []; let warPairs = [];
var onPairsReady = function() { const onPairsReady = function() {
var reng = µBlock.redirectEngine; const reng = µBlock.redirectEngine;
for ( var i = 0; i < warPairs.length; i += 2 ) { for ( let i = 0; i < warPairs.length; i += 2 ) {
var resource = reng.resources.get(warPairs[i+0]); const resource = reng.resources.get(warPairs[i+0]);
if ( resource === undefined ) { continue; } if ( resource === undefined ) { continue; }
resource.warURL = vAPI.getURL( resource.warURL = vAPI.getURL(
'/web_accessible_resources/' + warPairs[i+1] '/web_accessible_resources/' + warPairs[i+1]
@ -48,15 +48,15 @@ const warResolve = (function() {
return onPairsReady(); return onPairsReady();
} }
var onPairsLoaded = function(details) { const onPairsLoaded = function(details) {
var marker = '>>>>>'; const marker = '>>>>>';
var pos = details.content.indexOf(marker); const pos = details.content.indexOf(marker);
if ( pos === -1 ) { return; } if ( pos === -1 ) { return; }
var pairs = details.content.slice(pos + marker.length) const pairs = details.content.slice(pos + marker.length)
.trim() .trim()
.split('\n'); .split('\n');
if ( (pairs.length & 1) !== 0 ) { return; } if ( (pairs.length & 1) !== 0 ) { return; }
for ( var i = 0; i < pairs.length; i++ ) { for ( let i = 0; i < pairs.length; i++ ) {
pairs[i] = pairs[i].trim(); pairs[i] = pairs[i].trim();
} }
warPairs = pairs; warPairs = pairs;
@ -64,7 +64,7 @@ const warResolve = (function() {
}; };
µBlock.assets.fetchText( µBlock.assets.fetchText(
'/web_accessible_resources/imported.txt?secret=' + vAPI.warSecret, `/web_accessible_resources/imported.txt?secret=${vAPI.warSecret}`,
onPairsLoaded onPairsLoaded
); );
}; };
@ -374,18 +374,17 @@ RedirectEngine.prototype.supportedTypes = new Map([
/******************************************************************************/ /******************************************************************************/
RedirectEngine.prototype.toSelfie = function() { RedirectEngine.prototype.toSelfie = function(path) {
// Because rules may contains RegExp instances, we need to manually // Because rules may contains RegExp instances, we need to manually
// convert it to a serializable format. The serialized format must be // convert it to a serializable format. The serialized format must be
// suitable to be used as an argument to the Map() constructor. // suitable to be used as an argument to the Map() constructor.
var rules = [], const rules = [];
rule, entries, i, entry; for ( const item of this.rules ) {
for ( var item of this.rules ) { const rule = [ item[0], [] ];
rule = [ item[0], [] ]; const entries = item[1];
entries = item[1]; let i = entries.length;
i = entries.length;
while ( i-- ) { while ( i-- ) {
entry = entries[i]; const entry = entries[i];
rule[1].push({ rule[1].push({
tok: entry.tok, tok: entry.tok,
pat: entry.pat instanceof RegExp ? entry.pat.source : entry.pat pat: entry.pat instanceof RegExp ? entry.pat.source : entry.pat
@ -393,23 +392,34 @@ RedirectEngine.prototype.toSelfie = function() {
} }
rules.push(rule); rules.push(rule);
} }
return { return µBlock.assets.put(
`${path}/main`,
JSON.stringify({
rules: rules, rules: rules,
ruleTypes: Array.from(this.ruleTypes), ruleTypes: Array.from(this.ruleTypes),
ruleSources: Array.from(this.ruleSources), ruleSources: Array.from(this.ruleSources),
ruleDestinations: Array.from(this.ruleDestinations) ruleDestinations: Array.from(this.ruleDestinations)
}; })
);
}; };
/******************************************************************************/ /******************************************************************************/
RedirectEngine.prototype.fromSelfie = function(selfie) { RedirectEngine.prototype.fromSelfie = function(path) {
return µBlock.assets.get(`${path}/main`).then(details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
this.rules = new Map(selfie.rules); this.rules = new Map(selfie.rules);
this.ruleTypes = new Set(selfie.ruleTypes); this.ruleTypes = new Set(selfie.ruleTypes);
this.ruleSources = new Set(selfie.ruleSources); this.ruleSources = new Set(selfie.ruleSources);
this.ruleDestinations = new Set(selfie.ruleDestinations); this.ruleDestinations = new Set(selfie.ruleDestinations);
this.modifyTime = Date.now(); this.modifyTime = Date.now();
return true; return true;
});
}; };
/******************************************************************************/ /******************************************************************************/
@ -494,41 +504,46 @@ RedirectEngine.prototype.resourcesFromString = function(text) {
/******************************************************************************/ /******************************************************************************/
let resourcesSelfieVersion = 3; const resourcesSelfieVersion = 3;
RedirectEngine.prototype.selfieFromResources = function() { RedirectEngine.prototype.selfieFromResources = function() {
let selfie = { µBlock.assets.put(
'compiled/redirectEngine/resources',
JSON.stringify({
version: resourcesSelfieVersion, version: resourcesSelfieVersion,
resources: Array.from(this.resources) resources: Array.from(this.resources)
}; })
µBlock.cacheStorage.set({ resourcesSelfie: JSON.stringify(selfie) }); );
}; };
RedirectEngine.prototype.resourcesFromSelfie = function(callback) { RedirectEngine.prototype.resourcesFromSelfie = function() {
µBlock.cacheStorage.get('resourcesSelfie', bin => { return µBlock.assets.get(
let selfie = bin && bin.resourcesSelfie; 'compiled/redirectEngine/resources'
if ( typeof selfie === 'string' ) { ).then(details => {
let selfie;
try { try {
selfie = JSON.parse(selfie); selfie = JSON.parse(details.content);
} catch(ex) { } catch(ex) {
} }
}
if ( if (
selfie instanceof Object === false || selfie instanceof Object === false ||
selfie.version !== resourcesSelfieVersion || selfie.version !== resourcesSelfieVersion ||
Array.isArray(selfie.resources) === false Array.isArray(selfie.resources) === false
) { ) {
return callback(false); return false;
} }
this.resources = new Map(); this.resources = new Map();
for ( let entry of selfie.resources ) { for ( const [ token, entry ] of selfie.resources ) {
this.resources.set(entry[0], RedirectEntry.fromSelfie(entry[1])); this.resources.set(token, RedirectEntry.fromSelfie(entry));
} }
callback(true); return true;
}); });
}; };
RedirectEngine.prototype.invalidateResourcesSelfie = function() { RedirectEngine.prototype.invalidateResourcesSelfie = function() {
µBlock.assets.remove('compiled/redirectEngine/resources');
// TODO: obsolete, remove eventually
µBlock.cacheStorage.remove('resourcesSelfie'); µBlock.cacheStorage.remove('resourcesSelfie');
}; };

73
src/js/start.js
View File

@ -81,6 +81,8 @@ var onAllReady = function() {
µb.contextMenu.update(null); µb.contextMenu.update(null);
µb.firstInstall = false; µb.firstInstall = false;
log.info(`All ready ${Date.now()-vAPI.T0} ms after launch`);
}; };
/******************************************************************************/ /******************************************************************************/
@ -137,22 +139,29 @@ let initializeTabs = function() {
// Filtering engines dependencies: // Filtering engines dependencies:
// - PSL // - PSL
var onPSLReady = function() { const onPSLReady = function() {
µb.selfieManager.load(function(valid) { log.info(`PSL ready ${Date.now()-vAPI.T0} ms after launch`);
µb.selfieManager.load().then(valid => {
if ( valid === true ) { if ( valid === true ) {
return onAllReady(); log.info(`Selfie ready ${Date.now()-vAPI.T0} ms after launch`);
onAllReady();
return;
} }
µb.loadFilterLists(onAllReady); µb.loadFilterLists(( ) => {
log.info(`Filter lists ready ${Date.now()-vAPI.T0} ms after launch`);
onAllReady();
});
}); });
}; };
/******************************************************************************/ /******************************************************************************/
var onCommandShortcutsReady = function(commandShortcuts) { const onCommandShortcutsReady = function(commandShortcuts) {
if ( Array.isArray(commandShortcuts) === false ) { return; } if ( Array.isArray(commandShortcuts) === false ) { return; }
µb.commandShortcuts = new Map(commandShortcuts); µb.commandShortcuts = new Map(commandShortcuts);
if ( µb.canUpdateShortcuts === false ) { return; } if ( µb.canUpdateShortcuts === false ) { return; }
for ( let entry of commandShortcuts ) { for ( const entry of commandShortcuts ) {
vAPI.commands.update({ name: entry[0], shortcut: entry[1] }); vAPI.commands.update({ name: entry[0], shortcut: entry[1] });
} }
}; };
@ -161,7 +170,7 @@ var onCommandShortcutsReady = function(commandShortcuts) {
// To bring older versions up to date // To bring older versions up to date
var onVersionReady = function(lastVersion) { const onVersionReady = function(lastVersion) {
if ( lastVersion === vAPI.app.version ) { return; } if ( lastVersion === vAPI.app.version ) { return; }
// Since AMO does not allow updating resources.txt, force a reload when a // Since AMO does not allow updating resources.txt, force a reload when a
@ -176,7 +185,7 @@ var onVersionReady = function(lastVersion) {
// If unused, just comment out for when we need to compare versions in the // If unused, just comment out for when we need to compare versions in the
// future. // future.
let intFromVersion = function(s) { const intFromVersion = function(s) {
let parts = s.match(/(?:^|\.|b|rc)\d+/g); let parts = s.match(/(?:^|\.|b|rc)\d+/g);
if ( parts === null ) { return 0; } if ( parts === null ) { return 0; }
let vint = 0; let vint = 0;
@ -223,7 +232,7 @@ var onVersionReady = function(lastVersion) {
// Whitelist parser needs PSL to be ready. // Whitelist parser needs PSL to be ready.
// gorhill 2014-12-15: not anymore // gorhill 2014-12-15: not anymore
var onNetWhitelistReady = function(netWhitelistRaw) { const onNetWhitelistReady = function(netWhitelistRaw) {
µb.netWhitelist = µb.whitelistFromString(netWhitelistRaw); µb.netWhitelist = µb.whitelistFromString(netWhitelistRaw);
µb.netWhitelistModifyTime = Date.now(); µb.netWhitelistModifyTime = Date.now();
}; };
@ -232,8 +241,10 @@ var onNetWhitelistReady = function(netWhitelistRaw) {
// User settings are in memory // User settings are in memory
var onUserSettingsReady = function(fetched) { const onUserSettingsReady = function(fetched) {
var userSettings = µb.userSettings; log.info(`User settings ready ${Date.now()-vAPI.T0} ms after launch`);
const userSettings = µb.userSettings;
fromFetch(userSettings, fetched); fromFetch(userSettings, fetched);
@ -264,7 +275,7 @@ var onUserSettingsReady = function(fetched) {
// Housekeeping, as per system setting changes // Housekeeping, as per system setting changes
var onSystemSettingsReady = function(fetched) { const onSystemSettingsReady = function(fetched) {
var mustSaveSystemSettings = false; var mustSaveSystemSettings = false;
if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) {
µb.assets.remove(/^compiled\//); µb.assets.remove(/^compiled\//);
@ -282,7 +293,9 @@ var onSystemSettingsReady = function(fetched) {
/******************************************************************************/ /******************************************************************************/
var onFirstFetchReady = function(fetched) { const onFirstFetchReady = function(fetched) {
log.info(`First fetch ready ${Date.now()-vAPI.T0} ms after launch`);
// https://github.com/gorhill/uBlock/issues/747 // https://github.com/gorhill/uBlock/issues/747
µb.firstInstall = fetched.version === '0.0.0.0'; µb.firstInstall = fetched.version === '0.0.0.0';
@ -295,10 +308,7 @@ var onFirstFetchReady = function(fetched) {
onVersionReady(fetched.version); onVersionReady(fetched.version);
onCommandShortcutsReady(fetched.commandShortcuts); onCommandShortcutsReady(fetched.commandShortcuts);
Promise.all([ µb.loadPublicSuffixList().then(( ) => {
µb.loadPublicSuffixList(),
µb.staticNetFilteringEngine.readyToUse()
]).then(( ) => {
onPSLReady(); onPSLReady();
}); });
µb.loadRedirectResources(); µb.loadRedirectResources();
@ -306,31 +316,27 @@ var onFirstFetchReady = function(fetched) {
/******************************************************************************/ /******************************************************************************/
var toFetch = function(from, fetched) { const toFetch = function(from, fetched) {
for ( var k in from ) { for ( const k in from ) {
if ( from.hasOwnProperty(k) === false ) { if ( from.hasOwnProperty(k) === false ) { continue; }
continue;
}
fetched[k] = from[k]; fetched[k] = from[k];
} }
}; };
var fromFetch = function(to, fetched) { const fromFetch = function(to, fetched) {
for ( var k in to ) { for ( const k in to ) {
if ( to.hasOwnProperty(k) === false ) { if ( to.hasOwnProperty(k) === false ) { continue; }
continue; if ( fetched.hasOwnProperty(k) === false ) { continue; }
}
if ( fetched.hasOwnProperty(k) === false ) {
continue;
}
to[k] = fetched[k]; to[k] = fetched[k];
} }
}; };
/******************************************************************************/ /******************************************************************************/
var onSelectedFilterListsLoaded = function() { const onSelectedFilterListsLoaded = function() {
var fetchableProps = { log.info(`List selection ready ${Date.now()-vAPI.T0} ms after launch`);
const fetchableProps = {
'commandShortcuts': [], 'commandShortcuts': [],
'compiledMagic': 0, 'compiledMagic': 0,
'dynamicFilteringString': [ 'dynamicFilteringString': [
@ -371,7 +377,8 @@ var onSelectedFilterListsLoaded = function() {
// compatibility, this means a special asynchronous call to load selected // compatibility, this means a special asynchronous call to load selected
// filter lists. // filter lists.
var onAdminSettingsRestored = function() { const onAdminSettingsRestored = function() {
log.info(`Admin settings ready ${Date.now()-vAPI.T0} ms after launch`);
µb.loadSelectedFilterLists(onSelectedFilterListsLoaded); µb.loadSelectedFilterLists(onSelectedFilterListsLoaded);
}; };

20
src/js/static-ext-filtering.js
View File

@ -821,18 +821,30 @@
µb.htmlFilteringEngine.fromCompiledContent(reader, options); µb.htmlFilteringEngine.fromCompiledContent(reader, options);
}; };
api.toSelfie = function() { api.toSelfie = function(path) {
return { return µBlock.assets.put(
`${path}/main`,
JSON.stringify({
cosmetic: µb.cosmeticFilteringEngine.toSelfie(), cosmetic: µb.cosmeticFilteringEngine.toSelfie(),
scriptlets: µb.scriptletFilteringEngine.toSelfie(), scriptlets: µb.scriptletFilteringEngine.toSelfie(),
html: µb.htmlFilteringEngine.toSelfie() html: µb.htmlFilteringEngine.toSelfie()
}; })
);
}; };
api.fromSelfie = function(selfie) { api.fromSelfie = function(path) {
return µBlock.assets.get(`${path}/main`).then(details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic);
µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets);
µb.htmlFilteringEngine.fromSelfie(selfie.html); µb.htmlFilteringEngine.fromSelfie(selfie.html);
return true;
});
}; };
return api; return api;

77
src/js/static-net-filtering.js
View File

@ -2105,21 +2105,21 @@ FilterContainer.prototype.readyToUse = function() {
/******************************************************************************/ /******************************************************************************/
FilterContainer.prototype.toSelfie = function() { FilterContainer.prototype.toSelfie = function(path) {
let categoriesToSelfie = function(categoryMap) { const categoriesToSelfie = function(categoryMap) {
let selfie = []; const selfie = [];
for ( let categoryEntry of categoryMap ) { for ( const [ catbits, bucket ] of categoryMap ) {
let tokenEntries = []; const tokenEntries = [];
for ( let tokenEntry of categoryEntry[1] ) { for ( const [ token, filter ] of bucket ) {
tokenEntries.push([ tokenEntry[0], tokenEntry[1].compile() ]); tokenEntries.push([ token, filter.compile() ]);
} }
selfie.push([ categoryEntry[0], tokenEntries ]); selfie.push([ catbits, tokenEntries ]);
} }
return selfie; return selfie;
}; };
let dataFiltersToSelfie = function(dataFilters) { const dataFiltersToSelfie = function(dataFilters) {
let selfie = []; const selfie = [];
for ( let entry of dataFilters.values() ) { for ( let entry of dataFilters.values() ) {
do { do {
selfie.push(entry.compile()); selfie.push(entry.compile());
@ -2129,22 +2129,45 @@ FilterContainer.prototype.toSelfie = function() {
return selfie; return selfie;
}; };
return { return Promise.all([
µBlock.assets.put(
`${path}/trieContainer`,
FilterHostnameDict.trieContainer.serialize(µBlock.base128)
),
µBlock.assets.put(
`${path}/main`,
JSON.stringify({
processedFilterCount: this.processedFilterCount, processedFilterCount: this.processedFilterCount,
acceptedCount: this.acceptedCount, acceptedCount: this.acceptedCount,
rejectedCount: this.rejectedCount, rejectedCount: this.rejectedCount,
allowFilterCount: this.allowFilterCount, allowFilterCount: this.allowFilterCount,
blockFilterCount: this.blockFilterCount, blockFilterCount: this.blockFilterCount,
discardedCount: this.discardedCount, discardedCount: this.discardedCount,
trieContainer: FilterHostnameDict.trieContainer.serialize(),
categories: categoriesToSelfie(this.categories), categories: categoriesToSelfie(this.categories),
dataFilters: dataFiltersToSelfie(this.dataFilters) dataFilters: dataFiltersToSelfie(this.dataFilters),
}; })
)
]);
}; };
/******************************************************************************/ /******************************************************************************/
FilterContainer.prototype.fromSelfie = function(selfie) { FilterContainer.prototype.fromSelfie = function(path) {
return Promise.all([
µBlock.assets.get(`${path}/trieContainer`).then(details => {
FilterHostnameDict.trieContainer.unserialize(
details.content,
µBlock.base128
);
return true;
}),
µBlock.assets.get(`${path}/main`).then(details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
this.frozen = true; this.frozen = true;
this.processedFilterCount = selfie.processedFilterCount; this.processedFilterCount = selfie.processedFilterCount;
this.acceptedCount = selfie.acceptedCount; this.acceptedCount = selfie.acceptedCount;
@ -2152,24 +2175,26 @@ FilterContainer.prototype.fromSelfie = function(selfie) {
this.allowFilterCount = selfie.allowFilterCount; this.allowFilterCount = selfie.allowFilterCount;
this.blockFilterCount = selfie.blockFilterCount; this.blockFilterCount = selfie.blockFilterCount;
this.discardedCount = selfie.discardedCount; this.discardedCount = selfie.discardedCount;
FilterHostnameDict.trieContainer.unserialize(selfie.trieContainer); for ( const [ catbits, bucket ] of selfie.categories ) {
const tokenMap = new Map();
for ( let categoryEntry of selfie.categories ) { for ( const [ token, fdata ] of bucket ) {
let tokenMap = new Map(); tokenMap.set(token, filterFromCompiledData(fdata));
for ( let tokenEntry of categoryEntry[1] ) {
tokenMap.set(tokenEntry[0], filterFromCompiledData(tokenEntry[1]));
} }
this.categories.set(categoryEntry[0], tokenMap); this.categories.set(catbits, tokenMap);
} }
for ( const dataEntry of selfie.dataFilters ) {
for ( let dataEntry of selfie.dataFilters ) { const entry = FilterDataHolderEntry.load(dataEntry);
let entry = FilterDataHolderEntry.load(dataEntry); const bucket = this.dataFilters.get(entry.tokenHash);
let bucket = this.dataFilters.get(entry.tokenHash);
if ( bucket !== undefined ) { if ( bucket !== undefined ) {
entry.next = bucket; entry.next = bucket;
} }
this.dataFilters.set(entry.tokenHash, entry); this.dataFilters.set(entry.tokenHash, entry);
} }
return true;
}),
]).then(results =>
results.reduce((acc, v) => acc && v, true)
);
}; };
/******************************************************************************/ /******************************************************************************/

149
src/js/storage.js
View File

@ -32,7 +32,7 @@
let bytesInUse; let bytesInUse;
let countdown = 0; let countdown = 0;
let process = count => { const process = count => {
if ( typeof count === 'number' ) { if ( typeof count === 'number' ) {
if ( bytesInUse === undefined ) { if ( bytesInUse === undefined ) {
bytesInUse = 0; bytesInUse = 0;
@ -50,12 +50,11 @@
countdown += 1; countdown += 1;
vAPI.storage.getBytesInUse(null, process); vAPI.storage.getBytesInUse(null, process);
} }
if ( if ( this.cacheStorage !== vAPI.storage ) {
this.cacheStorage !== vAPI.storage &&
this.cacheStorage.getBytesInUse instanceof Function
) {
countdown += 1; countdown += 1;
this.cacheStorage.getBytesInUse(null, process); this.assets.getBytesInUse().then(count => {
process(count);
});
} }
if ( countdown === 0 ) { if ( countdown === 0 ) {
callback(); callback();
@ -94,10 +93,10 @@
µBlock.loadHiddenSettings = function() { µBlock.loadHiddenSettings = function() {
vAPI.storage.get('hiddenSettings', bin => { vAPI.storage.get('hiddenSettings', bin => {
if ( bin instanceof Object === false ) { return; } if ( bin instanceof Object === false ) { return; }
let hs = bin.hiddenSettings; const hs = bin.hiddenSettings;
if ( hs instanceof Object ) { if ( hs instanceof Object ) {
let hsDefault = this.hiddenSettingsDefault; const hsDefault = this.hiddenSettingsDefault;
for ( let key in hsDefault ) { for ( const key in hsDefault ) {
if ( if (
hsDefault.hasOwnProperty(key) && hsDefault.hasOwnProperty(key) &&
hs.hasOwnProperty(key) && hs.hasOwnProperty(key) &&
@ -110,6 +109,7 @@
if ( vAPI.localStorage.getItem('immediateHiddenSettings') === null ) { if ( vAPI.localStorage.getItem('immediateHiddenSettings') === null ) {
this.saveImmediateHiddenSettings(); this.saveImmediateHiddenSettings();
} }
self.log.verbosity = this.hiddenSettings.consoleLogLevel;
}); });
}; };
@ -118,8 +118,8 @@
// which were not modified by the user. // which were not modified by the user.
µBlock.saveHiddenSettings = function(callback) { µBlock.saveHiddenSettings = function(callback) {
let bin = { hiddenSettings: {} }; const bin = { hiddenSettings: {} };
for ( let prop in this.hiddenSettings ) { for ( const prop in this.hiddenSettings ) {
if ( if (
this.hiddenSettings.hasOwnProperty(prop) && this.hiddenSettings.hasOwnProperty(prop) &&
this.hiddenSettings[prop] !== this.hiddenSettingsDefault[prop] this.hiddenSettings[prop] !== this.hiddenSettingsDefault[prop]
@ -129,6 +129,7 @@
} }
vAPI.storage.set(bin, callback); vAPI.storage.set(bin, callback);
this.saveImmediateHiddenSettings(); this.saveImmediateHiddenSettings();
self.log.verbosity = this.hiddenSettings.consoleLogLevel;
}; };
/******************************************************************************/ /******************************************************************************/
@ -969,41 +970,41 @@
/******************************************************************************/ /******************************************************************************/
µBlock.loadRedirectResources = function(updatedContent) { µBlock.loadRedirectResources = function(updatedContent) {
var µb = this, let content = '';
content = '';
var onDone = function() { const onDone = ( ) => {
µb.redirectEngine.resourcesFromString(content); this.redirectEngine.resourcesFromString(content);
}; };
var onUserResourcesLoaded = function(details) { const onUserResourcesLoaded = details => {
if ( details.content !== '' ) { if ( details.content !== '' ) {
content += '\n\n' + details.content; content += '\n\n' + details.content;
} }
onDone(); onDone();
}; };
var onResourcesLoaded = function(details) { const onResourcesLoaded = details => {
if ( details.content !== '' ) { if ( details.content !== '' ) {
content = details.content; content = details.content;
} }
if ( µb.hiddenSettings.userResourcesLocation === 'unset' ) { if ( this.hiddenSettings.userResourcesLocation === 'unset' ) {
return onDone(); return onDone();
} }
µb.assets.fetchText(µb.hiddenSettings.userResourcesLocation, onUserResourcesLoaded); this.assets.fetchText(
this.hiddenSettings.userResourcesLocation,
onUserResourcesLoaded
);
}; };
if ( typeof updatedContent === 'string' && updatedContent.length !== 0 ) { if ( typeof updatedContent === 'string' && updatedContent.length !== 0 ) {
return onResourcesLoaded({ content: updatedContent }); return onResourcesLoaded({ content: updatedContent });
} }
var onSelfieReady = function(success) { this.redirectEngine.resourcesFromSelfie().then(success => {
if ( success !== true ) { if ( success !== true ) {
µb.assets.get('ublock-resources', onResourcesLoaded); this.assets.get('ublock-resources', onResourcesLoaded);
} }
}; });
µb.redirectEngine.resourcesFromSelfie(onSelfieReady);
}; };
/******************************************************************************/ /******************************************************************************/
@ -1013,39 +1014,25 @@
publicSuffixList.enableWASM(); publicSuffixList.enableWASM();
} }
return new Promise(resolve => { return this.assets.get(
// start of executor 'compiled/' + this.pslAssetKey
this.assets.get('compiled/' + this.pslAssetKey, details => { ).then(details =>
let selfie; publicSuffixList.fromSelfie(details.content, µBlock.base128)
try { ).then(valid => {
selfie = JSON.parse(details.content); if ( valid === true ) { return; }
} catch (ex) { return this.assets.get(this.pslAssetKey, details => {
}
if (
selfie instanceof Object &&
publicSuffixList.fromSelfie(selfie)
) {
resolve();
return;
}
this.assets.get(this.pslAssetKey, details => {
if ( details.content !== '' ) { if ( details.content !== '' ) {
this.compilePublicSuffixList(details.content); this.compilePublicSuffixList(details.content);
} }
resolve();
}); });
}); });
// end of executor
});
}; };
/******************************************************************************/
µBlock.compilePublicSuffixList = function(content) { µBlock.compilePublicSuffixList = function(content) {
publicSuffixList.parse(content, punycode.toASCII); publicSuffixList.parse(content, punycode.toASCII);
this.assets.put( this.assets.put(
'compiled/' + this.pslAssetKey, 'compiled/' + this.pslAssetKey,
JSON.stringify(publicSuffixList.toSelfie()) publicSuffixList.toSelfie(µBlock.base128)
); );
}; };
@ -1056,60 +1043,76 @@
// some set time. // some set time.
µBlock.selfieManager = (function() { µBlock.selfieManager = (function() {
let µb = µBlock; const µb = µBlock;
let timer = null; let timer;
// As of 2018-05-31: // As of 2018-05-31:
// JSON.stringify-ing ourselves results in a better baseline // JSON.stringify-ing ourselves results in a better baseline
// memory usage at selfie-load time. For some reasons. // memory usage at selfie-load time. For some reasons.
let create = function() { const create = function() {
timer = null; Promise.all([
let selfie = JSON.stringify({ µb.assets.put(
'selfie/main',
JSON.stringify({
magic: µb.systemSettings.selfieMagic, magic: µb.systemSettings.selfieMagic,
availableFilterLists: µb.availableFilterLists, availableFilterLists: µb.availableFilterLists,
staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), })
redirectEngine: µb.redirectEngine.toSelfie(), ),
staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie() µb.redirectEngine.toSelfie('selfie/redirectEngine'),
}); µb.staticExtFilteringEngine.toSelfie('selfie/staticExtFilteringEngine'),
µb.cacheStorage.set({ selfie: selfie }); µb.staticNetFilteringEngine.toSelfie('selfie/staticNetFilteringEngine'),
]).then(( ) => {
µb.lz4Codec.relinquish(); µb.lz4Codec.relinquish();
});
}; };
let load = function(callback) { const load = function() {
µb.cacheStorage.get('selfie', function(bin) { return Promise.all([
µb.assets.get('selfie/main').then(details => {
if ( if (
bin instanceof Object === false || details instanceof Object === false ||
typeof bin.selfie !== 'string' typeof details.content !== 'string' ||
details.content === ''
) { ) {
return callback(false); return false;
} }
let selfie; let selfie;
try { try {
selfie = JSON.parse(bin.selfie); selfie = JSON.parse(details.content);
} catch(ex) { } catch(ex) {
} }
if ( if (
selfie instanceof Object === false || selfie instanceof Object === false ||
selfie.magic !== µb.systemSettings.selfieMagic selfie.magic !== µb.systemSettings.selfieMagic
) { ) {
return callback(false); return false;
} }
µb.availableFilterLists = selfie.availableFilterLists; µb.availableFilterLists = selfie.availableFilterLists;
µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); return true;
µb.redirectEngine.fromSelfie(selfie.redirectEngine); }),
µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine); µb.redirectEngine.fromSelfie('selfie/redirectEngine'),
callback(true); µb.staticExtFilteringEngine.fromSelfie('selfie/staticExtFilteringEngine'),
µb.staticNetFilteringEngine.fromSelfie('selfie/staticNetFilteringEngine'),
]).then(results =>
results.reduce((acc, v) => acc && v, true)
).catch(reason => {
log.info(reason);
return false;
}); });
}; };
let destroy = function() { const destroy = function() {
if ( timer !== null ) { if ( timer !== undefined ) {
clearTimeout(timer); clearTimeout(timer);
timer = null; timer = undefined;
} }
µb.cacheStorage.remove('selfie'); µb.cacheStorage.remove('selfie'); // TODO: obsolete, remove eventually.
timer = vAPI.setTimeout(create, µb.selfieAfter); µb.assets.remove(/^selfie\//);
timer = vAPI.setTimeout(( ) => {
timer = undefined;
create();
}, µb.hiddenSettings.selfieAfter * 60000);
}; };
return { return {
@ -1299,6 +1302,8 @@
// Compile the list while we have the raw version in memory // Compile the list while we have the raw version in memory
if ( topic === 'after-asset-updated' ) { if ( topic === 'after-asset-updated' ) {
// Skip selfie-related content.
if ( details.assetKey.startsWith('selfie/') ) { return; }
var cached = typeof details.content === 'string' && details.content !== ''; var cached = typeof details.content === 'string' && details.content !== '';
if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) {
if ( cached ) { if ( cached ) {

109
src/js/utils.js
View File

@ -496,3 +496,112 @@
µBlock.orphanizeString = function(s) { µBlock.orphanizeString = function(s) {
return JSON.parse(JSON.stringify(s)); return JSON.parse(JSON.stringify(s));
}; };
/******************************************************************************/
// Custom base128 encoder/decoder
//
// TODO:
// Could expand the LZ4 codec API to be able to return UTF8-safe string
// representation of a compressed buffer, and thus the code below could be
// moved LZ4 codec-side.
µBlock.base128 = {
encode: function(arrbuf, arrlen) {
const inbuf = new Uint8Array(arrbuf, 0, arrlen);
const inputLength = arrlen;
let _7cnt = Math.floor(inputLength / 7);
let outputLength = _7cnt * 8;
let _7rem = inputLength % 7;
if ( _7rem !== 0 ) {
outputLength += 1 + _7rem;
}
const outbuf = new Uint8Array(outputLength);
let msbits, v;
let i = 0, j = 0;
while ( _7cnt-- ) {
v = inbuf[i+0];
msbits = (v & 0x80) >>> 7;
outbuf[j+1] = v & 0x7F;
v = inbuf[i+1];
msbits |= (v & 0x80) >>> 6;
outbuf[j+2] = v & 0x7F;
v = inbuf[i+2];
msbits |= (v & 0x80) >>> 5;
outbuf[j+3] = v & 0x7F;
v = inbuf[i+3];
msbits |= (v & 0x80) >>> 4;
outbuf[j+4] = v & 0x7F;
v = inbuf[i+4];
msbits |= (v & 0x80) >>> 3;
outbuf[j+5] = v & 0x7F;
v = inbuf[i+5];
msbits |= (v & 0x80) >>> 2;
outbuf[j+6] = v & 0x7F;
v = inbuf[i+6];
msbits |= (v & 0x80) >>> 1;
outbuf[j+7] = v & 0x7F;
outbuf[j+0] = msbits;
i += 7; j += 8;
}
if ( _7rem > 0 ) {
msbits = 0;
for ( let ir = 0; ir < _7rem; ir++ ) {
v = inbuf[i+ir];
msbits |= (v & 0x80) >>> (7 - ir);
outbuf[j+ir+1] = v & 0x7F;
}
outbuf[j+0] = msbits;
}
const textDecoder = new TextDecoder();
return textDecoder.decode(outbuf);
},
// TODO:
// Surprisingly, there does not seem to be any performance gain when
// first converting the input string into a Uint8Array through
// TextEncoder. Investigate again to confirm original findings and
// to find out whether results have changed. Not using TextEncoder()
// to create an intermediate input buffer lower peak memory usage
// at selfie load time.
//
// const textEncoder = new TextEncoder();
// const inbuf = textEncoder.encode(instr);
// const inputLength = inbuf.byteLength;
decode: function(instr, arrbuf) {
const inputLength = instr.length;
let _8cnt = inputLength >>> 3;
let outputLength = _8cnt * 7;
let _8rem = inputLength % 8;
if ( _8rem !== 0 ) {
outputLength += _8rem - 1;
}
const outbuf = arrbuf instanceof ArrayBuffer === false
? new Uint8Array(outputLength)
: new Uint8Array(arrbuf);
let msbits;
let i = 0, j = 0;
while ( _8cnt-- ) {
msbits = instr.charCodeAt(i+0);
outbuf[j+0] = msbits << 7 & 0x80 | instr.charCodeAt(i+1);
outbuf[j+1] = msbits << 6 & 0x80 | instr.charCodeAt(i+2);
outbuf[j+2] = msbits << 5 & 0x80 | instr.charCodeAt(i+3);
outbuf[j+3] = msbits << 4 & 0x80 | instr.charCodeAt(i+4);
outbuf[j+4] = msbits << 3 & 0x80 | instr.charCodeAt(i+5);
outbuf[j+5] = msbits << 2 & 0x80 | instr.charCodeAt(i+6);
outbuf[j+6] = msbits << 1 & 0x80 | instr.charCodeAt(i+7);
i += 8; j += 7;
}
if ( _8rem > 1 ) {
msbits = instr.charCodeAt(i+0);
for ( let ir = 1; ir < _8rem; ir++ ) {
outbuf[j+ir-1] = msbits << (8-ir) & 0x80 | instr.charCodeAt(i+ir);
}
}
return outbuf;
},
decodeSize: function(instr) {
const size = (instr.length >>> 3) * 7;
const rem = instr.length & 7;
return rem === 0 ? size : size + rem - 1;
},
};