Mirrors/uBlock
1
0
Fork 0
mirror of https://github.com/gorhill/uBlock.git synced 2024-11-22 02:12:44 +01:00
Code Issues Releases Wiki Activity

Refactor selfie generation into a more flexible persistence mechanism

Browse Source 
The motivation is to address the higher peak memory usage at launch
time with 3rd-gen HNTrie when a selfie was present.

The selfie generation prior to this change was to collect all
filtering data into a single data structure, and then to serialize
that whole structure at once into storage (using JSON.stringify).

However, HNTrie serialization requires that a large UintArray32 be
converted into a plain JS array, which itslef would be indirectly
converted into a JSON string. This was the main reason why peak
memory usage would be higher at launch from selfie, since the JSON
string would need to be wholly unserialized into JS objects, which
themselves would need to be converted into more specialized data
structures (like that Uint32Array one).

The solution to lower peak memory usage at launch is to refactor
selfie generation to allow a more piecemeal approach: each filtering
component is given the ability to serialize itself rather than to be
forced to be embedded in the master selfie. With this approach, the
HNTrie buffer can now serialize to its own storage by converting the
buffer data directly into a string which can be directly sent to
storage. This avoiding expensive intermediate steps such as
converting into a JS array and then to a JSON string.

As part of the refactoring, there was also opportunistic code
upgrade to ES6 and Promise (eventually all of uBO's code will be
proper ES6).

Additionally, the polyfill to bring getBytesInUse() to Firefox has
been revisited to replace the rather expensive previous
implementation with an implementation with virtually no overhead.
This commit is contained in:
Raymond Hill 2019-02-14 13:33:55 -05:00
parent 83a3767a16
commit ed7e34fb07
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
14 changed files with 588 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.jshintrc
View File

@ -7,6 +7,7 @@
"browser": false, // global variable in Firefox, Edge
"chrome": false, // global variable in Chromium, Chrome, Opera
"Components": false, // global variable in Firefox
"log": false,
"safari": false,
"self": false,
"vAPI": false,

4
platform/chromium/vapi-common.js
View File

@ -30,6 +30,10 @@
/******************************************************************************/
vAPI.T0 = Date.now();
/******************************************************************************/
vAPI.setTimeout = vAPI.setTimeout || self.setTimeout.bind(self);
/******************************************************************************/

1
src/background.html
View File

@ -5,6 +5,7 @@
<title>uBlock Origin</title>
</head>
<body>
<script src="js/console.js"></script>
<script src="lib/lz4/lz4-block-codec-any.js"></script>
<script src="lib/punycode.js"></script>
<script src="lib/publicsuffixlist/publicsuffixlist.js"></script>

125
src/js/assets.js
View File

@ -449,10 +449,9 @@ const assetCacheRegistryStartTime = Date.now();
let assetCacheRegistryPromise;
let assetCacheRegistry = {};
const getAssetCacheRegistry = function(callback) {
const getAssetCacheRegistry = function() {
if ( assetCacheRegistryPromise === undefined ) {
assetCacheRegistryPromise = new Promise(resolve => {
// start of executor
µBlock.cacheStorage.get('assetCacheRegistry', bin => {
if (
bin instanceof Object &&
@ -462,13 +461,10 @@ const getAssetCacheRegistry = function(callback) {
}
resolve();
});
// end of executor
});
}
assetCacheRegistryPromise.then(( ) => {
callback(assetCacheRegistry);
});
return assetCacheRegistryPromise.then(( ) => assetCacheRegistry);
};
const saveAssetCacheRegistry = (function() {
@ -513,11 +509,9 @@ const assetCacheRead = function(assetKey, callback) {
reportBack(bin[internalKey]);
};
let onReady = function() {
getAssetCacheRegistry().then(( ) => {
µBlock.cacheStorage.get(internalKey, onAssetRead);
};
getAssetCacheRegistry(onReady);
});
};
const assetCacheWrite = function(assetKey, details, callback) {
@ -542,7 +536,18 @@ const assetCacheWrite = function(assetKey, details, callback) {
if ( details instanceof Object && typeof details.url === 'string' ) {
entry.remoteURL = details.url;
}
µBlock.cacheStorage.set({ assetCacheRegistry, [internalKey]: content });
µBlock.cacheStorage.set(
{ [internalKey]: content },
details => {
if (
details instanceof Object &&
typeof details.bytesInUse === 'number'
) {
entry.byteLength = details.bytesInUse;
}
saveAssetCacheRegistry(true);
}
);
const result = { assetKey, content };
if ( typeof callback === 'function' ) {
callback(result);
@ -550,14 +555,16 @@ const assetCacheWrite = function(assetKey, details, callback) {
// https://github.com/uBlockOrigin/uBlock-issues/issues/248
fireNotification('after-asset-updated', result);
};
getAssetCacheRegistry(onReady);
getAssetCacheRegistry().then(( ) => {
µBlock.cacheStorage.get(internalKey, onReady);
});
};
const assetCacheRemove = function(pattern, callback) {
const onReady = function() {
const cacheDict = assetCacheRegistry,
removedEntries = [],
removedContent = [];
getAssetCacheRegistry().then(cacheDict => {
const removedEntries = [];
const removedContent = [];
for ( const assetKey in cacheDict ) {
if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
continue;
@ -582,14 +589,15 @@ const assetCacheRemove = function(pattern, callback) {
{ assetKey: removedEntries[i] }
);
}
};
getAssetCacheRegistry(onReady);
});
};
const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
const onReady = function() {
const cacheDict = assetCacheRegistry;
if ( typeof exclude === 'function' ) {
callback = exclude;
exclude = undefined;
}
getAssetCacheRegistry().then(cacheDict => {
let mustSave = false;
for ( const assetKey in cacheDict ) {
if ( pattern instanceof RegExp ) {
@ -617,12 +625,7 @@ const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
if ( typeof callback === 'function' ) {
callback();
}
};
if ( typeof exclude === 'function' ) {
callback = exclude;
exclude = undefined;
}
getAssetCacheRegistry(onReady);
});
};
/******************************************************************************/
@ -642,12 +645,12 @@ const stringIsNotEmpty = function(s) {
**/
var readUserAsset = function(assetKey, callback) {
var reportBack = function(content) {
const readUserAsset = function(assetKey, callback) {
const reportBack = function(content) {
callback({ assetKey: assetKey, content: content });
};
var onLoaded = function(bin) {
const onLoaded = function(bin) {
if ( !bin ) { return reportBack(''); }
var content = '';
if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
@ -671,7 +674,7 @@ var readUserAsset = function(assetKey, callback) {
}
return reportBack(content);
};
var toRead = assetKey;
let toRead = assetKey;
if ( assetKey === µBlock.userFiltersPath ) {
toRead = [
assetKey,
@ -682,7 +685,7 @@ var readUserAsset = function(assetKey, callback) {
vAPI.storage.get(toRead, onLoaded);
};
var saveUserAsset = function(assetKey, content, callback) {
const saveUserAsset = function(assetKey, content, callback) {
var bin = {};
bin[assetKey] = content;
// TODO(seamless migration):
@ -711,27 +714,33 @@ api.get = function(assetKey, options, callback) {
callback = noopfunc;
}
return new Promise(resolve => {
// start of executor
if ( assetKey === µBlock.userFiltersPath ) {
readUserAsset(assetKey, callback);
readUserAsset(assetKey, details => {
callback(details);
resolve(details);
});
return;
}
var assetDetails = {},
let assetDetails = {},
contentURLs,
contentURL;
var reportBack = function(content, err) {
var details = { assetKey: assetKey, content: content };
const reportBack = function(content, err) {
const details = { assetKey: assetKey, content: content };
if ( err ) {
details.error = assetDetails.lastError = err;
} else {
assetDetails.lastError = undefined;
}
callback(details);
resolve(details);
};
var onContentNotLoaded = function() {
var isExternal;
const onContentNotLoaded = function() {
let isExternal;
while ( (contentURL = contentURLs.shift()) ) {
isExternal = reIsExternalPath.test(contentURL);
if ( isExternal === false || assetDetails.hasLocalURL !== true ) {
@ -748,7 +757,7 @@ api.get = function(assetKey, options, callback) {
}
};
var onContentLoaded = function(details) {
const onContentLoaded = function(details) {
if ( stringIsNotEmpty(details.content) === false ) {
onContentNotLoaded();
return;
@ -762,7 +771,7 @@ api.get = function(assetKey, options, callback) {
reportBack(details.content);
};
var onCachedContentLoaded = function(details) {
const onCachedContentLoaded = function(details) {
if ( details.content !== '' ) {
return reportBack(details.content);
}
@ -780,11 +789,13 @@ api.get = function(assetKey, options, callback) {
};
assetCacheRead(assetKey, onCachedContentLoaded);
// end of executor
});
};
/******************************************************************************/
var getRemote = function(assetKey, callback) {
const getRemote = function(assetKey, callback) {
var assetDetails = {},
contentURLs,
contentURL;
@ -852,10 +863,19 @@ var getRemote = function(assetKey, callback) {
/******************************************************************************/
api.put = function(assetKey, content, callback) {
if ( reIsUserAsset.test(assetKey) ) {
return saveUserAsset(assetKey, content, callback);
return new Promise(resolve => {
const onDone = function(details) {
if ( typeof callback === 'function' ) {
callback(details);
}
assetCacheWrite(assetKey, content, callback);
resolve(details);
};
if ( reIsUserAsset.test(assetKey) ) {
saveUserAsset(assetKey, content, onDone);
} else {
assetCacheWrite(assetKey, content, onDone);
}
});
};
/******************************************************************************/
@ -895,7 +915,7 @@ api.metadata = function(callback) {
if ( cacheRegistryReady ) { onReady(); }
});
getAssetCacheRegistry(function() {
getAssetCacheRegistry().then(( ) => {
cacheRegistryReady = true;
if ( assetRegistryReady ) { onReady(); }
});
@ -903,6 +923,19 @@ api.metadata = function(callback) {
/******************************************************************************/
api.getBytesInUse = function() {
return getAssetCacheRegistry().then(cacheDict => {
let bytesUsed = 0;
for ( const assetKey in cacheDict ) {
if ( cacheDict.hasOwnProperty(assetKey) === false ) { continue; }
bytesUsed += cacheDict[assetKey].byteLength || 0;
}
return bytesUsed;
});
};
/******************************************************************************/
api.purge = assetCacheMarkAsDirty;
api.remove = function(pattern, callback) {
@ -1013,7 +1046,7 @@ var updateNext = function() {
updateOne();
});
getAssetCacheRegistry(function(dict) {
getAssetCacheRegistry().then(dict => {
cacheDict = dict;
if ( !assetDict ) { return; }
updateOne();

14
src/js/background.js
View File

@ -46,6 +46,7 @@ const µBlock = (function() { // jshint ignore:line
cacheStorageAPI: 'unset',
cacheStorageCompression: true,
cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
consoleLogLevel: 'unset',
debugScriptlets: false,
disableWebAssembly: false,
ignoreRedirectFilters: false,
@ -53,6 +54,7 @@ const µBlock = (function() { // jshint ignore:line
manualUpdateAssetFetchPeriod: 500,
popupFontSize: 'unset',
requestJournalProcessPeriod: 1000,
selfieAfter: 11,
strictBlockingBypassDuration: 120,
suspendTabsUntilReady: false,
userResourcesLocation: 'unset'
@ -95,13 +97,13 @@ const µBlock = (function() { // jshint ignore:line
hiddenSettingsDefault: hiddenSettingsDefault,
hiddenSettings: (function() {
let out = Object.assign({}, hiddenSettingsDefault),
const out = Object.assign({}, hiddenSettingsDefault),
json = vAPI.localStorage.getItem('immediateHiddenSettings');
if ( typeof json === 'string' ) {
try {
let o = JSON.parse(json);
const o = JSON.parse(json);
if ( o instanceof Object ) {
for ( let k in o ) {
for ( const k in o ) {
if ( out.hasOwnProperty(k) ) {
out[k] = o[k];
}
@ -111,8 +113,6 @@ const µBlock = (function() { // jshint ignore:line
catch(ex) {
}
}
// Remove once 1.15.12+ is widespread.
vAPI.localStorage.removeItem('hiddenSettings');
return out;
})(),
@ -138,7 +138,7 @@ const µBlock = (function() { // jshint ignore:line
// Read-only
systemSettings: {
compiledMagic: 6, // Increase when compiled format changes
selfieMagic: 7 // Increase when selfie format changes
selfieMagic: 8 // Increase when selfie format changes
},
restoreBackupSettings: {
@ -161,8 +161,6 @@ const µBlock = (function() { // jshint ignore:line
selectedFilterLists: [],
availableFilterLists: {},
selfieAfter: 17 * oneMinute,
pageStores: new Map(),
pageStoresToken: 0,

36
src/js/cachestorage.js
View File

@ -326,17 +326,27 @@
if ( typeof callback !== 'function' ) {
callback = noopfn;
}
let keys = Object.keys(keyvalStore);
const keys = Object.keys(keyvalStore);
if ( keys.length === 0 ) { return callback(); }
let promises = [ getDb() ];
let entries = [];
let dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true;
let handleEncodingResult = result => {
const promises = [ getDb() ];
const entries = [];
const dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true;
let bytesInUse = 0;
const handleEncodingResult = result => {
if ( typeof result.data === 'string' ) {
bytesInUse += result.data.length;
} else if ( result.data instanceof Blob ) {
bytesInUse += result.data.size;
}
entries.push({ key: result.key, value: result.data });
};
for ( let key of keys ) {
let data = keyvalStore[key];
if ( typeof data !== 'string' || dontCompress ) {
for ( const key of keys ) {
const data = keyvalStore[key];
const isString = typeof data === 'string';
if ( isString === false || dontCompress ) {
if ( isString ) {
bytesInUse += data.length;
}
entries.push({ key, value: data });
continue;
}
@ -346,20 +356,20 @@
}
Promise.all(promises).then(( ) => {
if ( !db ) { return callback(); }
let finish = ( ) => {
const finish = ( ) => {
dbBytesInUse = undefined;
if ( callback === undefined ) { return; }
let cb = callback;
callback = undefined;
cb();
cb({ bytesInUse });
};
try {
let transaction = db.transaction(STORAGE_NAME, 'readwrite');
const transaction = db.transaction(STORAGE_NAME, 'readwrite');
transaction.oncomplete =
transaction.onerror =
transaction.onabort = finish;
let table = transaction.objectStore(STORAGE_NAME);
for ( let entry of entries ) {
const table = transaction.objectStore(STORAGE_NAME);
for ( const entry of entries ) {
table.put(entry);
}
} catch (ex) {

34
src/js/console.js Normal file
View File

@ -0,0 +1,34 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2019-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
'use strict';
self.log = (function() {
const noopFunc = function() {};
const info = function(s) { console.log(`[uBO] ${s}`); };
return {
get verbosity( ) { return; },
set verbosity(level) {
this.info = console.info = level === 'info' ? info : noopFunc;
},
info,
};
})();

30
src/js/hntrie.js
View File

@ -355,7 +355,13 @@ HNTrieContainer.prototype = {
return trieRef;
},
serialize: function() {
serialize: function(encoder) {
if ( encoder instanceof Object ) {
return encoder.encode(
this.buf32.buffer,
this.buf32[HNTRIE_CHAR1_SLOT]
);
}
return Array.from(
new Uint32Array(
this.buf32.buffer,
@ -365,23 +371,29 @@ HNTrieContainer.prototype = {
);
},
unserialize: function(selfie) {
const len = (selfie.length << 2) + HNTRIE_PAGE_SIZE-1 & ~(HNTRIE_PAGE_SIZE-1);
unserialize: function(selfie, decoder) {
const shouldDecode = typeof selfie === 'string';
let byteLength = shouldDecode
? decoder.decodeSize(selfie)
: selfie.length << 2;
byteLength = byteLength + HNTRIE_PAGE_SIZE-1 & ~(HNTRIE_PAGE_SIZE-1);
if ( this.wasmMemory !== null ) {
const pageCountBefore = this.buf.length >>> 16;
const pageCountAfter = len >>> 16;
const pageCountAfter = byteLength >>> 16;
if ( pageCountAfter > pageCountBefore ) {
this.wasmMemory.grow(pageCountAfter - pageCountBefore);
this.buf = new Uint8Array(this.wasmMemory.buffer);
this.buf32 = new Uint32Array(this.buf.buffer);
}
} else {
if ( len > this.buf.length ) {
this.buf = new Uint8Array(len);
} else if ( byteLength > this.buf.length ) {
this.buf = new Uint8Array(byteLength);
this.buf32 = new Uint32Array(this.buf.buffer);
}
}
if ( shouldDecode ) {
decoder.decode(selfie, this.buf.buffer);
} else {
this.buf32.set(selfie);
}
this.needle = '';
},
@ -684,6 +696,6 @@ HNTrieContainer.prototype.HNTrieRef.prototype = {
WebAssembly.compileStreaming
).catch(reason => {
HNTrieContainer.wasmModulePromise = null;
console.info(reason);
log.info(reason);
});
})();

87
src/js/redirect-engine.js
View File

@ -29,12 +29,12 @@
/******************************************************************************/
const warResolve = (function() {
var warPairs = [];
let warPairs = [];
var onPairsReady = function() {
var reng = µBlock.redirectEngine;
for ( var i = 0; i < warPairs.length; i += 2 ) {
var resource = reng.resources.get(warPairs[i+0]);
const onPairsReady = function() {
const reng = µBlock.redirectEngine;
for ( let i = 0; i < warPairs.length; i += 2 ) {
const resource = reng.resources.get(warPairs[i+0]);
if ( resource === undefined ) { continue; }
resource.warURL = vAPI.getURL(
'/web_accessible_resources/' + warPairs[i+1]
@ -48,15 +48,15 @@ const warResolve = (function() {
return onPairsReady();
}
var onPairsLoaded = function(details) {
var marker = '>>>>>';
var pos = details.content.indexOf(marker);
const onPairsLoaded = function(details) {
const marker = '>>>>>';
const pos = details.content.indexOf(marker);
if ( pos === -1 ) { return; }
var pairs = details.content.slice(pos + marker.length)
const pairs = details.content.slice(pos + marker.length)
.trim()
.split('\n');
if ( (pairs.length & 1) !== 0 ) { return; }
for ( var i = 0; i < pairs.length; i++ ) {
for ( let i = 0; i < pairs.length; i++ ) {
pairs[i] = pairs[i].trim();
}
warPairs = pairs;
@ -64,7 +64,7 @@ const warResolve = (function() {
};
µBlock.assets.fetchText(
'/web_accessible_resources/imported.txt?secret=' + vAPI.warSecret,
`/web_accessible_resources/imported.txt?secret=${vAPI.warSecret}`,
onPairsLoaded
);
};
@ -374,18 +374,17 @@ RedirectEngine.prototype.supportedTypes = new Map([
/******************************************************************************/
RedirectEngine.prototype.toSelfie = function() {
RedirectEngine.prototype.toSelfie = function(path) {
// Because rules may contains RegExp instances, we need to manually
// convert it to a serializable format. The serialized format must be
// suitable to be used as an argument to the Map() constructor.
var rules = [],
rule, entries, i, entry;
for ( var item of this.rules ) {
rule = [ item[0], [] ];
entries = item[1];
i = entries.length;
const rules = [];
for ( const item of this.rules ) {
const rule = [ item[0], [] ];
const entries = item[1];
let i = entries.length;
while ( i-- ) {
entry = entries[i];
const entry = entries[i];
rule[1].push({
tok: entry.tok,
pat: entry.pat instanceof RegExp ? entry.pat.source : entry.pat
@ -393,23 +392,34 @@ RedirectEngine.prototype.toSelfie = function() {
}
rules.push(rule);
}
return {
return µBlock.assets.put(
`${path}/main`,
JSON.stringify({
rules: rules,
ruleTypes: Array.from(this.ruleTypes),
ruleSources: Array.from(this.ruleSources),
ruleDestinations: Array.from(this.ruleDestinations)
};
})
);
};
/******************************************************************************/
RedirectEngine.prototype.fromSelfie = function(selfie) {
RedirectEngine.prototype.fromSelfie = function(path) {
return µBlock.assets.get(`${path}/main`).then(details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
this.rules = new Map(selfie.rules);
this.ruleTypes = new Set(selfie.ruleTypes);
this.ruleSources = new Set(selfie.ruleSources);
this.ruleDestinations = new Set(selfie.ruleDestinations);
this.modifyTime = Date.now();
return true;
});
};
/******************************************************************************/
@ -494,41 +504,46 @@ RedirectEngine.prototype.resourcesFromString = function(text) {
/******************************************************************************/
let resourcesSelfieVersion = 3;
const resourcesSelfieVersion = 3;
RedirectEngine.prototype.selfieFromResources = function() {
let selfie = {
µBlock.assets.put(
'compiled/redirectEngine/resources',
JSON.stringify({
version: resourcesSelfieVersion,
resources: Array.from(this.resources)
};
µBlock.cacheStorage.set({ resourcesSelfie: JSON.stringify(selfie) });
})
);
};
RedirectEngine.prototype.resourcesFromSelfie = function(callback) {
µBlock.cacheStorage.get('resourcesSelfie', bin => {
let selfie = bin && bin.resourcesSelfie;
if ( typeof selfie === 'string' ) {
RedirectEngine.prototype.resourcesFromSelfie = function() {
return µBlock.assets.get(
'compiled/redirectEngine/resources'
).then(details => {
let selfie;
try {
selfie = JSON.parse(selfie);
selfie = JSON.parse(details.content);
} catch(ex) {
}
}
if (
selfie instanceof Object === false ||
selfie.version !== resourcesSelfieVersion ||
Array.isArray(selfie.resources) === false
) {
return callback(false);
return false;
}
this.resources = new Map();
for ( let entry of selfie.resources ) {
this.resources.set(entry[0], RedirectEntry.fromSelfie(entry[1]));
for ( const [ token, entry ] of selfie.resources ) {
this.resources.set(token, RedirectEntry.fromSelfie(entry));
}
callback(true);
return true;
});
};
RedirectEngine.prototype.invalidateResourcesSelfie = function() {
µBlock.assets.remove('compiled/redirectEngine/resources');
// TODO: obsolete, remove eventually
µBlock.cacheStorage.remove('resourcesSelfie');
};

73
src/js/start.js
View File

@ -81,6 +81,8 @@ var onAllReady = function() {
µb.contextMenu.update(null);
µb.firstInstall = false;
log.info(`All ready ${Date.now()-vAPI.T0} ms after launch`);
};
/******************************************************************************/
@ -137,22 +139,29 @@ let initializeTabs = function() {
// Filtering engines dependencies:
// - PSL
var onPSLReady = function() {
µb.selfieManager.load(function(valid) {
const onPSLReady = function() {
log.info(`PSL ready ${Date.now()-vAPI.T0} ms after launch`);
µb.selfieManager.load().then(valid => {
if ( valid === true ) {
return onAllReady();
log.info(`Selfie ready ${Date.now()-vAPI.T0} ms after launch`);
onAllReady();
return;
}
µb.loadFilterLists(onAllReady);
µb.loadFilterLists(( ) => {
log.info(`Filter lists ready ${Date.now()-vAPI.T0} ms after launch`);
onAllReady();
});
});
};
/******************************************************************************/
var onCommandShortcutsReady = function(commandShortcuts) {
const onCommandShortcutsReady = function(commandShortcuts) {
if ( Array.isArray(commandShortcuts) === false ) { return; }
µb.commandShortcuts = new Map(commandShortcuts);
if ( µb.canUpdateShortcuts === false ) { return; }
for ( let entry of commandShortcuts ) {
for ( const entry of commandShortcuts ) {
vAPI.commands.update({ name: entry[0], shortcut: entry[1] });
}
};
@ -161,7 +170,7 @@ var onCommandShortcutsReady = function(commandShortcuts) {
// To bring older versions up to date
var onVersionReady = function(lastVersion) {
const onVersionReady = function(lastVersion) {
if ( lastVersion === vAPI.app.version ) { return; }
// Since AMO does not allow updating resources.txt, force a reload when a
@ -176,7 +185,7 @@ var onVersionReady = function(lastVersion) {
// If unused, just comment out for when we need to compare versions in the
// future.
let intFromVersion = function(s) {
const intFromVersion = function(s) {
let parts = s.match(/(?:^|\.|b|rc)\d+/g);
if ( parts === null ) { return 0; }
let vint = 0;
@ -223,7 +232,7 @@ var onVersionReady = function(lastVersion) {
// Whitelist parser needs PSL to be ready.
// gorhill 2014-12-15: not anymore
var onNetWhitelistReady = function(netWhitelistRaw) {
const onNetWhitelistReady = function(netWhitelistRaw) {
µb.netWhitelist = µb.whitelistFromString(netWhitelistRaw);
µb.netWhitelistModifyTime = Date.now();
};
@ -232,8 +241,10 @@ var onNetWhitelistReady = function(netWhitelistRaw) {
// User settings are in memory
var onUserSettingsReady = function(fetched) {
var userSettings = µb.userSettings;
const onUserSettingsReady = function(fetched) {
log.info(`User settings ready ${Date.now()-vAPI.T0} ms after launch`);
const userSettings = µb.userSettings;
fromFetch(userSettings, fetched);
@ -264,7 +275,7 @@ var onUserSettingsReady = function(fetched) {
// Housekeeping, as per system setting changes
var onSystemSettingsReady = function(fetched) {
const onSystemSettingsReady = function(fetched) {
var mustSaveSystemSettings = false;
if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) {
µb.assets.remove(/^compiled\//);
@ -282,7 +293,9 @@ var onSystemSettingsReady = function(fetched) {
/******************************************************************************/
var onFirstFetchReady = function(fetched) {
const onFirstFetchReady = function(fetched) {
log.info(`First fetch ready ${Date.now()-vAPI.T0} ms after launch`);
// https://github.com/gorhill/uBlock/issues/747
µb.firstInstall = fetched.version === '0.0.0.0';
@ -295,10 +308,7 @@ var onFirstFetchReady = function(fetched) {
onVersionReady(fetched.version);
onCommandShortcutsReady(fetched.commandShortcuts);
Promise.all([
µb.loadPublicSuffixList(),
µb.staticNetFilteringEngine.readyToUse()
]).then(( ) => {
µb.loadPublicSuffixList().then(( ) => {
onPSLReady();
});
µb.loadRedirectResources();
@ -306,31 +316,27 @@ var onFirstFetchReady = function(fetched) {
/******************************************************************************/
var toFetch = function(from, fetched) {
for ( var k in from ) {
if ( from.hasOwnProperty(k) === false ) {
continue;
}
const toFetch = function(from, fetched) {
for ( const k in from ) {
if ( from.hasOwnProperty(k) === false ) { continue; }
fetched[k] = from[k];
}
};
var fromFetch = function(to, fetched) {
for ( var k in to ) {
if ( to.hasOwnProperty(k) === false ) {
continue;
}
if ( fetched.hasOwnProperty(k) === false ) {
continue;
}
const fromFetch = function(to, fetched) {
for ( const k in to ) {
if ( to.hasOwnProperty(k) === false ) { continue; }
if ( fetched.hasOwnProperty(k) === false ) { continue; }
to[k] = fetched[k];
}
};
/******************************************************************************/
var onSelectedFilterListsLoaded = function() {
var fetchableProps = {
const onSelectedFilterListsLoaded = function() {
log.info(`List selection ready ${Date.now()-vAPI.T0} ms after launch`);
const fetchableProps = {
'commandShortcuts': [],
'compiledMagic': 0,
'dynamicFilteringString': [
@ -371,7 +377,8 @@ var onSelectedFilterListsLoaded = function() {
// compatibility, this means a special asynchronous call to load selected
// filter lists.
var onAdminSettingsRestored = function() {
const onAdminSettingsRestored = function() {
log.info(`Admin settings ready ${Date.now()-vAPI.T0} ms after launch`);
µb.loadSelectedFilterLists(onSelectedFilterListsLoaded);
};

20
src/js/static-ext-filtering.js
View File

@ -821,18 +821,30 @@
µb.htmlFilteringEngine.fromCompiledContent(reader, options);
};
api.toSelfie = function() {
return {
api.toSelfie = function(path) {
return µBlock.assets.put(
`${path}/main`,
JSON.stringify({
cosmetic: µb.cosmeticFilteringEngine.toSelfie(),
scriptlets: µb.scriptletFilteringEngine.toSelfie(),
html: µb.htmlFilteringEngine.toSelfie()
};
})
);
};
api.fromSelfie = function(selfie) {
api.fromSelfie = function(path) {
return µBlock.assets.get(`${path}/main`).then(details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic);
µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets);
µb.htmlFilteringEngine.fromSelfie(selfie.html);
return true;
});
};
return api;

77
src/js/static-net-filtering.js
View File

@ -2105,21 +2105,21 @@ FilterContainer.prototype.readyToUse = function() {
/******************************************************************************/
FilterContainer.prototype.toSelfie = function() {
let categoriesToSelfie = function(categoryMap) {
let selfie = [];
for ( let categoryEntry of categoryMap ) {
let tokenEntries = [];
for ( let tokenEntry of categoryEntry[1] ) {
tokenEntries.push([ tokenEntry[0], tokenEntry[1].compile() ]);
FilterContainer.prototype.toSelfie = function(path) {
const categoriesToSelfie = function(categoryMap) {
const selfie = [];
for ( const [ catbits, bucket ] of categoryMap ) {
const tokenEntries = [];
for ( const [ token, filter ] of bucket ) {
tokenEntries.push([ token, filter.compile() ]);
}
selfie.push([ categoryEntry[0], tokenEntries ]);
selfie.push([ catbits, tokenEntries ]);
}
return selfie;
};
let dataFiltersToSelfie = function(dataFilters) {
let selfie = [];
const dataFiltersToSelfie = function(dataFilters) {
const selfie = [];
for ( let entry of dataFilters.values() ) {
do {
selfie.push(entry.compile());
@ -2129,22 +2129,45 @@ FilterContainer.prototype.toSelfie = function() {
return selfie;
};
return {
return Promise.all([
µBlock.assets.put(
`${path}/trieContainer`,
FilterHostnameDict.trieContainer.serialize(µBlock.base128)
),
µBlock.assets.put(
`${path}/main`,
JSON.stringify({
processedFilterCount: this.processedFilterCount,
acceptedCount: this.acceptedCount,
rejectedCount: this.rejectedCount,
allowFilterCount: this.allowFilterCount,
blockFilterCount: this.blockFilterCount,
discardedCount: this.discardedCount,
trieContainer: FilterHostnameDict.trieContainer.serialize(),
categories: categoriesToSelfie(this.categories),
dataFilters: dataFiltersToSelfie(this.dataFilters)
};
dataFilters: dataFiltersToSelfie(this.dataFilters),
})
)
]);
};
/******************************************************************************/
FilterContainer.prototype.fromSelfie = function(selfie) {
FilterContainer.prototype.fromSelfie = function(path) {
return Promise.all([
µBlock.assets.get(`${path}/trieContainer`).then(details => {
FilterHostnameDict.trieContainer.unserialize(
details.content,
µBlock.base128
);
return true;
}),
µBlock.assets.get(`${path}/main`).then(details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
this.frozen = true;
this.processedFilterCount = selfie.processedFilterCount;
this.acceptedCount = selfie.acceptedCount;
@ -2152,24 +2175,26 @@ FilterContainer.prototype.fromSelfie = function(selfie) {
this.allowFilterCount = selfie.allowFilterCount;
this.blockFilterCount = selfie.blockFilterCount;
this.discardedCount = selfie.discardedCount;
FilterHostnameDict.trieContainer.unserialize(selfie.trieContainer);
for ( let categoryEntry of selfie.categories ) {
let tokenMap = new Map();
for ( let tokenEntry of categoryEntry[1] ) {
tokenMap.set(tokenEntry[0], filterFromCompiledData(tokenEntry[1]));
for ( const [ catbits, bucket ] of selfie.categories ) {
const tokenMap = new Map();
for ( const [ token, fdata ] of bucket ) {
tokenMap.set(token, filterFromCompiledData(fdata));
}
this.categories.set(categoryEntry[0], tokenMap);
this.categories.set(catbits, tokenMap);
}
for ( let dataEntry of selfie.dataFilters ) {
let entry = FilterDataHolderEntry.load(dataEntry);
let bucket = this.dataFilters.get(entry.tokenHash);
for ( const dataEntry of selfie.dataFilters ) {
const entry = FilterDataHolderEntry.load(dataEntry);
const bucket = this.dataFilters.get(entry.tokenHash);
if ( bucket !== undefined ) {
entry.next = bucket;
}
this.dataFilters.set(entry.tokenHash, entry);
}
return true;
}),
]).then(results =>
results.reduce((acc, v) => acc && v, true)
);
};
/******************************************************************************/

149
src/js/storage.js
View File

@ -32,7 +32,7 @@
let bytesInUse;
let countdown = 0;
let process = count => {
const process = count => {
if ( typeof count === 'number' ) {
if ( bytesInUse === undefined ) {
bytesInUse = 0;
@ -50,12 +50,11 @@
countdown += 1;
vAPI.storage.getBytesInUse(null, process);
}
if (
this.cacheStorage !== vAPI.storage &&
this.cacheStorage.getBytesInUse instanceof Function
) {
if ( this.cacheStorage !== vAPI.storage ) {
countdown += 1;
this.cacheStorage.getBytesInUse(null, process);
this.assets.getBytesInUse().then(count => {
process(count);
});
}
if ( countdown === 0 ) {
callback();
@ -94,10 +93,10 @@
µBlock.loadHiddenSettings = function() {
vAPI.storage.get('hiddenSettings', bin => {
if ( bin instanceof Object === false ) { return; }
let hs = bin.hiddenSettings;
const hs = bin.hiddenSettings;
if ( hs instanceof Object ) {
let hsDefault = this.hiddenSettingsDefault;
for ( let key in hsDefault ) {
const hsDefault = this.hiddenSettingsDefault;
for ( const key in hsDefault ) {
if (
hsDefault.hasOwnProperty(key) &&
hs.hasOwnProperty(key) &&
@ -110,6 +109,7 @@
if ( vAPI.localStorage.getItem('immediateHiddenSettings') === null ) {
this.saveImmediateHiddenSettings();
}
self.log.verbosity = this.hiddenSettings.consoleLogLevel;
});
};
@ -118,8 +118,8 @@
// which were not modified by the user.
µBlock.saveHiddenSettings = function(callback) {
let bin = { hiddenSettings: {} };
for ( let prop in this.hiddenSettings ) {
const bin = { hiddenSettings: {} };
for ( const prop in this.hiddenSettings ) {
if (
this.hiddenSettings.hasOwnProperty(prop) &&
this.hiddenSettings[prop] !== this.hiddenSettingsDefault[prop]
@ -129,6 +129,7 @@
}
vAPI.storage.set(bin, callback);
this.saveImmediateHiddenSettings();
self.log.verbosity = this.hiddenSettings.consoleLogLevel;
};
/******************************************************************************/
@ -969,41 +970,41 @@
/******************************************************************************/
µBlock.loadRedirectResources = function(updatedContent) {
var µb = this,
content = '';
let content = '';
var onDone = function() {
µb.redirectEngine.resourcesFromString(content);
const onDone = ( ) => {
this.redirectEngine.resourcesFromString(content);
};
var onUserResourcesLoaded = function(details) {
const onUserResourcesLoaded = details => {
if ( details.content !== '' ) {
content += '\n\n' + details.content;
}
onDone();
};
var onResourcesLoaded = function(details) {
const onResourcesLoaded = details => {
if ( details.content !== '' ) {
content = details.content;
}
if ( µb.hiddenSettings.userResourcesLocation === 'unset' ) {
if ( this.hiddenSettings.userResourcesLocation === 'unset' ) {
return onDone();
}
µb.assets.fetchText(µb.hiddenSettings.userResourcesLocation, onUserResourcesLoaded);
this.assets.fetchText(
this.hiddenSettings.userResourcesLocation,
onUserResourcesLoaded
);
};
if ( typeof updatedContent === 'string' && updatedContent.length !== 0 ) {
return onResourcesLoaded({ content: updatedContent });
}
var onSelfieReady = function(success) {
this.redirectEngine.resourcesFromSelfie().then(success => {
if ( success !== true ) {
µb.assets.get('ublock-resources', onResourcesLoaded);
this.assets.get('ublock-resources', onResourcesLoaded);
}
};
µb.redirectEngine.resourcesFromSelfie(onSelfieReady);
});
};
/******************************************************************************/
@ -1013,39 +1014,25 @@
publicSuffixList.enableWASM();
}
return new Promise(resolve => {
// start of executor
this.assets.get('compiled/' + this.pslAssetKey, details => {
let selfie;
try {
selfie = JSON.parse(details.content);
} catch (ex) {
}
if (
selfie instanceof Object &&
publicSuffixList.fromSelfie(selfie)
) {
resolve();
return;
}
this.assets.get(this.pslAssetKey, details => {
return this.assets.get(
'compiled/' + this.pslAssetKey
).then(details =>
publicSuffixList.fromSelfie(details.content, µBlock.base128)
).then(valid => {
if ( valid === true ) { return; }
return this.assets.get(this.pslAssetKey, details => {
if ( details.content !== '' ) {
this.compilePublicSuffixList(details.content);
}
resolve();
});
});
// end of executor
});
};
/******************************************************************************/
µBlock.compilePublicSuffixList = function(content) {
publicSuffixList.parse(content, punycode.toASCII);
this.assets.put(
'compiled/' + this.pslAssetKey,
JSON.stringify(publicSuffixList.toSelfie())
publicSuffixList.toSelfie(µBlock.base128)
);
};
@ -1056,60 +1043,76 @@
// some set time.
µBlock.selfieManager = (function() {
let µb = µBlock;
let timer = null;
const µb = µBlock;
let timer;
// As of 2018-05-31:
// JSON.stringify-ing ourselves results in a better baseline
// memory usage at selfie-load time. For some reasons.
let create = function() {
timer = null;
let selfie = JSON.stringify({
const create = function() {
Promise.all([
µb.assets.put(
'selfie/main',
JSON.stringify({
magic: µb.systemSettings.selfieMagic,
availableFilterLists: µb.availableFilterLists,
staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(),
redirectEngine: µb.redirectEngine.toSelfie(),
staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie()
});
µb.cacheStorage.set({ selfie: selfie });
})
),
µb.redirectEngine.toSelfie('selfie/redirectEngine'),
µb.staticExtFilteringEngine.toSelfie('selfie/staticExtFilteringEngine'),
µb.staticNetFilteringEngine.toSelfie('selfie/staticNetFilteringEngine'),
]).then(( ) => {
µb.lz4Codec.relinquish();
});
};
let load = function(callback) {
µb.cacheStorage.get('selfie', function(bin) {
const load = function() {
return Promise.all([
µb.assets.get('selfie/main').then(details => {
if (
bin instanceof Object === false ||
typeof bin.selfie !== 'string'
details instanceof Object === false ||
typeof details.content !== 'string' ||
details.content === ''
) {
return callback(false);
return false;
}
let selfie;
try {
selfie = JSON.parse(bin.selfie);
selfie = JSON.parse(details.content);
} catch(ex) {
}
if (
selfie instanceof Object === false ||
selfie.magic !== µb.systemSettings.selfieMagic
) {
return callback(false);
return false;
}
µb.availableFilterLists = selfie.availableFilterLists;
µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine);
µb.redirectEngine.fromSelfie(selfie.redirectEngine);
µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine);
callback(true);
return true;
}),
µb.redirectEngine.fromSelfie('selfie/redirectEngine'),
µb.staticExtFilteringEngine.fromSelfie('selfie/staticExtFilteringEngine'),
µb.staticNetFilteringEngine.fromSelfie('selfie/staticNetFilteringEngine'),
]).then(results =>
results.reduce((acc, v) => acc && v, true)
).catch(reason => {
log.info(reason);
return false;
});
};
let destroy = function() {
if ( timer !== null ) {
const destroy = function() {
if ( timer !== undefined ) {
clearTimeout(timer);
timer = null;
timer = undefined;
}
µb.cacheStorage.remove('selfie');
timer = vAPI.setTimeout(create, µb.selfieAfter);
µb.cacheStorage.remove('selfie'); // TODO: obsolete, remove eventually.
µb.assets.remove(/^selfie\//);
timer = vAPI.setTimeout(( ) => {
timer = undefined;
create();
}, µb.hiddenSettings.selfieAfter * 60000);
};
return {
@ -1299,6 +1302,8 @@
// Compile the list while we have the raw version in memory
if ( topic === 'after-asset-updated' ) {
// Skip selfie-related content.
if ( details.assetKey.startsWith('selfie/') ) { return; }
var cached = typeof details.content === 'string' && details.content !== '';
if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) {
if ( cached ) {

109
src/js/utils.js
View File

@ -496,3 +496,112 @@
µBlock.orphanizeString = function(s) {
return JSON.parse(JSON.stringify(s));
};
/******************************************************************************/
// Custom base128 encoder/decoder
//
// TODO:
// Could expand the LZ4 codec API to be able to return UTF8-safe string
// representation of a compressed buffer, and thus the code below could be
// moved LZ4 codec-side.
µBlock.base128 = {
encode: function(arrbuf, arrlen) {
const inbuf = new Uint8Array(arrbuf, 0, arrlen);
const inputLength = arrlen;
let _7cnt = Math.floor(inputLength / 7);
let outputLength = _7cnt * 8;
let _7rem = inputLength % 7;
if ( _7rem !== 0 ) {
outputLength += 1 + _7rem;
}
const outbuf = new Uint8Array(outputLength);
let msbits, v;
let i = 0, j = 0;
while ( _7cnt-- ) {
v = inbuf[i+0];
msbits = (v & 0x80) >>> 7;
outbuf[j+1] = v & 0x7F;
v = inbuf[i+1];
msbits |= (v & 0x80) >>> 6;
outbuf[j+2] = v & 0x7F;
v = inbuf[i+2];
msbits |= (v & 0x80) >>> 5;
outbuf[j+3] = v & 0x7F;
v = inbuf[i+3];
msbits |= (v & 0x80) >>> 4;
outbuf[j+4] = v & 0x7F;
v = inbuf[i+4];
msbits |= (v & 0x80) >>> 3;
outbuf[j+5] = v & 0x7F;
v = inbuf[i+5];
msbits |= (v & 0x80) >>> 2;
outbuf[j+6] = v & 0x7F;
v = inbuf[i+6];
msbits |= (v & 0x80) >>> 1;
outbuf[j+7] = v & 0x7F;
outbuf[j+0] = msbits;
i += 7; j += 8;
}
if ( _7rem > 0 ) {
msbits = 0;
for ( let ir = 0; ir < _7rem; ir++ ) {
v = inbuf[i+ir];
msbits |= (v & 0x80) >>> (7 - ir);
outbuf[j+ir+1] = v & 0x7F;
}
outbuf[j+0] = msbits;
}
const textDecoder = new TextDecoder();
return textDecoder.decode(outbuf);
},
// TODO:
// Surprisingly, there does not seem to be any performance gain when
// first converting the input string into a Uint8Array through
// TextEncoder. Investigate again to confirm original findings and
// to find out whether results have changed. Not using TextEncoder()
// to create an intermediate input buffer lower peak memory usage
// at selfie load time.
//
// const textEncoder = new TextEncoder();
// const inbuf = textEncoder.encode(instr);
// const inputLength = inbuf.byteLength;
decode: function(instr, arrbuf) {
const inputLength = instr.length;
let _8cnt = inputLength >>> 3;
let outputLength = _8cnt * 7;
let _8rem = inputLength % 8;
if ( _8rem !== 0 ) {
outputLength += _8rem - 1;
}
const outbuf = arrbuf instanceof ArrayBuffer === false
? new Uint8Array(outputLength)
: new Uint8Array(arrbuf);
let msbits;
let i = 0, j = 0;
while ( _8cnt-- ) {
msbits = instr.charCodeAt(i+0);
outbuf[j+0] = msbits << 7 & 0x80 | instr.charCodeAt(i+1);
outbuf[j+1] = msbits << 6 & 0x80 | instr.charCodeAt(i+2);
outbuf[j+2] = msbits << 5 & 0x80 | instr.charCodeAt(i+3);
outbuf[j+3] = msbits << 4 & 0x80 | instr.charCodeAt(i+4);
outbuf[j+4] = msbits << 3 & 0x80 | instr.charCodeAt(i+5);
outbuf[j+5] = msbits << 2 & 0x80 | instr.charCodeAt(i+6);
outbuf[j+6] = msbits << 1 & 0x80 | instr.charCodeAt(i+7);
i += 8; j += 7;
}
if ( _8rem > 1 ) {
msbits = instr.charCodeAt(i+0);
for ( let ir = 1; ir < _8rem; ir++ ) {
outbuf[j+ir-1] = msbits << (8-ir) & 0x80 | instr.charCodeAt(i+ir);
}
}
return outbuf;
},
decodeSize: function(instr) {
const size = (instr.length >>> 3) * 7;
const rem = instr.length & 7;
return rem === 0 ? size : size + rem - 1;
},
};