mirror of
https://github.com/gorhill/uBlock.git
synced 2024-11-07 03:12:33 +01:00
added optional lz4 compression for cache storage (https://github.com/uBlockOrigin/uBlock-issues/issues/141)
Squashed commit of the following:
commit 6a8473822537636ac54d5dabdb14472114bb730b
Author: Raymond Hill <rhill@raymondhill.net>
Date: Mon Aug 6 10:56:44 2018 -0400
remove remnant of snappyjs and spurious instruction
commit 9a4b709bee97d3cc2235fab602359fa5953bdb46
Author: Raymond Hill <rhill@raymondhill.net>
Date: Mon Aug 6 09:48:58 2018 -0400
make cache storage compression optionally available on all platforms
New advanced setting: `cacheStorageCompression`. Default is `false`.
commit 22ee6547f2f7c9c5aefe25dea1262a1b31612155
Author: Raymond Hill <rhill@raymondhill.net>
Date: Sun Aug 5 19:16:26 2018 -0400
remove Chromium from lz4 experiment
commit ee3e201c45afe983508f70713a2d43af74737d8d
Author: Raymond Hill <rhill@raymondhill.net>
Date: Sun Aug 5 18:52:43 2018 -0400
import lz4-block-codec.wasm library
commit 883a3118efcfd749c82356fde7134754d6ae371d
Author: Raymond Hill <rhill@raymondhill.net>
Date: Sun Aug 5 18:50:46 2018 -0400
implement storage compression through lz4-wasm [draft]
commit 48d1ccaba407de447c2cd6747dc3a90839c260a7
Merge: 8ae77e6 b34c897
Author: Raymond Hill <rhill@raymondhill.net>
Date: Sat Aug 4 08:56:51 2018 -0400
Merge branch 'master' of github.com:gorhill/uBlock into lz4
commit 8ae77e6aeeaa85af335e664c2560d2afd37288c6
Author: Raymond Hill <rhill@raymondhill.net>
Date: Wed Jul 25 18:17:45 2018 -0400
experiment with compression
This commit is contained in:
parent
b34c897553
commit
e163080518
@ -1,7 +1,7 @@
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
|
|
||||||
uBlock Origin - a browser extension to block requests.
|
uBlock Origin - a browser extension to block requests.
|
||||||
Copyright (C) 2016-2018 The uBlock Origin authors
|
Copyright (C) 2016-present The uBlock Origin authors
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
@ -19,7 +19,7 @@
|
|||||||
Home: https://github.com/gorhill/uBlock
|
Home: https://github.com/gorhill/uBlock
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* global indexedDB, IDBDatabase */
|
/* global IDBDatabase, indexedDB */
|
||||||
|
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
@ -48,15 +48,10 @@ vAPI.cacheStorage = (function() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const STORAGE_NAME = 'uBlock0CacheStorage';
|
const STORAGE_NAME = 'uBlock0CacheStorage';
|
||||||
var db;
|
let db;
|
||||||
var pending = [];
|
let pendingInitialization;
|
||||||
|
|
||||||
// prime the db so that it's ready asap for next access.
|
let get = function get(input, callback) {
|
||||||
getDb(noopfn);
|
|
||||||
|
|
||||||
return { get, set, remove, clear, getBytesInUse };
|
|
||||||
|
|
||||||
function get(input, callback) {
|
|
||||||
if ( typeof callback !== 'function' ) { return; }
|
if ( typeof callback !== 'function' ) { return; }
|
||||||
if ( input === null ) {
|
if ( input === null ) {
|
||||||
return getAllFromDb(callback);
|
return getAllFromDb(callback);
|
||||||
@ -71,51 +66,48 @@ vAPI.cacheStorage = (function() {
|
|||||||
output = input;
|
output = input;
|
||||||
}
|
}
|
||||||
return getFromDb(toRead, output, callback);
|
return getFromDb(toRead, output, callback);
|
||||||
}
|
};
|
||||||
|
|
||||||
function set(input, callback) {
|
let set = function set(input, callback) {
|
||||||
putToDb(input, callback);
|
putToDb(input, callback);
|
||||||
}
|
};
|
||||||
|
|
||||||
function remove(key, callback) {
|
let remove = function remove(key, callback) {
|
||||||
deleteFromDb(key, callback);
|
deleteFromDb(key, callback);
|
||||||
}
|
};
|
||||||
|
|
||||||
function clear(callback) {
|
let clear = function clear(callback) {
|
||||||
clearDb(callback);
|
clearDb(callback);
|
||||||
}
|
};
|
||||||
|
|
||||||
function getBytesInUse(keys, callback) {
|
let getBytesInUse = function getBytesInUse(keys, callback) {
|
||||||
// TODO: implement this
|
// TODO: implement this
|
||||||
callback(0);
|
callback(0);
|
||||||
}
|
};
|
||||||
|
|
||||||
function genericErrorHandler(error) {
|
let api = { get, set, remove, clear, getBytesInUse, error: undefined };
|
||||||
console.error('[%s]', STORAGE_NAME, error);
|
|
||||||
|
let genericErrorHandler = function(ev) {
|
||||||
|
let error = ev.target && ev.target.error;
|
||||||
|
if ( error && error.name === 'QuotaExceededError' ) {
|
||||||
|
api.error = error.name;
|
||||||
}
|
}
|
||||||
|
console.error('[%s]', STORAGE_NAME, error && error.name);
|
||||||
|
};
|
||||||
|
|
||||||
function noopfn() {
|
function noopfn() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function processPendings() {
|
let getDb = function getDb() {
|
||||||
var cb;
|
|
||||||
while ( (cb = pending.shift()) ) {
|
|
||||||
cb(db);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function getDb(callback) {
|
|
||||||
if ( pending === undefined ) {
|
|
||||||
return callback();
|
|
||||||
}
|
|
||||||
if ( pending.length !== 0 ) {
|
|
||||||
return pending.push(callback);
|
|
||||||
}
|
|
||||||
if ( db instanceof IDBDatabase ) {
|
if ( db instanceof IDBDatabase ) {
|
||||||
return callback(db);
|
return Promise.resolve(db);
|
||||||
|
}
|
||||||
|
if ( db === null ) {
|
||||||
|
return Promise.resolve(null);
|
||||||
|
}
|
||||||
|
if ( pendingInitialization !== undefined ) {
|
||||||
|
return pendingInitialization;
|
||||||
}
|
}
|
||||||
pending.push(callback);
|
|
||||||
if ( pending.length !== 1 ) { return; }
|
|
||||||
// https://github.com/gorhill/uBlock/issues/3156
|
// https://github.com/gorhill/uBlock/issues/3156
|
||||||
// I have observed that no event was fired in Tor Browser 7.0.7 +
|
// I have observed that no event was fired in Tor Browser 7.0.7 +
|
||||||
// medium security level after the request to open the database was
|
// medium security level after the request to open the database was
|
||||||
@ -125,7 +117,8 @@ vAPI.cacheStorage = (function() {
|
|||||||
// necessary when reading the `error` property because we are not
|
// necessary when reading the `error` property because we are not
|
||||||
// allowed to read this propery outside of event handlers in newer
|
// allowed to read this propery outside of event handlers in newer
|
||||||
// implementation of IDBRequest (my understanding).
|
// implementation of IDBRequest (my understanding).
|
||||||
var req;
|
pendingInitialization = new Promise(resolve => {
|
||||||
|
let req;
|
||||||
try {
|
try {
|
||||||
req = indexedDB.open(STORAGE_NAME, 1);
|
req = indexedDB.open(STORAGE_NAME, 1);
|
||||||
if ( req.error ) {
|
if ( req.error ) {
|
||||||
@ -135,80 +128,81 @@ vAPI.cacheStorage = (function() {
|
|||||||
} catch(ex) {
|
} catch(ex) {
|
||||||
}
|
}
|
||||||
if ( req === undefined ) {
|
if ( req === undefined ) {
|
||||||
processPendings();
|
pendingInitialization = undefined;
|
||||||
pending = undefined;
|
db = null;
|
||||||
|
resolve(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
req.onupgradeneeded = function(ev) {
|
req.onupgradeneeded = function(ev) {
|
||||||
req = undefined;
|
req = undefined;
|
||||||
db = ev.target.result;
|
let db = ev.target.result;
|
||||||
db.onerror = db.onabort = genericErrorHandler;
|
db.onerror = db.onabort = genericErrorHandler;
|
||||||
var table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' });
|
let table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' });
|
||||||
table.createIndex('value', 'value', { unique: false });
|
table.createIndex('value', 'value', { unique: false });
|
||||||
};
|
};
|
||||||
req.onsuccess = function(ev) {
|
req.onsuccess = function(ev) {
|
||||||
|
pendingInitialization = undefined;
|
||||||
req = undefined;
|
req = undefined;
|
||||||
db = ev.target.result;
|
db = ev.target.result;
|
||||||
db.onerror = db.onabort = genericErrorHandler;
|
db.onerror = db.onabort = genericErrorHandler;
|
||||||
processPendings();
|
resolve(db);
|
||||||
};
|
};
|
||||||
req.onerror = req.onblocked = function() {
|
req.onerror = req.onblocked = function() {
|
||||||
|
pendingInitialization = undefined;
|
||||||
req = undefined;
|
req = undefined;
|
||||||
|
db = null;
|
||||||
console.log(this.error);
|
console.log(this.error);
|
||||||
processPendings();
|
resolve(null);
|
||||||
pending = undefined;
|
};
|
||||||
|
});
|
||||||
|
return pendingInitialization;
|
||||||
};
|
};
|
||||||
}
|
|
||||||
|
|
||||||
function getFromDb(keys, store, callback) {
|
let getFromDb = function(keys, keystore, callback) {
|
||||||
if ( typeof callback !== 'function' ) { return; }
|
if ( typeof callback !== 'function' ) { return; }
|
||||||
if ( keys.length === 0 ) { return callback(store); }
|
if ( keys.length === 0 ) { return callback(keystore); }
|
||||||
var gotOne = function() {
|
let gotOne = function() {
|
||||||
if ( typeof this.result === 'object' ) {
|
if ( typeof this.result === 'object' ) {
|
||||||
store[this.result.key] = this.result.value;
|
keystore[this.result.key] = this.result.value;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
getDb(function(db) {
|
getDb().then(( ) => {
|
||||||
if ( !db ) { return callback(); }
|
if ( !db ) { return callback(); }
|
||||||
var transaction = db.transaction(STORAGE_NAME);
|
let transaction = db.transaction(STORAGE_NAME);
|
||||||
transaction.oncomplete =
|
transaction.oncomplete =
|
||||||
transaction.onerror =
|
transaction.onerror =
|
||||||
transaction.onabort = function() {
|
transaction.onabort = ( ) => callback(keystore);
|
||||||
return callback(store);
|
let table = transaction.objectStore(STORAGE_NAME);
|
||||||
};
|
for ( let key of keys ) {
|
||||||
var table = transaction.objectStore(STORAGE_NAME);
|
let req = table.get(key);
|
||||||
for ( var key of keys ) {
|
|
||||||
var req = table.get(key);
|
|
||||||
req.onsuccess = gotOne;
|
req.onsuccess = gotOne;
|
||||||
req.onerror = noopfn;
|
req.onerror = noopfn;
|
||||||
req = undefined;
|
req = undefined;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
};
|
||||||
|
|
||||||
function getAllFromDb(callback) {
|
let getAllFromDb = function(callback) {
|
||||||
if ( typeof callback !== 'function' ) {
|
if ( typeof callback !== 'function' ) {
|
||||||
callback = noopfn;
|
callback = noopfn;
|
||||||
}
|
}
|
||||||
getDb(function(db) {
|
getDb().then(( ) => {
|
||||||
if ( !db ) { return callback(); }
|
if ( !db ) { return callback(); }
|
||||||
var output = {};
|
let keystore = {};
|
||||||
var transaction = db.transaction(STORAGE_NAME);
|
let transaction = db.transaction(STORAGE_NAME);
|
||||||
transaction.oncomplete =
|
transaction.oncomplete =
|
||||||
transaction.onerror =
|
transaction.onerror =
|
||||||
transaction.onabort = function() {
|
transaction.onabort = ( ) => callback(keystore);
|
||||||
callback(output);
|
let table = transaction.objectStore(STORAGE_NAME),
|
||||||
};
|
|
||||||
var table = transaction.objectStore(STORAGE_NAME),
|
|
||||||
req = table.openCursor();
|
req = table.openCursor();
|
||||||
req.onsuccess = function(ev) {
|
req.onsuccess = function(ev) {
|
||||||
var cursor = ev.target.result;
|
let cursor = ev.target.result;
|
||||||
if ( !cursor ) { return; }
|
if ( !cursor ) { return; }
|
||||||
output[cursor.key] = cursor.value;
|
keystore[cursor.key] = cursor.value;
|
||||||
cursor.continue();
|
cursor.continue();
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
}
|
};
|
||||||
|
|
||||||
// https://github.com/uBlockOrigin/uBlock-issues/issues/141
|
// https://github.com/uBlockOrigin/uBlock-issues/issues/141
|
||||||
// Mind that IDBDatabase.transaction() and IDBObjectStore.put()
|
// Mind that IDBDatabase.transaction() and IDBObjectStore.put()
|
||||||
@ -216,20 +210,19 @@ vAPI.cacheStorage = (function() {
|
|||||||
// https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction
|
// https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction
|
||||||
// https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put
|
// https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put
|
||||||
|
|
||||||
function putToDb(input, callback) {
|
let putToDb = function(keystore, callback) {
|
||||||
if ( typeof callback !== 'function' ) {
|
if ( typeof callback !== 'function' ) {
|
||||||
callback = noopfn;
|
callback = noopfn;
|
||||||
}
|
}
|
||||||
let keys = Object.keys(input);
|
let keys = Object.keys(keystore);
|
||||||
if ( keys.length === 0 ) { return callback(); }
|
if ( keys.length === 0 ) { return callback(); }
|
||||||
getDb(function(db) {
|
getDb().then(( ) => {
|
||||||
if ( !db ) { return callback(); }
|
if ( !db ) { return callback(); }
|
||||||
let finish = ( ) => {
|
let finish = ( ) => {
|
||||||
if ( callback !== undefined ) {
|
if ( callback === undefined ) { return; }
|
||||||
let cb = callback;
|
let cb = callback;
|
||||||
callback = undefined;
|
callback = undefined;
|
||||||
cb();
|
cb();
|
||||||
}
|
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
let transaction = db.transaction(STORAGE_NAME, 'readwrite');
|
let transaction = db.transaction(STORAGE_NAME, 'readwrite');
|
||||||
@ -240,7 +233,7 @@ vAPI.cacheStorage = (function() {
|
|||||||
for ( let key of keys ) {
|
for ( let key of keys ) {
|
||||||
let entry = {};
|
let entry = {};
|
||||||
entry.key = key;
|
entry.key = key;
|
||||||
entry.value = input[key];
|
entry.value = keystore[key];
|
||||||
table.put(entry);
|
table.put(entry);
|
||||||
entry = undefined;
|
entry = undefined;
|
||||||
}
|
}
|
||||||
@ -248,39 +241,64 @@ vAPI.cacheStorage = (function() {
|
|||||||
finish();
|
finish();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
};
|
||||||
|
|
||||||
function deleteFromDb(input, callback) {
|
let deleteFromDb = function(input, callback) {
|
||||||
if ( typeof callback !== 'function' ) {
|
if ( typeof callback !== 'function' ) {
|
||||||
callback = noopfn;
|
callback = noopfn;
|
||||||
}
|
}
|
||||||
var keys = Array.isArray(input) ? input.slice() : [ input ];
|
let keys = Array.isArray(input) ? input.slice() : [ input ];
|
||||||
if ( keys.length === 0 ) { return callback(); }
|
if ( keys.length === 0 ) { return callback(); }
|
||||||
getDb(function(db) {
|
getDb().then(db => {
|
||||||
if ( !db ) { return callback(); }
|
if ( !db ) { return callback(); }
|
||||||
var transaction = db.transaction(STORAGE_NAME, 'readwrite');
|
let finish = ( ) => {
|
||||||
|
if ( callback === undefined ) { return; }
|
||||||
|
let cb = callback;
|
||||||
|
callback = undefined;
|
||||||
|
cb();
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
let transaction = db.transaction(STORAGE_NAME, 'readwrite');
|
||||||
transaction.oncomplete =
|
transaction.oncomplete =
|
||||||
transaction.onerror =
|
transaction.onerror =
|
||||||
transaction.onabort = callback;
|
transaction.onabort = finish;
|
||||||
var table = transaction.objectStore(STORAGE_NAME);
|
let table = transaction.objectStore(STORAGE_NAME);
|
||||||
for ( var key of keys ) {
|
for ( let key of keys ) {
|
||||||
table.delete(key);
|
table.delete(key);
|
||||||
}
|
}
|
||||||
});
|
} catch (ex) {
|
||||||
|
finish();
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
function clearDb(callback) {
|
let clearDb = function(callback) {
|
||||||
if ( typeof callback !== 'function' ) {
|
if ( typeof callback !== 'function' ) {
|
||||||
callback = noopfn;
|
callback = noopfn;
|
||||||
}
|
}
|
||||||
getDb(function(db) {
|
getDb().then(db => {
|
||||||
if ( !db ) { return callback(); }
|
if ( !db ) { return callback(); }
|
||||||
var req = db.transaction(STORAGE_NAME, 'readwrite')
|
let finish = ( ) => {
|
||||||
|
if ( callback === undefined ) { return; }
|
||||||
|
let cb = callback;
|
||||||
|
callback = undefined;
|
||||||
|
cb();
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
let req = db.transaction(STORAGE_NAME, 'readwrite')
|
||||||
.objectStore(STORAGE_NAME)
|
.objectStore(STORAGE_NAME)
|
||||||
.clear();
|
.clear();
|
||||||
req.onsuccess = req.onerror = callback;
|
req.onsuccess = req.onerror = finish;
|
||||||
});
|
} catch (ex) {
|
||||||
|
finish();
|
||||||
}
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
// prime the db so that it's ready asap for next access.
|
||||||
|
getDb(noopfn);
|
||||||
|
|
||||||
|
return api;
|
||||||
}());
|
}());
|
||||||
|
|
||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
|
286
src/js/assets.js
286
src/js/assets.js
@ -19,6 +19,8 @@
|
|||||||
Home: https://github.com/gorhill/uBlock
|
Home: https://github.com/gorhill/uBlock
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/* global WebAssembly */
|
||||||
|
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
@ -409,6 +411,247 @@ api.unregisterAssetSource = function(assetKey) {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
|
||||||
|
Experimental support for cache storage compression.
|
||||||
|
|
||||||
|
For background information on the topic, see:
|
||||||
|
https://github.com/uBlockOrigin/uBlock-issues/issues/141#issuecomment-407737186
|
||||||
|
|
||||||
|
**/
|
||||||
|
|
||||||
|
let lz4Codec = (function() {
|
||||||
|
let lz4wasmInstance;
|
||||||
|
let pendingInitialization;
|
||||||
|
let textEncoder, textDecoder;
|
||||||
|
let ttlCount = 0;
|
||||||
|
let ttlTimer;
|
||||||
|
|
||||||
|
const ttlDelay = 60 * 1000;
|
||||||
|
|
||||||
|
let init = function() {
|
||||||
|
if (
|
||||||
|
lz4wasmInstance === null ||
|
||||||
|
WebAssembly instanceof Object === false ||
|
||||||
|
typeof WebAssembly.instantiateStreaming !== 'function'
|
||||||
|
) {
|
||||||
|
lz4wasmInstance = null;
|
||||||
|
return Promise.resolve(null);
|
||||||
|
}
|
||||||
|
if ( lz4wasmInstance instanceof WebAssembly.Instance ) {
|
||||||
|
return Promise.resolve(lz4wasmInstance);
|
||||||
|
}
|
||||||
|
if ( pendingInitialization === undefined ) {
|
||||||
|
pendingInitialization = WebAssembly.instantiateStreaming(
|
||||||
|
fetch('lib/lz4-block-codec.wasm', { mode: 'same-origin' })
|
||||||
|
).then(result => {
|
||||||
|
pendingInitialization = undefined;
|
||||||
|
lz4wasmInstance = result && result.instance || null;
|
||||||
|
});
|
||||||
|
pendingInitialization.catch(( ) => {
|
||||||
|
lz4wasmInstance = null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return pendingInitialization;
|
||||||
|
};
|
||||||
|
|
||||||
|
// We can't shrink memory usage of wasm instances, and in the current
|
||||||
|
// case memory usage can grow to a significant amount given that
|
||||||
|
// a single contiguous memory buffer is required to accommodate both
|
||||||
|
// input and output data. Thus a time-to-live implementation which
|
||||||
|
// will cause the wasm instance to be forgotten after enough time
|
||||||
|
// elapse without the instance being used.
|
||||||
|
|
||||||
|
let destroy = function() {
|
||||||
|
console.info(
|
||||||
|
'uBO: freeing lz4-block-codec.wasm instance (memory.buffer = %d kB)',
|
||||||
|
lz4wasmInstance.exports.memory.buffer.byteLength >>> 10
|
||||||
|
);
|
||||||
|
lz4wasmInstance = undefined;
|
||||||
|
textEncoder = textDecoder = undefined;
|
||||||
|
ttlCount = 0;
|
||||||
|
ttlTimer = undefined;
|
||||||
|
};
|
||||||
|
|
||||||
|
let ttlManage = function(count) {
|
||||||
|
if ( ttlTimer !== undefined ) {
|
||||||
|
clearTimeout(ttlTimer);
|
||||||
|
ttlTimer = undefined;
|
||||||
|
}
|
||||||
|
ttlCount += count;
|
||||||
|
if ( ttlCount > 0 ) { return; }
|
||||||
|
if ( lz4wasmInstance === null ) { return; }
|
||||||
|
ttlTimer = vAPI.setTimeout(destroy, ttlDelay);
|
||||||
|
};
|
||||||
|
|
||||||
|
let growMemoryTo = function(byteLength) {
|
||||||
|
let lz4api = lz4wasmInstance.exports;
|
||||||
|
let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength;
|
||||||
|
let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16;
|
||||||
|
let pageCountAfter = (neededByteLength + 65535) >>> 16;
|
||||||
|
if ( pageCountAfter > pageCountBefore ) {
|
||||||
|
lz4api.memory.grow(pageCountAfter - pageCountBefore);
|
||||||
|
}
|
||||||
|
return lz4api.memory;
|
||||||
|
};
|
||||||
|
|
||||||
|
let resolveEncodedValue = function(resolve, key, value) {
|
||||||
|
let t0 = window.performance.now();
|
||||||
|
let lz4api = lz4wasmInstance.exports;
|
||||||
|
let mem0 = lz4api.getLinearMemoryOffset();
|
||||||
|
let memory = growMemoryTo(mem0 + 65536 * 4);
|
||||||
|
let hashTable = new Int32Array(memory.buffer, mem0, 65536);
|
||||||
|
hashTable.fill(-65536, 0, 65536);
|
||||||
|
let hashTableSize = hashTable.byteLength;
|
||||||
|
if ( textEncoder === undefined ) {
|
||||||
|
textEncoder = new TextEncoder();
|
||||||
|
}
|
||||||
|
let inputArray = textEncoder.encode(value);
|
||||||
|
let inputSize = inputArray.byteLength;
|
||||||
|
let memSize =
|
||||||
|
hashTableSize +
|
||||||
|
inputSize +
|
||||||
|
8 + lz4api.lz4BlockEncodeBound(inputSize);
|
||||||
|
memory = growMemoryTo(memSize);
|
||||||
|
let inputMem = new Uint8Array(
|
||||||
|
memory.buffer,
|
||||||
|
mem0 + hashTableSize,
|
||||||
|
inputSize
|
||||||
|
);
|
||||||
|
inputMem.set(inputArray);
|
||||||
|
let outputSize = lz4api.lz4BlockEncode(
|
||||||
|
mem0 + hashTableSize,
|
||||||
|
inputSize,
|
||||||
|
mem0 + hashTableSize + inputSize + 8
|
||||||
|
);
|
||||||
|
if ( outputSize === 0 ) { resolve(value); }
|
||||||
|
let outputMem = new Uint8Array(
|
||||||
|
memory.buffer,
|
||||||
|
mem0 + hashTableSize + inputSize,
|
||||||
|
8 + outputSize
|
||||||
|
);
|
||||||
|
outputMem[0] = 0x18;
|
||||||
|
outputMem[1] = 0x4D;
|
||||||
|
outputMem[2] = 0x22;
|
||||||
|
outputMem[3] = 0x04;
|
||||||
|
outputMem[4] = (inputSize >>> 0) & 0xFF;
|
||||||
|
outputMem[5] = (inputSize >>> 8) & 0xFF;
|
||||||
|
outputMem[6] = (inputSize >>> 16) & 0xFF;
|
||||||
|
outputMem[7] = (inputSize >>> 24) & 0xFF;
|
||||||
|
console.info(
|
||||||
|
'uBO: [%s] compressed %d bytes into %d bytes in %s ms',
|
||||||
|
key,
|
||||||
|
inputSize,
|
||||||
|
outputSize,
|
||||||
|
(window.performance.now() - t0).toFixed(2)
|
||||||
|
);
|
||||||
|
resolve(new Blob([ outputMem ]));
|
||||||
|
};
|
||||||
|
|
||||||
|
let resolveDecodedValue = function(resolve, ev, key, value) {
|
||||||
|
let inputBuffer = ev.target.result;
|
||||||
|
if ( inputBuffer instanceof ArrayBuffer === false ) {
|
||||||
|
return resolve(value);
|
||||||
|
}
|
||||||
|
let t0 = window.performance.now();
|
||||||
|
let metadata = new Uint8Array(inputBuffer, 0, 8);
|
||||||
|
if (
|
||||||
|
metadata[0] !== 0x18 ||
|
||||||
|
metadata[1] !== 0x4D ||
|
||||||
|
metadata[2] !== 0x22 ||
|
||||||
|
metadata[3] !== 0x04
|
||||||
|
) {
|
||||||
|
return resolve(value);
|
||||||
|
}
|
||||||
|
let inputSize = inputBuffer.byteLength - 8;
|
||||||
|
let outputSize =
|
||||||
|
(metadata[4] << 0) |
|
||||||
|
(metadata[5] << 8) |
|
||||||
|
(metadata[6] << 16) |
|
||||||
|
(metadata[7] << 24);
|
||||||
|
let lz4api = lz4wasmInstance.exports;
|
||||||
|
let mem0 = lz4api.getLinearMemoryOffset();
|
||||||
|
let memSize = inputSize + outputSize;
|
||||||
|
let memory = growMemoryTo(memSize);
|
||||||
|
let inputArea = new Uint8Array(
|
||||||
|
memory.buffer,
|
||||||
|
mem0,
|
||||||
|
inputSize
|
||||||
|
);
|
||||||
|
inputArea.set(new Uint8Array(inputBuffer, 8, inputSize));
|
||||||
|
outputSize = lz4api.lz4BlockDecode(inputSize);
|
||||||
|
if ( outputSize === 0 ) {
|
||||||
|
return resolve(value);
|
||||||
|
}
|
||||||
|
let outputArea = new Uint8Array(
|
||||||
|
memory.buffer,
|
||||||
|
mem0 + inputSize,
|
||||||
|
outputSize
|
||||||
|
);
|
||||||
|
if ( textDecoder === undefined ) {
|
||||||
|
textDecoder = new TextDecoder();
|
||||||
|
}
|
||||||
|
value = textDecoder.decode(outputArea);
|
||||||
|
console.info(
|
||||||
|
'uBO: [%s] decompressed %d bytes into %d bytes in %s ms',
|
||||||
|
key,
|
||||||
|
inputSize,
|
||||||
|
outputSize,
|
||||||
|
(window.performance.now() - t0).toFixed(2)
|
||||||
|
);
|
||||||
|
resolve(value);
|
||||||
|
};
|
||||||
|
|
||||||
|
let encodeValue = function(key, value) {
|
||||||
|
if ( !lz4wasmInstance ) {
|
||||||
|
return Promise.resolve(value);
|
||||||
|
}
|
||||||
|
return new Promise(resolve => {
|
||||||
|
resolveEncodedValue(resolve, key, value);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
let decodeValue = function(key, value) {
|
||||||
|
if ( !lz4wasmInstance ) {
|
||||||
|
return Promise.resolve(value);
|
||||||
|
}
|
||||||
|
return new Promise(resolve => {
|
||||||
|
let blobReader = new FileReader();
|
||||||
|
blobReader.onloadend = ev => {
|
||||||
|
resolveDecodedValue(resolve, ev, key, value);
|
||||||
|
};
|
||||||
|
blobReader.readAsArrayBuffer(value);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
encode: function(key, value) {
|
||||||
|
if ( typeof value !== 'string' || value.length < 4096 ) {
|
||||||
|
return Promise.resolve(value);
|
||||||
|
}
|
||||||
|
ttlManage(1);
|
||||||
|
return init().then(( ) => {
|
||||||
|
return encodeValue(key, value);
|
||||||
|
}).then(result => {
|
||||||
|
ttlManage(-1);
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
decode: function(key, value) {
|
||||||
|
if ( value instanceof Blob === false ) {
|
||||||
|
return Promise.resolve(value);
|
||||||
|
}
|
||||||
|
ttlManage(1);
|
||||||
|
return init().then(( ) => {
|
||||||
|
return decodeValue(key, value);
|
||||||
|
}).then(result => {
|
||||||
|
ttlManage(-1);
|
||||||
|
return result;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
})();
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
|
|
||||||
The purpose of the asset cache registry is to keep track of all assets
|
The purpose of the asset cache registry is to keep track of all assets
|
||||||
@ -472,26 +715,32 @@ var saveAssetCacheRegistry = (function() {
|
|||||||
var assetCacheRead = function(assetKey, callback) {
|
var assetCacheRead = function(assetKey, callback) {
|
||||||
let internalKey = 'cache/' + assetKey;
|
let internalKey = 'cache/' + assetKey;
|
||||||
|
|
||||||
let reportBack = function(content, err) {
|
let reportBack = function(content) {
|
||||||
|
if ( content instanceof Blob ) { content = ''; }
|
||||||
let details = { assetKey: assetKey, content: content };
|
let details = { assetKey: assetKey, content: content };
|
||||||
if ( err ) { details.error = err; }
|
if ( content === '' ) { details.error = 'E_NOTFOUND'; }
|
||||||
callback(details);
|
callback(details);
|
||||||
};
|
};
|
||||||
|
|
||||||
let onAssetRead = function(bin) {
|
let onAssetRead = function(bin) {
|
||||||
if (
|
if (
|
||||||
bin instanceof Object === false ||
|
bin instanceof Object === false ||
|
||||||
stringIsNotEmpty(bin[internalKey]) === false
|
bin.hasOwnProperty(internalKey) === false
|
||||||
) {
|
) {
|
||||||
return reportBack('', 'E_NOTFOUND');
|
return reportBack('');
|
||||||
}
|
}
|
||||||
let entry = assetCacheRegistry[assetKey];
|
let entry = assetCacheRegistry[assetKey];
|
||||||
if ( entry === undefined ) {
|
if ( entry === undefined ) {
|
||||||
return reportBack('', 'E_NOTFOUND');
|
return reportBack('');
|
||||||
}
|
}
|
||||||
entry.readTime = Date.now();
|
entry.readTime = Date.now();
|
||||||
saveAssetCacheRegistry(true);
|
saveAssetCacheRegistry(true);
|
||||||
reportBack(bin[internalKey]);
|
if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) {
|
||||||
|
return reportBack(bin[internalKey]);
|
||||||
|
}
|
||||||
|
lz4Codec.decode(internalKey, bin[internalKey]).then(result => {
|
||||||
|
reportBack(result);
|
||||||
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
let onReady = function() {
|
let onReady = function() {
|
||||||
@ -502,8 +751,8 @@ var assetCacheRead = function(assetKey, callback) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
var assetCacheWrite = function(assetKey, details, callback) {
|
var assetCacheWrite = function(assetKey, details, callback) {
|
||||||
var internalKey = 'cache/' + assetKey;
|
let internalKey = 'cache/' + assetKey;
|
||||||
var content = '';
|
let content = '';
|
||||||
if ( typeof details === 'string' ) {
|
if ( typeof details === 'string' ) {
|
||||||
content = details;
|
content = details;
|
||||||
} else if ( details instanceof Object ) {
|
} else if ( details instanceof Object ) {
|
||||||
@ -514,16 +763,19 @@ var assetCacheWrite = function(assetKey, details, callback) {
|
|||||||
return assetCacheRemove(assetKey, callback);
|
return assetCacheRemove(assetKey, callback);
|
||||||
}
|
}
|
||||||
|
|
||||||
var reportBack = function(content) {
|
let reportBack = function(content) {
|
||||||
var details = { assetKey: assetKey, content: content };
|
let bin = { assetCacheRegistry: assetCacheRegistry };
|
||||||
|
bin[internalKey] = content;
|
||||||
|
vAPI.cacheStorage.set(bin);
|
||||||
|
let details = { assetKey: assetKey, content: content };
|
||||||
if ( typeof callback === 'function' ) {
|
if ( typeof callback === 'function' ) {
|
||||||
callback(details);
|
callback(details);
|
||||||
}
|
}
|
||||||
fireNotification('after-asset-updated', details);
|
fireNotification('after-asset-updated', details);
|
||||||
};
|
};
|
||||||
|
|
||||||
var onReady = function() {
|
let onReady = function() {
|
||||||
var entry = assetCacheRegistry[assetKey];
|
let entry = assetCacheRegistry[assetKey];
|
||||||
if ( entry === undefined ) {
|
if ( entry === undefined ) {
|
||||||
entry = assetCacheRegistry[assetKey] = {};
|
entry = assetCacheRegistry[assetKey] = {};
|
||||||
}
|
}
|
||||||
@ -531,10 +783,12 @@ var assetCacheWrite = function(assetKey, details, callback) {
|
|||||||
if ( details instanceof Object && typeof details.url === 'string' ) {
|
if ( details instanceof Object && typeof details.url === 'string' ) {
|
||||||
entry.remoteURL = details.url;
|
entry.remoteURL = details.url;
|
||||||
}
|
}
|
||||||
var bin = { assetCacheRegistry: assetCacheRegistry };
|
if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) {
|
||||||
bin[internalKey] = content;
|
return reportBack(content);
|
||||||
vAPI.cacheStorage.set(bin);
|
}
|
||||||
reportBack(content);
|
lz4Codec.encode(internalKey, content).then(result => {
|
||||||
|
reportBack(result);
|
||||||
|
});
|
||||||
};
|
};
|
||||||
getAssetCacheRegistry(onReady);
|
getAssetCacheRegistry(onReady);
|
||||||
};
|
};
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
|
|
||||||
uBlock Origin - a browser extension to block requests.
|
uBlock Origin - a browser extension to block requests.
|
||||||
Copyright (C) 2014-2018 Raymond Hill
|
Copyright (C) 2014-present Raymond Hill
|
||||||
|
|
||||||
This program is free software: you can redistribute it and/or modify
|
This program is free software: you can redistribute it and/or modify
|
||||||
it under the terms of the GNU General Public License as published by
|
it under the terms of the GNU General Public License as published by
|
||||||
@ -42,6 +42,7 @@ var µBlock = (function() { // jshint ignore:line
|
|||||||
assetFetchTimeout: 30,
|
assetFetchTimeout: 30,
|
||||||
autoUpdateAssetFetchPeriod: 120,
|
autoUpdateAssetFetchPeriod: 120,
|
||||||
autoUpdatePeriod: 7,
|
autoUpdatePeriod: 7,
|
||||||
|
cacheStorageCompression: false,
|
||||||
debugScriptlets: false,
|
debugScriptlets: false,
|
||||||
ignoreRedirectFilters: false,
|
ignoreRedirectFilters: false,
|
||||||
ignoreScriptInjectFilters: false,
|
ignoreScriptInjectFilters: false,
|
||||||
@ -138,7 +139,7 @@ var µBlock = (function() { // jshint ignore:line
|
|||||||
// Read-only
|
// Read-only
|
||||||
systemSettings: {
|
systemSettings: {
|
||||||
compiledMagic: 3, // Increase when compiled format changes
|
compiledMagic: 3, // Increase when compiled format changes
|
||||||
selfieMagic: 3 // Increase when selfie format changes
|
selfieMagic: 4 // Increase when selfie format changes
|
||||||
},
|
},
|
||||||
|
|
||||||
restoreBackupSettings: {
|
restoreBackupSettings: {
|
||||||
|
@ -1025,13 +1025,13 @@
|
|||||||
|
|
||||||
let create = function() {
|
let create = function() {
|
||||||
timer = null;
|
timer = null;
|
||||||
let selfie = {
|
let selfie = JSON.stringify({
|
||||||
magic: µb.systemSettings.selfieMagic,
|
magic: µb.systemSettings.selfieMagic,
|
||||||
availableFilterLists: JSON.stringify(µb.availableFilterLists),
|
availableFilterLists: µb.availableFilterLists,
|
||||||
staticNetFilteringEngine: JSON.stringify(µb.staticNetFilteringEngine.toSelfie()),
|
staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(),
|
||||||
redirectEngine: JSON.stringify(µb.redirectEngine.toSelfie()),
|
redirectEngine: µb.redirectEngine.toSelfie(),
|
||||||
staticExtFilteringEngine: JSON.stringify(µb.staticExtFilteringEngine.toSelfie())
|
staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie()
|
||||||
};
|
});
|
||||||
vAPI.cacheStorage.set({ selfie: selfie });
|
vAPI.cacheStorage.set({ selfie: selfie });
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -1039,16 +1039,25 @@
|
|||||||
vAPI.cacheStorage.get('selfie', function(bin) {
|
vAPI.cacheStorage.get('selfie', function(bin) {
|
||||||
if (
|
if (
|
||||||
bin instanceof Object === false ||
|
bin instanceof Object === false ||
|
||||||
bin.selfie instanceof Object === false ||
|
typeof bin.selfie !== 'string'
|
||||||
bin.selfie.magic !== µb.systemSettings.selfieMagic ||
|
|
||||||
bin.selfie.redirectEngine === undefined
|
|
||||||
) {
|
) {
|
||||||
return callback(false);
|
return callback(false);
|
||||||
}
|
}
|
||||||
µb.availableFilterLists = JSON.parse(bin.selfie.availableFilterLists);
|
let selfie;
|
||||||
µb.staticNetFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticNetFilteringEngine));
|
try {
|
||||||
µb.redirectEngine.fromSelfie(JSON.parse(bin.selfie.redirectEngine));
|
selfie = JSON.parse(bin.selfie);
|
||||||
µb.staticExtFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticExtFilteringEngine));
|
} catch(ex) {
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
selfie instanceof Object === false ||
|
||||||
|
selfie.magic !== µb.systemSettings.selfieMagic
|
||||||
|
) {
|
||||||
|
return callback(false);
|
||||||
|
}
|
||||||
|
µb.availableFilterLists = selfie.availableFilterLists;
|
||||||
|
µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine);
|
||||||
|
µb.redirectEngine.fromSelfie(selfie.redirectEngine);
|
||||||
|
µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine);
|
||||||
callback(true);
|
callback(true);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
BIN
src/lib/lz4-block-codec.wasm
Normal file
BIN
src/lib/lz4-block-codec.wasm
Normal file
Binary file not shown.
Loading…
Reference in New Issue
Block a user