diff --git a/platform/chromium/vapi-cachestorage.js b/platform/chromium/vapi-cachestorage.js index a25c7bc2c..7671fe5d3 100644 --- a/platform/chromium/vapi-cachestorage.js +++ b/platform/chromium/vapi-cachestorage.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2016-2018 The uBlock Origin authors + Copyright (C) 2016-present The uBlock Origin authors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uBlock */ -/* global indexedDB, IDBDatabase */ +/* global IDBDatabase, indexedDB */ 'use strict'; @@ -48,15 +48,10 @@ vAPI.cacheStorage = (function() { } const STORAGE_NAME = 'uBlock0CacheStorage'; - var db; - var pending = []; + let db; + let pendingInitialization; - // prime the db so that it's ready asap for next access. - getDb(noopfn); - - return { get, set, remove, clear, getBytesInUse }; - - function get(input, callback) { + let get = function get(input, callback) { if ( typeof callback !== 'function' ) { return; } if ( input === null ) { return getAllFromDb(callback); @@ -71,51 +66,48 @@ vAPI.cacheStorage = (function() { output = input; } return getFromDb(toRead, output, callback); - } + }; - function set(input, callback) { + let set = function set(input, callback) { putToDb(input, callback); - } + }; - function remove(key, callback) { + let remove = function remove(key, callback) { deleteFromDb(key, callback); - } + }; - function clear(callback) { + let clear = function clear(callback) { clearDb(callback); - } + }; - function getBytesInUse(keys, callback) { + let getBytesInUse = function getBytesInUse(keys, callback) { // TODO: implement this callback(0); - } + }; - function genericErrorHandler(error) { - console.error('[%s]', STORAGE_NAME, error); - } + let api = { get, set, remove, clear, getBytesInUse, error: undefined }; + + let genericErrorHandler = function(ev) { + let error = ev.target && ev.target.error; + if ( error && error.name === 'QuotaExceededError' ) { + api.error = error.name; + } + console.error('[%s]', STORAGE_NAME, error && error.name); + }; function noopfn() { } - function processPendings() { - var cb; - while ( (cb = pending.shift()) ) { - cb(db); - } - } - - function getDb(callback) { - if ( pending === undefined ) { - return callback(); - } - if ( pending.length !== 0 ) { - return pending.push(callback); - } + let getDb = function getDb() { if ( db instanceof IDBDatabase ) { - return callback(db); + return Promise.resolve(db); + } + if ( db === null ) { + return Promise.resolve(null); + } + if ( pendingInitialization !== undefined ) { + return pendingInitialization; } - pending.push(callback); - if ( pending.length !== 1 ) { return; } // https://github.com/gorhill/uBlock/issues/3156 // I have observed that no event was fired in Tor Browser 7.0.7 + // medium security level after the request to open the database was @@ -125,90 +117,92 @@ vAPI.cacheStorage = (function() { // necessary when reading the `error` property because we are not // allowed to read this propery outside of event handlers in newer // implementation of IDBRequest (my understanding). - var req; - try { - req = indexedDB.open(STORAGE_NAME, 1); - if ( req.error ) { - console.log(req.error); + pendingInitialization = new Promise(resolve => { + let req; + try { + req = indexedDB.open(STORAGE_NAME, 1); + if ( req.error ) { + console.log(req.error); + req = undefined; + } + } catch(ex) { + } + if ( req === undefined ) { + pendingInitialization = undefined; + db = null; + resolve(null); + return; + } + req.onupgradeneeded = function(ev) { req = undefined; - } - } catch(ex) { - } - if ( req === undefined ) { - processPendings(); - pending = undefined; - return; - } - req.onupgradeneeded = function(ev) { - req = undefined; - db = ev.target.result; - db.onerror = db.onabort = genericErrorHandler; - var table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' }); - table.createIndex('value', 'value', { unique: false }); - }; - req.onsuccess = function(ev) { - req = undefined; - db = ev.target.result; - db.onerror = db.onabort = genericErrorHandler; - processPendings(); - }; - req.onerror = req.onblocked = function() { - req = undefined; - console.log(this.error); - processPendings(); - pending = undefined; - }; - } + let db = ev.target.result; + db.onerror = db.onabort = genericErrorHandler; + let table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' }); + table.createIndex('value', 'value', { unique: false }); + }; + req.onsuccess = function(ev) { + pendingInitialization = undefined; + req = undefined; + db = ev.target.result; + db.onerror = db.onabort = genericErrorHandler; + resolve(db); + }; + req.onerror = req.onblocked = function() { + pendingInitialization = undefined; + req = undefined; + db = null; + console.log(this.error); + resolve(null); + }; + }); + return pendingInitialization; + }; - function getFromDb(keys, store, callback) { + let getFromDb = function(keys, keystore, callback) { if ( typeof callback !== 'function' ) { return; } - if ( keys.length === 0 ) { return callback(store); } - var gotOne = function() { + if ( keys.length === 0 ) { return callback(keystore); } + let gotOne = function() { if ( typeof this.result === 'object' ) { - store[this.result.key] = this.result.value; + keystore[this.result.key] = this.result.value; } }; - getDb(function(db) { + getDb().then(( ) => { if ( !db ) { return callback(); } - var transaction = db.transaction(STORAGE_NAME); + let transaction = db.transaction(STORAGE_NAME); transaction.oncomplete = transaction.onerror = - transaction.onabort = function() { - return callback(store); - }; - var table = transaction.objectStore(STORAGE_NAME); - for ( var key of keys ) { - var req = table.get(key); + transaction.onabort = ( ) => callback(keystore); + let table = transaction.objectStore(STORAGE_NAME); + for ( let key of keys ) { + let req = table.get(key); req.onsuccess = gotOne; req.onerror = noopfn; req = undefined; } }); - } + }; - function getAllFromDb(callback) { + let getAllFromDb = function(callback) { if ( typeof callback !== 'function' ) { callback = noopfn; } - getDb(function(db) { + getDb().then(( ) => { if ( !db ) { return callback(); } - var output = {}; - var transaction = db.transaction(STORAGE_NAME); + let keystore = {}; + let transaction = db.transaction(STORAGE_NAME); transaction.oncomplete = transaction.onerror = - transaction.onabort = function() { - callback(output); - }; - var table = transaction.objectStore(STORAGE_NAME), + transaction.onabort = ( ) => callback(keystore); + let table = transaction.objectStore(STORAGE_NAME), req = table.openCursor(); req.onsuccess = function(ev) { - var cursor = ev.target.result; + let cursor = ev.target.result; if ( !cursor ) { return; } - output[cursor.key] = cursor.value; + keystore[cursor.key] = cursor.value; cursor.continue(); }; }); - } + }; // https://github.com/uBlockOrigin/uBlock-issues/issues/141 // Mind that IDBDatabase.transaction() and IDBObjectStore.put() @@ -216,20 +210,19 @@ vAPI.cacheStorage = (function() { // https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction // https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put - function putToDb(input, callback) { + let putToDb = function(keystore, callback) { if ( typeof callback !== 'function' ) { callback = noopfn; } - let keys = Object.keys(input); + let keys = Object.keys(keystore); if ( keys.length === 0 ) { return callback(); } - getDb(function(db) { + getDb().then(( ) => { if ( !db ) { return callback(); } - let finish = () => { - if ( callback !== undefined ) { - let cb = callback; - callback = undefined; - cb(); - } + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); }; try { let transaction = db.transaction(STORAGE_NAME, 'readwrite'); @@ -240,7 +233,7 @@ vAPI.cacheStorage = (function() { for ( let key of keys ) { let entry = {}; entry.key = key; - entry.value = input[key]; + entry.value = keystore[key]; table.put(entry); entry = undefined; } @@ -248,39 +241,64 @@ vAPI.cacheStorage = (function() { finish(); } }); - } + }; - function deleteFromDb(input, callback) { + let deleteFromDb = function(input, callback) { if ( typeof callback !== 'function' ) { callback = noopfn; } - var keys = Array.isArray(input) ? input.slice() : [ input ]; + let keys = Array.isArray(input) ? input.slice() : [ input ]; if ( keys.length === 0 ) { return callback(); } - getDb(function(db) { + getDb().then(db => { if ( !db ) { return callback(); } - var transaction = db.transaction(STORAGE_NAME, 'readwrite'); - transaction.oncomplete = - transaction.onerror = - transaction.onabort = callback; - var table = transaction.objectStore(STORAGE_NAME); - for ( var key of keys ) { - table.delete(key); + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + let transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = finish; + let table = transaction.objectStore(STORAGE_NAME); + for ( let key of keys ) { + table.delete(key); + } + } catch (ex) { + finish(); } }); - } + }; - function clearDb(callback) { + let clearDb = function(callback) { if ( typeof callback !== 'function' ) { callback = noopfn; } - getDb(function(db) { + getDb().then(db => { if ( !db ) { return callback(); } - var req = db.transaction(STORAGE_NAME, 'readwrite') - .objectStore(STORAGE_NAME) - .clear(); - req.onsuccess = req.onerror = callback; + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + let req = db.transaction(STORAGE_NAME, 'readwrite') + .objectStore(STORAGE_NAME) + .clear(); + req.onsuccess = req.onerror = finish; + } catch (ex) { + finish(); + } }); - } + }; + + // prime the db so that it's ready asap for next access. + getDb(noopfn); + + return api; }()); /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index 233fa303e..a514b1110 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -19,6 +19,8 @@ Home: https://github.com/gorhill/uBlock */ +/* global WebAssembly */ + 'use strict'; /******************************************************************************/ @@ -409,6 +411,247 @@ api.unregisterAssetSource = function(assetKey) { }); }; +/******************************************************************************* + + Experimental support for cache storage compression. + + For background information on the topic, see: + https://github.com/uBlockOrigin/uBlock-issues/issues/141#issuecomment-407737186 + +**/ + +let lz4Codec = (function() { + let lz4wasmInstance; + let pendingInitialization; + let textEncoder, textDecoder; + let ttlCount = 0; + let ttlTimer; + + const ttlDelay = 60 * 1000; + + let init = function() { + if ( + lz4wasmInstance === null || + WebAssembly instanceof Object === false || + typeof WebAssembly.instantiateStreaming !== 'function' + ) { + lz4wasmInstance = null; + return Promise.resolve(null); + } + if ( lz4wasmInstance instanceof WebAssembly.Instance ) { + return Promise.resolve(lz4wasmInstance); + } + if ( pendingInitialization === undefined ) { + pendingInitialization = WebAssembly.instantiateStreaming( + fetch('lib/lz4-block-codec.wasm', { mode: 'same-origin' }) + ).then(result => { + pendingInitialization = undefined; + lz4wasmInstance = result && result.instance || null; + }); + pendingInitialization.catch(( ) => { + lz4wasmInstance = null; + }); + } + return pendingInitialization; + }; + + // We can't shrink memory usage of wasm instances, and in the current + // case memory usage can grow to a significant amount given that + // a single contiguous memory buffer is required to accommodate both + // input and output data. Thus a time-to-live implementation which + // will cause the wasm instance to be forgotten after enough time + // elapse without the instance being used. + + let destroy = function() { + console.info( + 'uBO: freeing lz4-block-codec.wasm instance (memory.buffer = %d kB)', + lz4wasmInstance.exports.memory.buffer.byteLength >>> 10 + ); + lz4wasmInstance = undefined; + textEncoder = textDecoder = undefined; + ttlCount = 0; + ttlTimer = undefined; + }; + + let ttlManage = function(count) { + if ( ttlTimer !== undefined ) { + clearTimeout(ttlTimer); + ttlTimer = undefined; + } + ttlCount += count; + if ( ttlCount > 0 ) { return; } + if ( lz4wasmInstance === null ) { return; } + ttlTimer = vAPI.setTimeout(destroy, ttlDelay); + }; + + let growMemoryTo = function(byteLength) { + let lz4api = lz4wasmInstance.exports; + let neededByteLength = lz4api.getLinearMemoryOffset() + byteLength; + let pageCountBefore = lz4api.memory.buffer.byteLength >>> 16; + let pageCountAfter = (neededByteLength + 65535) >>> 16; + if ( pageCountAfter > pageCountBefore ) { + lz4api.memory.grow(pageCountAfter - pageCountBefore); + } + return lz4api.memory; + }; + + let resolveEncodedValue = function(resolve, key, value) { + let t0 = window.performance.now(); + let lz4api = lz4wasmInstance.exports; + let mem0 = lz4api.getLinearMemoryOffset(); + let memory = growMemoryTo(mem0 + 65536 * 4); + let hashTable = new Int32Array(memory.buffer, mem0, 65536); + hashTable.fill(-65536, 0, 65536); + let hashTableSize = hashTable.byteLength; + if ( textEncoder === undefined ) { + textEncoder = new TextEncoder(); + } + let inputArray = textEncoder.encode(value); + let inputSize = inputArray.byteLength; + let memSize = + hashTableSize + + inputSize + + 8 + lz4api.lz4BlockEncodeBound(inputSize); + memory = growMemoryTo(memSize); + let inputMem = new Uint8Array( + memory.buffer, + mem0 + hashTableSize, + inputSize + ); + inputMem.set(inputArray); + let outputSize = lz4api.lz4BlockEncode( + mem0 + hashTableSize, + inputSize, + mem0 + hashTableSize + inputSize + 8 + ); + if ( outputSize === 0 ) { resolve(value); } + let outputMem = new Uint8Array( + memory.buffer, + mem0 + hashTableSize + inputSize, + 8 + outputSize + ); + outputMem[0] = 0x18; + outputMem[1] = 0x4D; + outputMem[2] = 0x22; + outputMem[3] = 0x04; + outputMem[4] = (inputSize >>> 0) & 0xFF; + outputMem[5] = (inputSize >>> 8) & 0xFF; + outputMem[6] = (inputSize >>> 16) & 0xFF; + outputMem[7] = (inputSize >>> 24) & 0xFF; + console.info( + 'uBO: [%s] compressed %d bytes into %d bytes in %s ms', + key, + inputSize, + outputSize, + (window.performance.now() - t0).toFixed(2) + ); + resolve(new Blob([ outputMem ])); + }; + + let resolveDecodedValue = function(resolve, ev, key, value) { + let inputBuffer = ev.target.result; + if ( inputBuffer instanceof ArrayBuffer === false ) { + return resolve(value); + } + let t0 = window.performance.now(); + let metadata = new Uint8Array(inputBuffer, 0, 8); + if ( + metadata[0] !== 0x18 || + metadata[1] !== 0x4D || + metadata[2] !== 0x22 || + metadata[3] !== 0x04 + ) { + return resolve(value); + } + let inputSize = inputBuffer.byteLength - 8; + let outputSize = + (metadata[4] << 0) | + (metadata[5] << 8) | + (metadata[6] << 16) | + (metadata[7] << 24); + let lz4api = lz4wasmInstance.exports; + let mem0 = lz4api.getLinearMemoryOffset(); + let memSize = inputSize + outputSize; + let memory = growMemoryTo(memSize); + let inputArea = new Uint8Array( + memory.buffer, + mem0, + inputSize + ); + inputArea.set(new Uint8Array(inputBuffer, 8, inputSize)); + outputSize = lz4api.lz4BlockDecode(inputSize); + if ( outputSize === 0 ) { + return resolve(value); + } + let outputArea = new Uint8Array( + memory.buffer, + mem0 + inputSize, + outputSize + ); + if ( textDecoder === undefined ) { + textDecoder = new TextDecoder(); + } + value = textDecoder.decode(outputArea); + console.info( + 'uBO: [%s] decompressed %d bytes into %d bytes in %s ms', + key, + inputSize, + outputSize, + (window.performance.now() - t0).toFixed(2) + ); + resolve(value); + }; + + let encodeValue = function(key, value) { + if ( !lz4wasmInstance ) { + return Promise.resolve(value); + } + return new Promise(resolve => { + resolveEncodedValue(resolve, key, value); + }); + }; + + let decodeValue = function(key, value) { + if ( !lz4wasmInstance ) { + return Promise.resolve(value); + } + return new Promise(resolve => { + let blobReader = new FileReader(); + blobReader.onloadend = ev => { + resolveDecodedValue(resolve, ev, key, value); + }; + blobReader.readAsArrayBuffer(value); + }); + }; + + return { + encode: function(key, value) { + if ( typeof value !== 'string' || value.length < 4096 ) { + return Promise.resolve(value); + } + ttlManage(1); + return init().then(( ) => { + return encodeValue(key, value); + }).then(result => { + ttlManage(-1); + return result; + }); + }, + decode: function(key, value) { + if ( value instanceof Blob === false ) { + return Promise.resolve(value); + } + ttlManage(1); + return init().then(( ) => { + return decodeValue(key, value); + }).then(result => { + ttlManage(-1); + return result; + }); + } + }; +})(); + /******************************************************************************* The purpose of the asset cache registry is to keep track of all assets @@ -472,26 +715,32 @@ var saveAssetCacheRegistry = (function() { var assetCacheRead = function(assetKey, callback) { let internalKey = 'cache/' + assetKey; - let reportBack = function(content, err) { + let reportBack = function(content) { + if ( content instanceof Blob ) { content = ''; } let details = { assetKey: assetKey, content: content }; - if ( err ) { details.error = err; } + if ( content === '' ) { details.error = 'E_NOTFOUND'; } callback(details); }; let onAssetRead = function(bin) { if ( bin instanceof Object === false || - stringIsNotEmpty(bin[internalKey]) === false + bin.hasOwnProperty(internalKey) === false ) { - return reportBack('', 'E_NOTFOUND'); + return reportBack(''); } let entry = assetCacheRegistry[assetKey]; if ( entry === undefined ) { - return reportBack('', 'E_NOTFOUND'); + return reportBack(''); } entry.readTime = Date.now(); saveAssetCacheRegistry(true); - reportBack(bin[internalKey]); + if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) { + return reportBack(bin[internalKey]); + } + lz4Codec.decode(internalKey, bin[internalKey]).then(result => { + reportBack(result); + }); }; let onReady = function() { @@ -502,8 +751,8 @@ var assetCacheRead = function(assetKey, callback) { }; var assetCacheWrite = function(assetKey, details, callback) { - var internalKey = 'cache/' + assetKey; - var content = ''; + let internalKey = 'cache/' + assetKey; + let content = ''; if ( typeof details === 'string' ) { content = details; } else if ( details instanceof Object ) { @@ -514,16 +763,19 @@ var assetCacheWrite = function(assetKey, details, callback) { return assetCacheRemove(assetKey, callback); } - var reportBack = function(content) { - var details = { assetKey: assetKey, content: content }; + let reportBack = function(content) { + let bin = { assetCacheRegistry: assetCacheRegistry }; + bin[internalKey] = content; + vAPI.cacheStorage.set(bin); + let details = { assetKey: assetKey, content: content }; if ( typeof callback === 'function' ) { callback(details); } fireNotification('after-asset-updated', details); }; - var onReady = function() { - var entry = assetCacheRegistry[assetKey]; + let onReady = function() { + let entry = assetCacheRegistry[assetKey]; if ( entry === undefined ) { entry = assetCacheRegistry[assetKey] = {}; } @@ -531,10 +783,12 @@ var assetCacheWrite = function(assetKey, details, callback) { if ( details instanceof Object && typeof details.url === 'string' ) { entry.remoteURL = details.url; } - var bin = { assetCacheRegistry: assetCacheRegistry }; - bin[internalKey] = content; - vAPI.cacheStorage.set(bin); - reportBack(content); + if ( µBlock.hiddenSettings.cacheStorageCompression !== true ) { + return reportBack(content); + } + lz4Codec.encode(internalKey, content).then(result => { + reportBack(result); + }); }; getAssetCacheRegistry(onReady); }; diff --git a/src/js/background.js b/src/js/background.js index d711a615a..ae683e508 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2018 Raymond Hill + Copyright (C) 2014-present Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -42,6 +42,7 @@ var µBlock = (function() { // jshint ignore:line assetFetchTimeout: 30, autoUpdateAssetFetchPeriod: 120, autoUpdatePeriod: 7, + cacheStorageCompression: false, debugScriptlets: false, ignoreRedirectFilters: false, ignoreScriptInjectFilters: false, @@ -138,7 +139,7 @@ var µBlock = (function() { // jshint ignore:line // Read-only systemSettings: { compiledMagic: 3, // Increase when compiled format changes - selfieMagic: 3 // Increase when selfie format changes + selfieMagic: 4 // Increase when selfie format changes }, restoreBackupSettings: { diff --git a/src/js/storage.js b/src/js/storage.js index f8c429db7..d96a5dc01 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1025,13 +1025,13 @@ let create = function() { timer = null; - let selfie = { + let selfie = JSON.stringify({ magic: µb.systemSettings.selfieMagic, - availableFilterLists: JSON.stringify(µb.availableFilterLists), - staticNetFilteringEngine: JSON.stringify(µb.staticNetFilteringEngine.toSelfie()), - redirectEngine: JSON.stringify(µb.redirectEngine.toSelfie()), - staticExtFilteringEngine: JSON.stringify(µb.staticExtFilteringEngine.toSelfie()) - }; + availableFilterLists: µb.availableFilterLists, + staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), + redirectEngine: µb.redirectEngine.toSelfie(), + staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie() + }); vAPI.cacheStorage.set({ selfie: selfie }); }; @@ -1039,16 +1039,25 @@ vAPI.cacheStorage.get('selfie', function(bin) { if ( bin instanceof Object === false || - bin.selfie instanceof Object === false || - bin.selfie.magic !== µb.systemSettings.selfieMagic || - bin.selfie.redirectEngine === undefined + typeof bin.selfie !== 'string' ) { return callback(false); } - µb.availableFilterLists = JSON.parse(bin.selfie.availableFilterLists); - µb.staticNetFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticNetFilteringEngine)); - µb.redirectEngine.fromSelfie(JSON.parse(bin.selfie.redirectEngine)); - µb.staticExtFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticExtFilteringEngine)); + let selfie; + try { + selfie = JSON.parse(bin.selfie); + } catch(ex) { + } + if ( + selfie instanceof Object === false || + selfie.magic !== µb.systemSettings.selfieMagic + ) { + return callback(false); + } + µb.availableFilterLists = selfie.availableFilterLists; + µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); + µb.redirectEngine.fromSelfie(selfie.redirectEngine); + µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine); callback(true); }); }; diff --git a/src/lib/lz4-block-codec.wasm b/src/lib/lz4-block-codec.wasm new file mode 100644 index 000000000..becc155cc Binary files /dev/null and b/src/lib/lz4-block-codec.wasm differ