From beb79330164bb285b94874f86b5d007ec3ae47c1 Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 29 Aug 2017 18:32:00 -0400 Subject: [PATCH] fix #2925 --- .jshintrc | 3 +- platform/webext/background.html | 1 + platform/webext/from-legacy.js | 9 +- platform/webext/vapi-cachestorage.js | 323 +++++++++++++++++---------- src/background.html | 1 + src/js/start.js | 20 +- tools/make-webext-hybrid.sh | 45 ++-- tools/make-webext.sh | 33 +-- 8 files changed, 264 insertions(+), 171 deletions(-) diff --git a/.jshintrc b/.jshintrc index de72266b3..fd2d65d0c 100644 --- a/.jshintrc +++ b/.jshintrc @@ -4,7 +4,8 @@ "eqeqeq": true, "esnext": true, "globals": { - "chrome": false, + "browser": false, // global variable in Firefox, Edge + "chrome": false, // global variable in Chromium, Chrome, Opera "Components": false, // global variable in Firefox "safari": false, "self": false, diff --git a/platform/webext/background.html b/platform/webext/background.html index 659797740..1f84280da 100644 --- a/platform/webext/background.html +++ b/platform/webext/background.html @@ -10,6 +10,7 @@ + diff --git a/platform/webext/from-legacy.js b/platform/webext/from-legacy.js index d69f5d2d2..cf0fd4927 100644 --- a/platform/webext/from-legacy.js +++ b/platform/webext/from-legacy.js @@ -28,6 +28,7 @@ (function() { let µb = µBlock; let migratedKeys = new Set(); + let reCacheStorageKeys = /^(?:assetCacheRegistry|assetSourceRegistry|cache\/.+|selfie)$/; let migrateAll = function(callback) { let migrateKeyValue = function(details, callback) { @@ -40,7 +41,11 @@ migratedKeys.add(details.key); let bin = {}; bin[details.key] = JSON.parse(details.value); - self.browser.storage.local.set(bin, callback); + if ( reCacheStorageKeys.test(details.key) ) { + vAPI.cacheStorage.set(bin, callback); + } else { + vAPI.storage.set(bin, callback); + } }; let migrateNext = function() { @@ -62,7 +67,7 @@ self.browser.runtime.sendMessage({ what: 'webext:storageMigrateDone' }); return callback(); } - self.browser.storage.local.set({ legacyStorageMigrated: true }); + vAPI.storage.set({ legacyStorageMigrated: true }); migrateNext(); }); }; diff --git a/platform/webext/vapi-cachestorage.js b/platform/webext/vapi-cachestorage.js index 96a6cc647..ada79a49d 100644 --- a/platform/webext/vapi-cachestorage.js +++ b/platform/webext/vapi-cachestorage.js @@ -19,6 +19,8 @@ Home: https://github.com/gorhill/uBlock */ +/* global indexedDB, IDBDatabase */ + 'use strict'; /******************************************************************************/ @@ -29,64 +31,50 @@ // Commit author: https://github.com/nikrolls // Commit message: "Implement cacheStorage using IndexedDB" +// The original imported code has been subsequently modified as it was not +// compatible with Firefox. +// (a Promise thing, see https://github.com/dfahlander/Dexie.js/issues/317) +// Furthermore, code to migrate from browser.storage.local to vAPI.cacheStorage +// has been added, for seamless migration of cache-related entries into +// indexedDB. + vAPI.cacheStorage = (function() { - const STORAGE_NAME = 'uBlockStorage'; - const db = getDb(); + const STORAGE_NAME = 'uBlock0CacheStorage'; + var db; + var pending = []; - return {get, set, remove, clear, getBytesInUse}; + // prime the db so that it's ready asap for next access. + getDb(noopfn); - function get(key, callback) { - let promise; + return { get, set, remove, clear, getBytesInUse }; - if (key === null) { - promise = getAllFromDb(); - } else if (typeof key === 'string') { - promise = getFromDb(key).then(result => [result]); - } else if (typeof key === 'object') { - const keys = Array.isArray(key) ? [].concat(key) : Object.keys(key); - const requests = keys.map(key => getFromDb(key)); - promise = Promise.all(requests); - } else { - promise = Promise.resolve([]); + function get(input, callback) { + if ( typeof callback !== 'function' ) { return; } + if ( input === null ) { + return getAllFromDb(callback); } - - promise.then(results => convertResultsToHash(results)) - .then((converted) => { - if (typeof key === 'object' && !Array.isArray(key)) { - callback(Object.assign({}, key, converted)); - } else { - callback(converted); - } - }) - .catch((e) => { - browser.runtime.lastError = e; - callback(null); - }); + var toRead, output = {}; + if ( typeof input === 'string' ) { + toRead = [ input ]; + } else if ( Array.isArray(input) ) { + toRead = input; + } else /* if ( typeof input === 'object' ) */ { + toRead = Object.keys(input); + output = input; + } + return getFromDb(toRead, output, callback); } - function set(data, callback) { - const requests = Object.keys(data).map( - key => putToDb(key, data[key]) - ); - - Promise.all(requests) - .then(() => callback && callback()) - .catch(e => (browser.runtime.lastError = e, callback && callback())); + function set(input, callback) { + putToDb(input, callback); } function remove(key, callback) { - const keys = [].concat(key); - const requests = keys.map(key => deleteFromDb(key)); - - Promise.all(requests) - .then(() => callback && callback()) - .catch(e => (browser.runtime.lastError = e, callback && callback())); + deleteFromDb(key, callback); } function clear(callback) { - clearDb() - .then(() => callback && callback()) - .catch(e => (browser.runtime.lastError = e, callback && callback())); + clearDb(callback); } function getBytesInUse(keys, callback) { @@ -94,87 +82,180 @@ vAPI.cacheStorage = (function() { callback(0); } - function getDb() { - const openRequest = window.indexedDB.open(STORAGE_NAME, 1); - openRequest.onupgradeneeded = upgradeSchema; - return convertToPromise(openRequest).then((db) => { - db.onerror = console.error; - return db; - }); + function genericErrorHandler(error) { + console.error('[uBlock0 cacheStorage]', error); } - function upgradeSchema(event) { - const db = event.target.result; - db.onerror = (error) => console.error('[storage] Error updating IndexedDB schema:', error); - - const objectStore = db.createObjectStore(STORAGE_NAME, {keyPath: 'key'}); - objectStore.createIndex('value', 'value', {unique: false}); + function noopfn() { } - function getNewTransaction(mode = 'readonly') { - return db.then(db => db.transaction(STORAGE_NAME, mode).objectStore(STORAGE_NAME)); - } - - function getFromDb(key) { - return getNewTransaction() - .then(store => store.get(key)) - .then(request => convertToPromise(request)); - } - - function getAllFromDb() { - return getNewTransaction() - .then((store) => { - return new Promise((resolve, reject) => { - const request = store.openCursor(); - const output = []; - - request.onsuccess = (event) => { - const cursor = event.target.result; - if (cursor) { - output.push(cursor.value); - cursor.continue(); - } else { - resolve(output); - } - }; - - request.onerror = reject; - }); - }); - } - - function putToDb(key, value) { - return getNewTransaction('readwrite') - .then(store => store.put({key, value})) - .then(request => convertToPromise(request)); - } - - function deleteFromDb(key) { - return getNewTransaction('readwrite') - .then(store => store.delete(key)) - .then(request => convertToPromise(request)); - } - - function clearDb() { - return getNewTransaction('readwrite') - .then(store => store.clear()) - .then(request => convertToPromise(request)); - } - - function convertToPromise(eventTarget) { - return new Promise((resolve, reject) => { - eventTarget.onsuccess = () => resolve(eventTarget.result); - eventTarget.onerror = reject; - }); - } - - function convertResultsToHash(results) { - return results.reduce((output, item) => { - if (item) { - output[item.key] = item.value; + function getDb(callback) { + if ( pending.length !== 0 ) { + pending.push(callback); + return; + } + if ( db instanceof IDBDatabase ) { + return callback(db); + } + pending.push(callback); + if ( pending.length !== 1 ) { return; } + var req = indexedDB.open(STORAGE_NAME, 1); + req.onupgradeneeded = function(ev) { + db = ev.target.result; + db.onerror = genericErrorHandler; + var table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' }); + table.createIndex('value', 'value', { unique: false }); + }; + req.onsuccess = function(ev) { + db = ev.target.result; + db.onerror = genericErrorHandler; + var cb; + while ( (cb = pending.shift()) ) { + cb(db); } - return output; - }, {}); + }; + req.onerror = function(ev) { + console.log(ev); + var cb; + while ( (cb = pending.shift()) ) { + cb(db); + } + }; + } + + function getFromDb(keys, store, callback) { + if ( typeof callback !== 'function' ) { return; } + if ( keys.length === 0 ) { return callback(store); } + var notfoundKeys = new Set(keys); + var gotOne = function() { + if ( typeof this.result === 'object' ) { + store[this.result.key] = this.result.value; + notfoundKeys.delete(this.result.key); + } + }; + getDb(function(db) { + if ( !db ) { return callback(); } + var transaction = db.transaction(STORAGE_NAME); + transaction.oncomplete = transaction.onerror = function() { + if ( notfoundKeys.size === 0 ) { + return callback(store); + } + maybeMigrate(Array.from(notfoundKeys), store, callback); + }; + var table = transaction.objectStore(STORAGE_NAME); + for ( var key of keys ) { + var req = table.get(key); + req.onsuccess = gotOne; + req.onerror = noopfn; + } + }); + } + + // Migrate from storage API + // TODO: removes once all users are migrated to the new cacheStorage. + function maybeMigrate(keys, store, callback) { + var toMigrate = new Set(), + i = keys.length; + while ( i-- ) { + var key = keys[i]; + toMigrate.add(key); + // If migrating a compiled list, also migrate the non-compiled + // counterpart. + if ( /^cache\/compiled\//.test(key) ) { + toMigrate.add(key.replace('/compiled', '')); + } + } + vAPI.storage.get(Array.from(toMigrate), function(bin) { + if ( bin instanceof Object === false ) { + return callback(store); + } + var migratedKeys = Object.keys(bin); + if ( migratedKeys.length === 0 ) { + return callback(store); + } + var i = migratedKeys.length; + while ( i-- ) { + var key = migratedKeys[i]; + store[key] = bin[key]; + } + vAPI.storage.remove(migratedKeys); + vAPI.cacheStorage.set(bin); + callback(store); + }); + } + + function getAllFromDb(callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + getDb(function(db) { + if ( !db ) { return callback(); } + var output = {}; + var transaction = db.transaction(STORAGE_NAME); + transaction.oncomplete = transaction.onerror = function() { + callback(output); + }; + var table = transaction.objectStore(STORAGE_NAME), + req = table.openCursor(); + req.onsuccess = function(ev) { + var cursor = ev.target.result; + if ( !cursor ) { return; } + output[cursor.key] = cursor.value; + cursor.continue(); + }; + }); + } + + function putToDb(input, callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + var keys = Object.keys(input); + if ( keys.length === 0 ) { return callback(); } + getDb(function(db) { + if ( !db ) { return callback(); } + var transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = transaction.onerror = callback; + var table = transaction.objectStore(STORAGE_NAME), + entry = {}; + for ( var key of keys ) { + entry.key = key; + entry.value = input[key]; + table.put(entry); + } + }); + } + + function deleteFromDb(input, callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + var keys = Array.isArray(input) ? input.slice() : [ input ]; + if ( keys.length === 0 ) { return callback(); } + getDb(function(db) { + if ( !db ) { return callback(); } + var transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = transaction.onerror = callback; + var table = transaction.objectStore(STORAGE_NAME); + for ( var key of keys ) { + table.delete(key); + } + }); + // TODO: removes once all users are migrated to the new cacheStorage. + vAPI.storage.remove(keys); + } + + function clearDb(callback) { + if ( typeof callback !== 'function' ) { + callback = noopfn; + } + getDb(function(db) { + if ( !db ) { return callback(); } + var req = db.transaction(STORAGE_NAME, 'readwrite') + .objectStore(STORAGE_NAME) + .clear(); + req.onsuccess = req.onerror = callback; + }); } }()); diff --git a/src/background.html b/src/background.html index 219f0390b..4ca1e39b7 100644 --- a/src/background.html +++ b/src/background.html @@ -10,6 +10,7 @@ + diff --git a/src/js/start.js b/src/js/start.js index 4fabbbac2..44f63bf2b 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -127,7 +127,10 @@ var onVersionReady = function(lastVersion) { /******************************************************************************/ var onSelfieReady = function(selfie) { - if ( selfie === null || selfie.magic !== µb.systemSettings.selfieMagic ) { + if ( + selfie instanceof Object === false || + selfie.magic !== µb.systemSettings.selfieMagic + ) { return false; } if ( publicSuffixList.fromSelfie(selfie.publicSuffixList) !== true ) { @@ -221,13 +224,13 @@ var onFirstFetchReady = function(fetched) { onNetWhitelistReady(fetched.netWhitelist); onVersionReady(fetched.version); - // If we have a selfie, skip loading PSL, filters - if ( onSelfieReady(fetched.selfie) ) { - onAllReady(); - return; - } - - µb.loadPublicSuffixList(onPSLReady); + // If we have a selfie, skip loading PSL, filter lists + vAPI.cacheStorage.get('selfie', function(bin) { + if ( bin instanceof Object && onSelfieReady(bin.selfie) ) { + return onAllReady(); + } + µb.loadPublicSuffixList(onPSLReady); + }); }; /******************************************************************************/ @@ -266,7 +269,6 @@ var onSelectedFilterListsLoaded = function() { 'lastBackupFile': '', 'lastBackupTime': 0, 'netWhitelist': µb.netWhitelistDefault, - 'selfie': null, 'selfieMagic': '', 'version': '0.0.0.0' }; diff --git a/tools/make-webext-hybrid.sh b/tools/make-webext-hybrid.sh index fb8896369..728570410 100755 --- a/tools/make-webext-hybrid.sh +++ b/tools/make-webext-hybrid.sh @@ -11,31 +11,32 @@ mkdir -p $DES/webextension bash ./tools/make-assets.sh $DES/webextension -cp -R src/css $DES/webextension/ -cp -R src/img $DES/webextension/ -cp -R src/js $DES/webextension/ -cp -R src/lib $DES/webextension/ -cp -R src/_locales $DES/webextension/ -cp -R $DES/webextension/_locales/nb $DES/webextension/_locales/no -cp src/*.html $DES/webextension/ -cp platform/chromium/*.js $DES/webextension/js/ -cp -R platform/chromium/img $DES/webextension/ -cp platform/chromium/*.html $DES/webextension/ -cp platform/chromium/*.json $DES/webextension/ -cp LICENSE.txt $DES/webextension/ +cp -R src/css $DES/webextension/ +cp -R src/img $DES/webextension/ +cp -R src/js $DES/webextension/ +cp -R src/lib $DES/webextension/ +cp -R src/_locales $DES/webextension/ +cp -R $DES/webextension/_locales/nb $DES/webextension/_locales/no +cp src/*.html $DES/webextension/ +cp platform/chromium/*.js $DES/webextension/js/ +cp -R platform/chromium/img $DES/webextension/ +cp platform/chromium/*.html $DES/webextension/ +cp platform/chromium/*.json $DES/webextension/ +cp LICENSE.txt $DES/webextension/ -cp platform/webext/manifest.json $DES/webextension/ -cp platform/webext/background.html $DES/webextension/ -cp platform/webext/options_ui.html $DES/webextension/ -cp platform/webext/polyfill.js $DES/webextension/js/ -cp platform/webext/vapi-usercss.js $DES/webextension/js/ -cp platform/webext/from-legacy.js $DES/webextension/js/ +cp platform/webext/manifest.json $DES/webextension/ +cp platform/webext/background.html $DES/webextension/ +cp platform/webext/options_ui.html $DES/webextension/ +cp platform/webext/polyfill.js $DES/webextension/js/ +cp platform/webext/vapi-usercss.js $DES/webextension/js/ +cp platform/webext/vapi-cachestorage.js $DES/webextension/js/ +cp platform/webext/from-legacy.js $DES/webextension/js/ rm $DES/webextension/js/options_ui.js -cp platform/webext/bootstrap.js $DES/ -cp platform/webext/chrome.manifest $DES/ -cp platform/webext/install.rdf $DES/ -mv $DES/webextension/img/icon_128.png $DES/icon.png +cp platform/webext/bootstrap.js $DES/ +cp platform/webext/chrome.manifest $DES/ +cp platform/webext/install.rdf $DES/ +mv $DES/webextension/img/icon_128.png $DES/icon.png echo "*** uBlock0.webext-hybrid: Generating meta..." python tools/make-webext-hybrid-meta.py $DES/ diff --git a/tools/make-webext.sh b/tools/make-webext.sh index 38f183fc8..dc1c579a3 100755 --- a/tools/make-webext.sh +++ b/tools/make-webext.sh @@ -11,23 +11,24 @@ mkdir -p $DES bash ./tools/make-assets.sh $DES -cp -R src/css $DES/ -cp -R src/img $DES/ -cp -R src/js $DES/ -cp -R src/lib $DES/ -cp -R src/_locales $DES/ -cp -R $DES/_locales/nb $DES/_locales/no -cp src/*.html $DES/ -cp platform/chromium/*.js $DES/js/ -cp -R platform/chromium/img $DES/ -cp platform/chromium/*.html $DES/ -cp platform/chromium/*.json $DES/ -cp LICENSE.txt $DES/ +cp -R src/css $DES/ +cp -R src/img $DES/ +cp -R src/js $DES/ +cp -R src/lib $DES/ +cp -R src/_locales $DES/ +cp -R $DES/_locales/nb $DES/_locales/no +cp src/*.html $DES/ +cp platform/chromium/*.js $DES/js/ +cp -R platform/chromium/img $DES/ +cp platform/chromium/*.html $DES/ +cp platform/chromium/*.json $DES/ +cp LICENSE.txt $DES/ -cp platform/webext/manifest.json $DES/ -cp platform/webext/options_ui.html $DES/ -cp platform/webext/polyfill.js $DES/js/ -cp platform/webext/vapi-usercss.js $DES/js/ +cp platform/webext/manifest.json $DES/ +cp platform/webext/options_ui.html $DES/ +cp platform/webext/polyfill.js $DES/js/ +cp platform/webext/vapi-cachestorage.js $DES/js/ +cp platform/webext/vapi-usercss.js $DES/js/ rm $DES/js/options_ui.js echo "*** uBlock0.webext: Generating meta..."