Allow use of browser.storage.local as cache storage backend in Firefox

Related issue:
- https://github.com/uBlockOrigin/uBlock-issues/issues/409

By default `indexedDB` is used in Firefox for purpose of cache storage
backend.

This commit allows to force the use of `browser.storage.local` instead
as cache storage backend. For this to happen, set `cacheStorageAPI` to
`browser.storage.local` in advanced settings.

Additionally, should `indexedDB` not be available for whatever reason,
uBO will automatically fallback to `browser.storage.local`.

platform/chromium/vapi-background.js

@@ -91,7 +91,6 @@ vAPI.app.restart = function() {
 // chrome.storage.local.get(null, function(bin){ console.debug('%o', bin); });
 
 vAPI.storage = chrome.storage.local;
-vAPI.cacheStorage = chrome.storage.local;
 
 /******************************************************************************/
 /******************************************************************************/

src/js/assets.js

@@ -396,13 +396,17 @@ const updateAssetSourceRegistry = function(json, silent) {
 
 const getAssetSourceRegistry = function(callback) {
     if ( assetSourceRegistryPromise === undefined ) {
-        assetSourceRegistryPromise = new Promise(resolve => {
+        assetSourceRegistryPromise = µBlock.cacheStorage.get(
-        // start of executor
+            'assetSourceRegistry'
-        µBlock.cacheStorage.get('assetSourceRegistry', bin => {
+        ).then(bin => {
             if (
-                bin instanceof Object === false ||
+                bin instanceof Object &&
-                bin.assetSourceRegistry instanceof Object === false
+                bin.assetSourceRegistry instanceof Object
             ) {
+                assetSourceRegistry = bin.assetSourceRegistry;
+                return;
+            }
+            return new Promise(resolve => {
                 api.fetchText(
                     µBlock.assetsBootstrapLocation || 'assets/assets.json',
                     details => {
@@ -410,12 +414,7 @@ const getAssetSourceRegistry = function(callback) {
                         resolve();
                     }
                 );
-                return;
+            });
-            }
-            assetSourceRegistry = bin.assetSourceRegistry;
-            resolve();
-        });
-        // end of executor
         });
     }
 
@@ -451,16 +450,15 @@ let assetCacheRegistry = {};
 
 const getAssetCacheRegistry = function() {
     if ( assetCacheRegistryPromise === undefined ) {
-        assetCacheRegistryPromise = new Promise(resolve => {
+        assetCacheRegistryPromise = µBlock.cacheStorage.get(
-            µBlock.cacheStorage.get('assetCacheRegistry', bin => {
+            'assetCacheRegistry'
-                if (
+        ).then(bin => {
-                    bin instanceof Object &&
+            if (
-                    bin.assetCacheRegistry instanceof Object
+                bin instanceof Object &&
-                ) {
+                bin.assetCacheRegistry instanceof Object
-                    assetCacheRegistry = bin.assetCacheRegistry;
+            ) {
-                }
+                assetCacheRegistry = bin.assetCacheRegistry;
-                resolve();
+            }
-            });
         });
     }
 
@@ -509,8 +507,11 @@ const assetCacheRead = function(assetKey, callback) {
         reportBack(bin[internalKey]);
     };
 
-    getAssetCacheRegistry().then(( ) => {
+    Promise.all([
-        µBlock.cacheStorage.get(internalKey, onAssetRead);
+        getAssetCacheRegistry(),
+        µBlock.cacheStorage.get(internalKey),
+    ]).then(results => {
+        onAssetRead(results[1]);
     });
 };
 
@@ -537,17 +538,16 @@ const assetCacheWrite = function(assetKey, details, callback) {
             entry.remoteURL = details.url;
         }
         µBlock.cacheStorage.set(
-            { [internalKey]: content },
+            { [internalKey]: content }
-            details => {
+        ).then(details => {
-                if (
+            if (
-                    details instanceof Object &&
+                details instanceof Object &&
-                    typeof details.bytesInUse === 'number'
+                typeof details.bytesInUse === 'number'
-                ) {
+            ) {
-                    entry.byteLength = details.bytesInUse;
+                entry.byteLength = details.bytesInUse;
-                }
-                saveAssetCacheRegistry(true);
             }
-        );
+            saveAssetCacheRegistry(true);
+        });
         const result = { assetKey, content };
         if ( typeof callback === 'function' ) {
             callback(result);
@@ -556,9 +556,7 @@ const assetCacheWrite = function(assetKey, details, callback) {
         fireNotification('after-asset-updated', result);
     };
 
-    getAssetCacheRegistry().then(( ) => {
+    getAssetCacheRegistry().then(( ) => onReady());
-        µBlock.cacheStorage.get(internalKey, onReady);
-    });
 };
 
 const assetCacheRemove = function(pattern, callback) {

src/js/cachestorage.js

@@ -34,129 +34,155 @@
 // The original imported code has been subsequently modified as it was not
 // compatible with Firefox.
 // (a Promise thing, see https://github.com/dfahlander/Dexie.js/issues/317)
-// Furthermore, code to migrate from browser.storage.local to vAPI.cacheStorage
+// Furthermore, code to migrate from browser.storage.local to vAPI.storage
 // has been added, for seamless migration of cache-related entries into
 // indexedDB.
 
+// https://bugzilla.mozilla.org/show_bug.cgi?id=1371255
+//   Firefox-specific: we use indexedDB because chrome.storage.local() has
+//   poor performance in Firefox.
+// https://github.com/uBlockOrigin/uBlock-issues/issues/328
+//   Use IndexedDB for Chromium as well, to take advantage of LZ4
+//   compression.
+// https://github.com/uBlockOrigin/uBlock-issues/issues/399
+//   Revert Chromium support of IndexedDB, use advanced setting to force
+//   IndexedDB.
+// https://github.com/uBlockOrigin/uBlock-issues/issues/409
+//   Allow forcing the use of webext storage on Firefox.
+
 µBlock.cacheStorage = (function() {
 
     const STORAGE_NAME = 'uBlock0CacheStorage';
 
-    // https://bugzilla.mozilla.org/show_bug.cgi?id=1371255
+    // Default to webext storage. Wrapped into promises if the API does not
-    //   Firefox-specific: we use indexedDB because chrome.storage.local() has
+    // support returning promises.
-    //   poor performance in Firefox.
+    const promisified = (function() {
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/328
+        try {
-    //   Use IndexedDB for Chromium as well, to take advantage of LZ4
+            return browser.storage.local.get('_') instanceof Promise;
-    //   compression.
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/399
-    //   Revert Chromium support of IndexedDB, use advanced setting to force
-    //   IndexedDB.
-    if (
-        vAPI.webextFlavor.soup.has('firefox') === false &&
-        µBlock.hiddenSettings.cacheStorageAPI !== 'indexedDB'
-    ) {
-        // In case IndexedDB was used as cache storage, remove it.
-        indexedDB.deleteDatabase(STORAGE_NAME);
-        return vAPI.cacheStorage;
-    }
-
-    let db;
-    let pendingInitialization;
-    let dbBytesInUse;
-
-    const get = function get(input, callback) {
-        if ( typeof callback !== 'function' ) { return; }
-        if ( input === null ) {
-            return getAllFromDb(callback);
         }
-        var toRead, output = {};
+        catch(ex) {
-        if ( typeof input === 'string' ) {
-            toRead = [ input ];
-        } else if ( Array.isArray(input) ) {
-            toRead = input;
-        } else /* if ( typeof input === 'object' ) */ {
-            toRead = Object.keys(input);
-            output = input;
         }
-        return getFromDb(toRead, output, callback);
+        return false;
-    };
+    })();
-
-    const set = function set(input, callback) {
-        putToDb(input, callback);
-    };
-
-    const remove = function remove(key, callback) {
-        deleteFromDb(key, callback);
-    };
-
-    const clear = function clear(callback) {
-        clearDb(callback);
-    };
-
-    const getBytesInUse = function getBytesInUse(keys, callback) {
-        getDbSize(callback);
-    };
 
     const api = {
-        get,
+        name: 'browser.storage.local',
-        set,
+        get: promisified ?
-        remove,
+            browser.storage.local.get :
-        clear,
+            function(keys) {
-        getBytesInUse,
+                return new Promise(resolve => {
+                    browser.storage.local.get(keys, bin => {
+                        resolve(bin);
+                    });
+                });
+            },
+        set: promisified ?
+            browser.storage.local.set :
+            function(keys) {
+                return new Promise(resolve => {
+                    browser.storage.local.set(keys, ( ) => {
+                        resolve();
+                    });
+                });
+            },
+        remove: promisified ?
+            browser.storage.local.remove :
+            function(keys) {
+                return new Promise(resolve => {
+                    browser.storage.local.remove(keys, ( ) => {
+                        resolve();
+                    });
+                });
+            },
+        clear: promisified ?
+            browser.storage.local.clear :
+            function() {
+                return new Promise(resolve => {
+                    browser.storage.local.clear(( ) => {
+                        resolve();
+                    });
+                });
+            },
+        getBytesInUse: promisified ?
+            browser.storage.local.getBytesInUse :
+            function(keys) {
+                return new Promise(resolve => {
+                    browser.storage.local.getBytesInUse(keys, count => {
+                        resolve(count);
+                    });
+                });
+            },
+        select: function(backend) {
+            if ( backend === undefined || backend === 'unset' ) {
+                backend = vAPI.webextFlavor.soup.has('firefox')
+                    ? 'indexedDB'
+                    : 'browser.storage.local';
+            }
+            if ( backend === 'indexedDB' ) {
+                return selectIDB().then(success => {
+                    if ( success ) {
+                        clearWebext();
+                        return 'indexedDB';
+                    }
+                    clearIDB();
+                    return 'browser.storage.local';
+                });
+            }
+            if ( backend === 'browser.storage.local' ) {
+                clearIDB();
+            }
+            return Promise.resolve('browser.storage.local');
+            
+        },
         error: undefined
     };
 
-    const genericErrorHandler = function(ev) {
+    // Reassign API entries to that of indexedDB-based ones
-        let error = ev.target && ev.target.error;
+    const selectIDB = function() {
-        if ( error && error.name === 'QuotaExceededError' ) {
+        let dbPromise;
-            api.error = error.name;
+        let dbTimer;
-        }
-        console.error('[%s]', STORAGE_NAME, error && error.name);
-    };
 
-    const noopfn = function () {
+        const genericErrorHandler = function(ev) {
-    };
+            let error = ev.target && ev.target.error;
+            if ( error && error.name === 'QuotaExceededError' ) {
+                api.error = error.name;
+            }
+            console.error('[%s]', STORAGE_NAME, error && error.name);
+        };
 
-    const disconnect = function() {
+        const noopfn = function () {
-        if ( dbTimer !== undefined ) {
+        };
-            clearTimeout(dbTimer);
-            dbTimer = undefined;
-        }
-        if ( db instanceof IDBDatabase ) { 
-            db.close();
-            db = undefined;
-        }
-    };
 
-    let dbTimer;
+        const disconnect = function() {
-
+            if ( dbTimer !== undefined ) {
-    const keepAlive = function() {
+                clearTimeout(dbTimer);
-        if ( dbTimer !== undefined ) {
-            clearTimeout(dbTimer);
-        }
-        dbTimer = vAPI.setTimeout(
-            ( ) => {
                 dbTimer = undefined;
-                disconnect();
+            }
-            },
+            if ( dbPromise === undefined ) { return; }
-            Math.max(
+            dbPromise.then(db => {
-                µBlock.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000,
+                if ( db instanceof IDBDatabase ) {
-                180000
+                    db.close();
-            )
+                }
-        );
+                dbPromise = undefined;
-    };
+            });
+        };
+
+        const keepAlive = function() {
+            if ( dbTimer !== undefined ) {
+                clearTimeout(dbTimer);
+            }
+            dbTimer = vAPI.setTimeout(
+                ( ) => {
+                    dbTimer = undefined;
+                    disconnect();
+                },
+                Math.max(
+                    µBlock.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000,
+                    180000
+                )
+            );
+        };
 
-    const getDb = function getDb() {
-        if ( db === null ) {
-            return Promise.resolve(null);
-        }
-        keepAlive();
-        if ( db instanceof IDBDatabase ) {
-            return Promise.resolve(db);
-        }
-        if ( pendingInitialization !== undefined ) {
-            return pendingInitialization;
-        }
         // https://github.com/gorhill/uBlock/issues/3156
         //   I have observed that no event was fired in Tor Browser 7.0.7 +
         //   medium security level after the request to open the database was
@@ -166,327 +192,320 @@
         //   necessary when reading the `error` property because we are not
         //   allowed to read this propery outside of event handlers in newer
         //   implementation of IDBRequest (my understanding).
-        pendingInitialization = new Promise(resolve => {
+
-            let req;
+        const getDb = function() {
-            try {
+            keepAlive();
-                req = indexedDB.open(STORAGE_NAME, 1);
+            if ( dbPromise !== undefined ) {
-                if ( req.error ) {
+                return dbPromise;
-                    console.log(req.error);
+            }
+            dbPromise = new Promise(resolve => {
+                let req;
+                try {
+                    req = indexedDB.open(STORAGE_NAME, 1);
+                    if ( req.error ) {
+                        console.log(req.error);
+                        req = undefined;
+                    }
+                } catch(ex) {
+                }
+                if ( req === undefined ) {
+                    return resolve(null);
+                }
+                req.onupgradeneeded = function(ev) {
+                    req = undefined;
+                    const db = ev.target.result;
+                    db.onerror = db.onabort = genericErrorHandler;
+                    const table = db.createObjectStore(
+                        STORAGE_NAME,
+                        { keyPath: 'key' }
+                    );
+                    table.createIndex('value', 'value', { unique: false });
+                };
+                req.onsuccess = function(ev) {
+                    req = undefined;
+                    const db = ev.target.result;
+                    db.onerror = db.onabort = genericErrorHandler;
+                    resolve(db);
+                };
+                req.onerror = req.onblocked = function() {
+                    req = undefined;
+                    console.log(this.error);
+                    resolve(null);
+                };
+            });
+            return dbPromise;
+        };
+
+        const getFromDb = function(keys, keyvalStore, callback) {
+            if ( typeof callback !== 'function' ) { return; }
+            if ( keys.length === 0 ) { return callback(keyvalStore); }
+            let promises = [];
+            let gotOne = function() {
+                if ( typeof this.result !== 'object' ) { return; }
+                keyvalStore[this.result.key] = this.result.value;
+                if ( this.result.value instanceof Blob === false ) { return; }
+                promises.push(
+                    µBlock.lz4Codec.decode(
+                        this.result.key,
+                        this.result.value
+                    ).then(result => {
+                        keyvalStore[result.key] = result.data;
+                    })
+                );
+            };
+            getDb().then(db => {
+                if ( !db ) { return callback(); }
+                const transaction = db.transaction(STORAGE_NAME);
+                transaction.oncomplete =
+                transaction.onerror =
+                transaction.onabort = ( ) => {
+                    Promise.all(promises).then(( ) => {
+                        callback(keyvalStore);
+                    });
+                };
+                const table = transaction.objectStore(STORAGE_NAME);
+                for ( const key of keys ) {
+                    let req = table.get(key);
+                    req.onsuccess = gotOne;
+                    req.onerror = noopfn;
                     req = undefined;
                 }
-            } catch(ex) {
-            }
-            if ( req === undefined ) {
-                pendingInitialization = undefined;
-                db = null;
-                resolve(null);
-                return;
-            }
-            req.onupgradeneeded = function(ev) {
-                req = undefined;
-                let db = ev.target.result;
-                db.onerror = db.onabort = genericErrorHandler;
-                let table = db.createObjectStore(STORAGE_NAME, { keyPath: 'key' });
-                table.createIndex('value', 'value', { unique: false });
-            };
-            req.onsuccess = function(ev) {
-                pendingInitialization = undefined;
-                req = undefined;
-                db = ev.target.result;
-                db.onerror = db.onabort = genericErrorHandler;
-                resolve(db);
-            };
-            req.onerror = req.onblocked = function() {
-                pendingInitialization = undefined;
-                req = undefined;
-                db = null;
-                console.log(this.error);
-                resolve(null);
-            };
-        });
-        return pendingInitialization;
-    };
-
-    const getFromDb = function(keys, keyvalStore, callback) {
-        if ( typeof callback !== 'function' ) { return; }
-        if ( keys.length === 0 ) { return callback(keyvalStore); }
-        let promises = [];
-        let gotOne = function() {
-            if ( typeof this.result !== 'object' ) { return; }
-            keyvalStore[this.result.key] = this.result.value;
-            if ( this.result.value instanceof Blob === false ) { return; }
-            promises.push(
-                µBlock.lz4Codec.decode(
-                    this.result.key,
-                    this.result.value
-                ).then(result => {
-                    keyvalStore[result.key] = result.data;
-                })
-            );
-        };
-        getDb().then(( ) => {
-            if ( !db ) { return callback(); }
-            let transaction = db.transaction(STORAGE_NAME);
-            transaction.oncomplete =
-            transaction.onerror =
-            transaction.onabort = ( ) => {
-                Promise.all(promises).then(( ) => {
-                    callback(keyvalStore);
-                });
-            };
-            let table = transaction.objectStore(STORAGE_NAME);
-            for ( let key of keys ) {
-                let req = table.get(key);
-                req.onsuccess = gotOne;
-                req.onerror = noopfn;
-                req = undefined;
-            }
-        });
-    };
-
-    const visitAllFromDb = function(visitFn) {
-        getDb().then(( ) => {
-            if ( !db ) { return visitFn(); }
-            let transaction = db.transaction(STORAGE_NAME);
-            transaction.oncomplete =
-            transaction.onerror =
-            transaction.onabort = ( ) => visitFn();
-            let table = transaction.objectStore(STORAGE_NAME);
-            let req = table.openCursor();
-            req.onsuccess = function(ev) {
-                let cursor = ev.target && ev.target.result;
-                if ( !cursor ) { return; }
-                let entry = cursor.value;
-                visitFn(entry);
-                cursor.continue();
-            };
-        });
-    };
-
-    const getAllFromDb = function(callback) {
-        if ( typeof callback !== 'function' ) { return; }
-        let promises = [];
-        let keyvalStore = {};
-        visitAllFromDb(entry => {
-            if ( entry === undefined ) {
-                Promise.all(promises).then(( ) => {
-                    callback(keyvalStore);
-                });
-                return;
-            }
-            keyvalStore[entry.key] = entry.value;
-            if ( entry.value instanceof Blob === false ) { return; }
-            promises.push(
-                µBlock.lz4Codec.decode(
-                    entry.key,
-                    entry.value
-                ).then(result => {
-                    keyvalStore[result.key] = result.value;
-                })
-            );
-        });
-    };
-
-    const getDbSize = function(callback) {
-        if ( typeof callback !== 'function' ) { return; }
-        if ( typeof dbBytesInUse === 'number' ) {
-            return Promise.resolve().then(( ) => {
-                callback(dbBytesInUse);
             });
-        }
-        const textEncoder = new TextEncoder();
-        let totalByteLength = 0;
-        visitAllFromDb(entry => {
-            if ( entry === undefined ) {
-                dbBytesInUse = totalByteLength;
-                return callback(totalByteLength);
-            }
-            let value = entry.value;
-            if ( typeof value === 'string' ) {
-                totalByteLength += textEncoder.encode(value).byteLength;
-            } else if ( value instanceof Blob ) {
-                totalByteLength += value.size;
-            } else {
-                totalByteLength += textEncoder.encode(JSON.stringify(value)).byteLength;
-            }
-            if ( typeof entry.key === 'string' ) {
-                totalByteLength += textEncoder.encode(entry.key).byteLength;
-            }
-        });
-    };
-
-
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/141
-    //   Mind that IDBDatabase.transaction() and IDBObjectStore.put()
-    //   can throw:
-    //   https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction
-    //   https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put
-
-    const putToDb = function(keyvalStore, callback) {
-        if ( typeof callback !== 'function' ) {
-            callback = noopfn;
-        }
-        const keys = Object.keys(keyvalStore);
-        if ( keys.length === 0 ) { return callback(); }
-        const promises = [ getDb() ];
-        const entries = [];
-        const dontCompress = µBlock.hiddenSettings.cacheStorageCompression !== true;
-        let bytesInUse = 0;
-        const handleEncodingResult = result => {
-            if ( typeof result.data === 'string' ) {
-                bytesInUse += result.data.length;
-            } else if ( result.data instanceof Blob ) {
-                bytesInUse += result.data.size;
-            }
-            entries.push({ key: result.key, value: result.data });
         };
-        for ( const key of keys ) {
+
-            const data = keyvalStore[key];
+        const visitAllFromDb = function(visitFn) {
-            const isString = typeof data === 'string';
+            getDb().then(db => {
-            if ( isString === false || dontCompress ) {
+                if ( !db ) { return visitFn(); }
-                if ( isString ) {
+                const transaction = db.transaction(STORAGE_NAME);
-                    bytesInUse += data.length;
-                }
-                entries.push({ key, value: data });
-                continue;
-            }
-            promises.push(
-                µBlock.lz4Codec.encode(key, data).then(handleEncodingResult)
-            );
-        }
-        Promise.all(promises).then(( ) => {
-            if ( !db ) { return callback(); }
-            const finish = ( ) => {
-                dbBytesInUse = undefined;
-                if ( callback === undefined ) { return; }
-                let cb = callback;
-                callback = undefined;
-                cb({ bytesInUse });
-            };
-            try {
-                const transaction = db.transaction(STORAGE_NAME, 'readwrite');
                 transaction.oncomplete =
                 transaction.onerror =
-                transaction.onabort = finish;
+                transaction.onabort = ( ) => visitFn();
                 const table = transaction.objectStore(STORAGE_NAME);
-                for ( const entry of entries ) {
+                const req = table.openCursor();
-                    table.put(entry);
+                req.onsuccess = function(ev) {
-                }
+                    let cursor = ev.target && ev.target.result;
-            } catch (ex) {
+                    if ( !cursor ) { return; }
-                finish();
+                    let entry = cursor.value;
-            }
+                    visitFn(entry);
-        });
+                    cursor.continue();
-    };
+                };
+            });
+        };
 
-    const deleteFromDb = function(input, callback) {
+        const getAllFromDb = function(callback) {
-        if ( typeof callback !== 'function' ) {
+            if ( typeof callback !== 'function' ) { return; }
-            callback = noopfn;
+            const promises = [];
-        }
+            const keyvalStore = {};
-        let keys = Array.isArray(input) ? input.slice() : [ input ];
+            visitAllFromDb(entry => {
-        if ( keys.length === 0 ) { return callback(); }
+                if ( entry === undefined ) {
-        getDb().then(db => {
+                    Promise.all(promises).then(( ) => {
-            if ( !db ) { return callback(); }
+                        callback(keyvalStore);
-            let finish = ( ) => {
+                    });
-                dbBytesInUse = undefined;
-                if ( callback === undefined ) { return; }
-                let cb = callback;
-                callback = undefined;
-                cb();
-            };
-            try {
-                let transaction = db.transaction(STORAGE_NAME, 'readwrite');
-                transaction.oncomplete =
-                transaction.onerror =
-                transaction.onabort = finish;
-                let table = transaction.objectStore(STORAGE_NAME);
-                for ( let key of keys ) {
-                    table.delete(key);
-                }
-            } catch (ex) {
-                finish();
-            }
-        });
-    };
-
-    const clearDb = function(callback) {
-        if ( typeof callback !== 'function' ) {
-            callback = noopfn;
-        }
-        getDb().then(db => {
-            if ( !db ) { return callback(); }
-            let finish = ( ) => {
-                disconnect();
-                indexedDB.deleteDatabase(STORAGE_NAME);
-                dbBytesInUse = 0;
-                if ( callback === undefined ) { return; }
-                let cb = callback;
-                callback = undefined;
-                cb();
-            };
-            try {
-                let req = db.transaction(STORAGE_NAME, 'readwrite')
-                            .objectStore(STORAGE_NAME)
-                            .clear();
-                req.onsuccess = req.onerror = finish;
-            } catch (ex) {
-                finish();
-            }
-        });
-    };
-
-    // prime the db so that it's ready asap for next access.
-    getDb(noopfn);
-
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/328
-    //   Detect whether browser.storage.local was used as cache storage,
-    //   and if so, move cache-related entries to the new storage.
-    {
-        const srcStorage = vAPI.cacheStorage;
-        const desStorage = api;
-        srcStorage.get(
-            [ 'assetCacheRegistry', 'assetSourceRegistry' ],
-            bin => {
-                if (
-                    bin instanceof Object === false ||
-                    bin.assetSourceRegistry instanceof Object === false
-                ) {
                     return;
                 }
-                desStorage.set(bin);
+                keyvalStore[entry.key] = entry.value;
-                const toRemove = [
+                if ( entry.value instanceof Blob === false ) { return; }
-                    'assetCacheRegistry',
+                promises.push(
-                    'assetSourceRegistry',
+                    µBlock.lz4Codec.decode(
-                    'resourcesSelfie',
+                        entry.key,
-                    'selfie'
+                        entry.value
-                ];
+                    ).then(result => {
-                let toMigrate = 0;
+                        keyvalStore[result.key] = result.value;
-                const setEntry = function(assetKey, bin) {
+                    })
-                    if (
+                );
-                        bin instanceof Object &&
+            });
-                        bin[assetKey] !== undefined
+        };
-                    ) {
+
-                        desStorage.set(bin);
+        // https://github.com/uBlockOrigin/uBlock-issues/issues/141
-                    }
+        //   Mind that IDBDatabase.transaction() and IDBObjectStore.put()
-                    toMigrate -= 1;
+        //   can throw:
-                    if ( toMigrate === 0 ) {
+        //   https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase/transaction
-                        srcStorage.remove(toRemove);
+        //   https://developer.mozilla.org/en-US/docs/Web/API/IDBObjectStore/put
-                    }
+
-                };
+        const putToDb = function(keyvalStore, callback) {
-                for ( const key in bin.assetCacheRegistry ) {
+            if ( typeof callback !== 'function' ) {
-                    if ( bin.assetCacheRegistry.hasOwnProperty(key) === false ) {
+                callback = noopfn;
-                        continue;
+            }
-                    }
+            const keys = Object.keys(keyvalStore);
-                    const assetKey = 'cache/' + key;
+            if ( keys.length === 0 ) { return callback(); }
-                    srcStorage.get(assetKey, setEntry.bind(null, assetKey));
+            const promises = [ getDb() ];
-                    toMigrate += 1;
+            const entries = [];
-                    toRemove.push(assetKey);
+            const dontCompress =
+                µBlock.hiddenSettings.cacheStorageCompression !== true;
+            let bytesInUse = 0;
+            const handleEncodingResult = result => {
+                if ( typeof result.data === 'string' ) {
+                    bytesInUse += result.data.length;
+                } else if ( result.data instanceof Blob ) {
+                    bytesInUse += result.data.size;
                 }
-                if ( toMigrate === 0 ) {
+                entries.push({ key: result.key, value: result.data });
-                    srcStorage.remove(toRemove);
+            };
+            for ( const key of keys ) {
+                const data = keyvalStore[key];
+                const isString = typeof data === 'string';
+                if ( isString === false || dontCompress ) {
+                    if ( isString ) {
+                        bytesInUse += data.length;
+                    }
+                    entries.push({ key, value: data });
+                    continue;
+                }
+                promises.push(
+                    µBlock.lz4Codec.encode(key, data).then(handleEncodingResult)
+                );
+            }
+            Promise.all(promises).then(results => {
+                const db = results[0];
+                if ( !db ) { return callback(); }
+                const finish = ( ) => {
+                    if ( callback === undefined ) { return; }
+                    let cb = callback;
+                    callback = undefined;
+                    cb({ bytesInUse });
+                };
+                try {
+                    const transaction = db.transaction(
+                        STORAGE_NAME,
+                        'readwrite'
+                    );
+                    transaction.oncomplete =
+                    transaction.onerror =
+                    transaction.onabort = finish;
+                    const table = transaction.objectStore(STORAGE_NAME);
+                    for ( const entry of entries ) {
+                        table.put(entry);
+                    }
+                } catch (ex) {
+                    finish();
+                }
+            });
+        };
+
+        const deleteFromDb = function(input, callback) {
+            if ( typeof callback !== 'function' ) {
+                callback = noopfn;
+            }
+            const keys = Array.isArray(input) ? input.slice() : [ input ];
+            if ( keys.length === 0 ) { return callback(); }
+            getDb().then(db => {
+                if ( !db ) { return callback(); }
+                let finish = ( ) => {
+                    if ( callback === undefined ) { return; }
+                    let cb = callback;
+                    callback = undefined;
+                    cb();
+                };
+                try {
+                    let transaction = db.transaction(STORAGE_NAME, 'readwrite');
+                    transaction.oncomplete =
+                    transaction.onerror =
+                    transaction.onabort = finish;
+                    let table = transaction.objectStore(STORAGE_NAME);
+                    for ( let key of keys ) {
+                        table.delete(key);
+                    }
+                } catch (ex) {
+                    finish();
+                }
+            });
+        };
+
+        const clearDb = function(callback) {
+            if ( typeof callback !== 'function' ) {
+                callback = noopfn;
+            }
+            getDb().then(db => {
+                if ( !db ) { return callback(); }
+                const finish = ( ) => {
+                    disconnect();
+                    indexedDB.deleteDatabase(STORAGE_NAME);
+                    if ( callback === undefined ) { return; }
+                    let cb = callback;
+                    callback = undefined;
+                    cb();
+                };
+                try {
+                    const req = db.transaction(STORAGE_NAME, 'readwrite')
+                                  .objectStore(STORAGE_NAME)
+                                  .clear();
+                    req.onsuccess = req.onerror = finish;
+                } catch (ex) {
+                    finish();
+                }
+            });
+        };
+
+        return getDb().then(db => {
+            if ( !db ) { return false; }
+            api.name = 'indexedDB';
+            api.get = function get(keys) {
+                return new Promise(resolve => {
+                    if ( keys === null ) {
+                        return getAllFromDb(bin => resolve(bin));
+                    }
+                    let toRead, output = {};
+                    if ( typeof keys === 'string' ) {
+                        toRead = [ keys ];
+                    } else if ( Array.isArray(keys) ) {
+                        toRead = keys;
+                    } else /* if ( typeof keys === 'object' ) */ {
+                        toRead = Object.keys(keys);
+                        output = keys;
+                    }
+                    getFromDb(toRead, output, bin => resolve(bin));
+                });
+            };
+            api.set = function set(keys) {
+                return new Promise(resolve => {
+                    putToDb(keys, details => resolve(details));
+                });
+            };
+            api.remove = function remove(keys) {
+                return new Promise(resolve => {
+                    deleteFromDb(keys, ( ) => resolve());
+                });
+            };
+            api.clear = function clear() {
+                return new Promise(resolve => {
+                    clearDb(( ) => resolve());
+                });
+            };
+            api.getBytesInUse = function getBytesInUse() {
+                return Promise.resolve(0);
+            };
+            return true;
+        });
+    };
+
+    // https://github.com/uBlockOrigin/uBlock-issues/issues/328
+    //   Delete cache-related entries from webext storage.
+    const clearWebext = function() {
+        browser.storage.local.get('assetCacheRegistry', bin => {
+            if (
+                bin instanceof Object === false ||
+                bin.assetCacheRegistry instanceof Object === false
+            ) {
+                return;
+            }
+            const toRemove = [
+                'assetCacheRegistry',
+                'assetSourceRegistry',
+                'resourcesSelfie',
+                'selfie'
+            ];
+            for ( const key in bin.assetCacheRegistry ) {
+                if ( bin.assetCacheRegistry.hasOwnProperty(key) ) {
+                    toRemove.push('cache/' + key);
                 }
             }
-        );
+            browser.storage.local.remove(toRemove);
-    }
+        });
+    };
+
+    const clearIDB = function() {
+        indexedDB.deleteDatabase(STORAGE_NAME);
+    };
 
     return api;
 }());

src/js/messaging.js

@@ -848,17 +848,17 @@ var restoreUserData = function(request) {
 // Remove all stored data but keep global counts, people can become
 // quite attached to numbers
 
-var resetUserData = function() {
+const resetUserData = function() {
     let count = 3;
-    let countdown = ( ) => {
+    const countdown = ( ) => {
         count -= 1;
         if ( count === 0 ) {
             vAPI.app.restart();
         }
     };
-    µb.cacheStorage.clear(countdown); // 1
+    µb.cacheStorage.clear().then(( ) => countdown()); // 1
-    vAPI.storage.clear(countdown);    // 2
+    vAPI.storage.clear(countdown);             // 2
-    µb.saveLocalSettings(countdown);  // 3
+    µb.saveLocalSettings(countdown);           // 3
     vAPI.localStorage.removeItem('immediateHiddenSettings');
 };
 

src/js/start.js

@@ -333,7 +333,7 @@ const fromFetch = function(to, fetched) {
 
 /******************************************************************************/
 
-const onSelectedFilterListsLoaded = function() {
+const onSelectedFilterListsReady = function() {
     log.info(`List selection ready ${Date.now()-vAPI.T0} ms after launch`);
 
     const fetchableProps = {
@@ -371,6 +371,16 @@ const onSelectedFilterListsLoaded = function() {
 
 /******************************************************************************/
 
+const onHiddenSettingsReady = function() {
+    return µb.cacheStorage.select(
+        µb.hiddenSettings.cacheStorageAPI
+    ).then(backend => {
+        log.info(`Backend storage for cache will be ${backend}`);
+    });
+};
+
+/******************************************************************************/
+
 // TODO(seamless migration):
 // Eventually selected filter list keys will be loaded as a fetchable
 // property. Until then we need to handle backward and forward
@@ -379,14 +389,24 @@ const onSelectedFilterListsLoaded = function() {
 
 const onAdminSettingsRestored = function() {
     log.info(`Admin settings ready ${Date.now()-vAPI.T0} ms after launch`);
-    µb.loadSelectedFilterLists(onSelectedFilterListsLoaded);
+
+    Promise.all([
+        µb.loadHiddenSettings().then(( ) =>
+            onHiddenSettingsReady()
+        ),
+        µb.loadSelectedFilterLists(),
+    ]).then(( ) =>
+        onSelectedFilterListsReady()
+    );
 };
 
 /******************************************************************************/
 
 return function() {
     // https://github.com/gorhill/uBlock/issues/531
-    µb.restoreAdminSettings(onAdminSettingsRestored);
+    µb.restoreAdminSettings().then(( ) => {
+        onAdminSettingsRestored();
+    });
 };
 
 /******************************************************************************/

src/js/storage.js

@@ -50,7 +50,7 @@
         countdown += 1;
         vAPI.storage.getBytesInUse(null, process);
     }
-    if ( this.cacheStorage !== vAPI.storage ) {
+    if ( this.cacheStorage.name !== 'browser.storage.local' ) {
         countdown += 1;
         this.assets.getBytesInUse().then(count => {
             process(count);
@@ -91,8 +91,13 @@
 /******************************************************************************/
 
 µBlock.loadHiddenSettings = function() {
+    return new Promise(resolve => {
+    // >>>> start of executor
+
     vAPI.storage.get('hiddenSettings', bin => {
-        if ( bin instanceof Object === false ) { return; }
+        if ( bin instanceof Object === false ) {
+            return resolve();
+        }
         const hs = bin.hiddenSettings;
         if ( hs instanceof Object ) {
             const hsDefault = this.hiddenSettingsDefault;
@@ -110,6 +115,10 @@
             this.saveImmediateHiddenSettings();
         }
         self.log.verbosity = this.hiddenSettings.consoleLogLevel;
+        resolve();
+    });
+
+    // <<<< end of executor
     });
 };
 
@@ -195,9 +204,6 @@
     );
 };
 
-// Do this here to have these hidden settings loaded ASAP.
-µBlock.loadHiddenSettings();
-
 /******************************************************************************/
 
 µBlock.savePermanentFirewallRules = function() {
@@ -240,24 +246,29 @@
 
 **/
 
-µBlock.loadSelectedFilterLists = function(callback) {
+µBlock.loadSelectedFilterLists = function() {
-    var µb = this;
+    return new Promise(resolve => {
-    vAPI.storage.get('selectedFilterLists', function(bin) {
+    // >>>> start of executor
+
+    vAPI.storage.get('selectedFilterLists', bin => {
         // Select default filter lists if first-time launch.
-        if ( !bin || Array.isArray(bin.selectedFilterLists) === false ) {
+        if (
-            µb.assets.metadata(function(availableLists) {
+            bin instanceof Object === false ||
-                µb.saveSelectedFilterLists(
+            Array.isArray(bin.selectedFilterLists) === false
-                    µb.autoSelectRegionalFilterLists(availableLists)
+        ) {
+            this.assets.metadata(function(availableLists) {
+                this.saveSelectedFilterLists(
+                    this.autoSelectRegionalFilterLists(availableLists)
                 );
-                callback();
+                resolve();
             });
             return;
         }
-        // TODO: Removes once 1.1.15 is in widespread use.
+        this.selectedFilterLists = bin.selectedFilterLists;
-        // https://github.com/gorhill/uBlock/issues/3383
+        resolve();
-        vAPI.storage.remove('remoteBlacklists');
+    });
-        µb.selectedFilterLists = bin.selectedFilterLists;
+
-        callback();
+    // <<<< end of executor
     });
 };
 
@@ -1130,16 +1141,16 @@
 // necessarily present, i.e. administrators may removed entries which
 // values are left to the user's choice.
 
-µBlock.restoreAdminSettings = function(callback) {
+µBlock.restoreAdminSettings = function() {
-    // Support for vAPI.adminStorage is optional (webext).
+    return new Promise(resolve => {
+    // >>>> start of executor
+
     if ( vAPI.adminStorage instanceof Object === false ) {
-        callback();
+        return resolve();
-        return;
     }
 
-    var onRead = function(json) {
+    vAPI.adminStorage.getItem('adminSettings', json => {
-        var µb = µBlock;
+        let data;
-        var data;
         if ( typeof json === 'string' && json !== '' ) {
             try {
                 data = JSON.parse(json);
@@ -1148,13 +1159,12 @@
             }
         }
 
-        if ( typeof data !== 'object' || data === null ) {
+        if ( data instanceof Object === false ) {
-            callback();
+            return resolve();
-            return;
         }
 
-        var bin = {};
+        const bin = {};
-        var binNotEmpty = false;
+        let binNotEmpty = false;
 
         // Allows an admin to set their own 'assets.json' file, with their own
         // set of stock assets.
@@ -1164,8 +1174,8 @@
         }
 
         if ( typeof data.userSettings === 'object' ) {
-            for ( var name in µb.userSettings ) {
+            for ( const name in this.userSettings ) {
-                if ( µb.userSettings.hasOwnProperty(name) === false ) {
+                if ( this.userSettings.hasOwnProperty(name) === false ) {
                     continue;
                 }
                 if ( data.userSettings.hasOwnProperty(name) === false ) {
@@ -1208,13 +1218,14 @@
         }
 
         if ( typeof data.userFilters === 'string' ) {
-            µb.assets.put(µb.userFiltersPath, data.userFilters);
+            this.assets.put(this.userFiltersPath, data.userFilters);
         }
 
-        callback();
+        resolve();
-    };
+    });
 
-    vAPI.adminStorage.getItem('adminSettings', onRead);
+    // <<<< end of executor
+    });
 };
 
 /******************************************************************************/