Storage: distinguish preferences from caches (addresses #1366)

platform/chromium/vapi-background.js

@@ -71,6 +71,7 @@ vAPI.app.restart = function() {
 // chrome.storage.local.get(null, function(bin){ console.debug('%o', bin); });
 
 vAPI.storage = chrome.storage.local;
+vAPI.storage.preferences = vAPI.storage;
 
 /******************************************************************************/
 

platform/firefox/vapi-background.js

@@ -276,6 +276,8 @@ vAPI.storage = {
     }
 };
 
+vAPI.storage.preferences = vAPI.storage;
+
 /******************************************************************************/
 
 var windowWatcher = {

platform/safari/vapi-background.js

@@ -105,16 +105,7 @@
         size: storageQuota,
         storeName: "keyvaluepairs"
     });
-    var oldSettings = safari.extension.settings; // To smoothly transition users
+    
-    if(oldSettings.hasOwnProperty("version")) { // Old 'storage'!
-        for(var key in oldSettings) {
-            if(!oldSettings.hasOwnProperty(key) || key === "open_prefs") {
-                continue;
-            }
-            localforage.setItem(key, oldSettings[key]);
-        }
-        oldSettings.clear();
-    }
     vAPI.storage = {
         QUOTA_BYTES: storageQuota, // copied from Info.plist
 
@@ -212,6 +203,7 @@
         },
 
         clear: function(callback) {
+            this.preferences.clear();
             localforage.clear(function() {
                 callback();
             });
@@ -229,6 +221,124 @@
             });
         }
     };
+    /******************************************************************************/
+
+    if(!safari.extension.settings.migratedStorage) {
+        var migrationMap = {
+            "cached_asset_content://assets/user/filters.txt": "userFilters"
+        };
+
+        var delayed = [];
+        
+        vAPI.storage.preferences = {
+            get: function(a, b) {
+                delayed.push(settingsStorage.get.bind(settingsStorage, a, b));
+            },
+            set: function(a, b) {
+                delayed.push(settingsStorage.set.bind(settingsStorage, a, b));
+            },
+            remove: function(a, b) {
+                delayed.push(settingsStorage.remove.bind(settingsStorage, a, b));
+            },
+            clear: function() {
+                delayed.push(settingsStorage.clear.bind(settingsStorage));
+            },
+        };
+
+        localforage.iterate(function(value, key) {
+            if(migrationMap[key]) {
+                safari.extension.settings[migrationMap[key]] = value;
+                return;
+            }
+            if(key.lastIndexOf("cached_asset", 0) === 0) {
+                return;
+            }
+            safari.extension.settings[key] = value;
+            localforage.removeItem(key);
+        }, function() {
+            var func;
+            while(func = delayed.pop()) {
+                func();
+            }
+            delayed = null;
+            vAPI.storage.preferences = settingsStorage;
+        });
+        safari.extension.settings.migratedStorage = true;
+    }
+
+    var settingsStorage = {
+        _storage: safari.extension.settings,
+        get: function(keys, callback) {
+            if(typeof callback !== "function") {
+                return;
+            }
+
+            var i, value, result = {};
+
+            if(keys === null) {
+                for(i in this._storage) {
+                    if(!this._storage.hasOwnProperty(i)) continue;
+                    value = this._storage[i];
+                    if(typeof value === "string") {
+                        result[i] = JSON.parse(value);
+                    }
+                }
+            }
+            else if(typeof keys === "string") {
+                value = this._storage[keys];
+                if(typeof value === "string") {
+                    result[keys] = JSON.parse(value);
+                }
+            }
+            else if(Array.isArray(keys)) {
+                for(i = 0; i < keys.length; i++) {
+                    value = this._storage[i];
+
+                    if(typeof value === "string") {
+                        result[keys[i]] = JSON.parse(value);
+                    }
+                }
+            }
+            else if(typeof keys === "object") {
+                for(i in keys) {
+                    if(!keys.hasOwnProperty(i)) {
+                        continue;
+                    }
+                    value = this._storage[i];
+
+                    if(typeof value === "string") {
+                        result[i] = JSON.parse(value);
+                    }
+                    else {
+                        result[i] = keys[i];
+                    }
+                }
+            }
+            callback(result);
+        },
+        set: function(details, callback) {
+            for(var key in details) {
+                if(!details.hasOwnProperty(key)) {
+                    continue;
+                }
+                this._storage.setItem(key, JSON.stringify(details[key]));
+            }
+            if(typeof callback === "function") {
+                callback();
+            }
+        },
+        remove: function(keys) {
+            if(typeof keys === "string") {
+                keys = [keys];
+            }
+            for(var i = 0; i < keys.length; i++) {
+                this._storage.removeItem(keys[i]);
+            }
+        },
+        clear: function() {
+            this._storage.clear();
+        } 
+    };
 
     /******************************************************************************/
 
@@ -567,7 +677,7 @@
         if(iconState.dirty & 2) {
             icon.badge = iconState.badge;
         }
-        if(iconState.dirty & 1 && icon.image !== ICON_URLS[iconState.img]) {
+        if((iconState.dirty & 1) && icon.image !== ICON_URLS[iconState.img]) {
             icon.image = ICON_URLS[iconState.img];
         }
         iconState.dirty = 0;

src/js/background.js

@@ -63,7 +63,7 @@ return {
         firewallPaneMinimized: true,
         parseAllABPHideFilters: true,
         requestLogMaxEntries: 1000,
-        showIconBadge: true
+        showIconBadge: true,
     },
 
     // https://github.com/chrisaljoudi/uBlock/issues/180
@@ -79,6 +79,8 @@ return {
         'opera-scheme'
     ].join('\n').trim(),
     
+    userFiltersPath: "assets/user/filters.txt",
+
     localSettings: {
         blockedRequestCount: 0,
         allowedRequestCount: 0,
@@ -103,7 +105,6 @@ return {
     // as per list headers.
     updateAssetsEvery: 97 * oneHour,
     projectServerRoot: 'https://raw.githubusercontent.com/chrisaljoudi/uBlock/master/',
-    userFiltersPath: 'assets/user/filters.txt',
     pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat',
 
     // permanent lists

src/js/messaging.js

@@ -797,10 +797,10 @@ var onMessage = function(request, sender, callback) {
     // Async
     switch ( request.what ) {
         case 'readUserFilters':
-            return µb.assets.get(µb.userFiltersPath, callback);
+            return µb.loadUserFilters(callback);
 
         case 'writeUserFilters':
-            return µb.assets.put(µb.userFiltersPath, request.content, callback);
+            return µb.saveUserFilters(request.content, callback);
 
         default:
             break;
@@ -1083,7 +1083,7 @@ var backupUserData = function(callback) {
 
         µb.restoreBackupSettings.lastBackupFile = filename;
         µb.restoreBackupSettings.lastBackupTime = Date.now();
-        vAPI.storage.set(µb.restoreBackupSettings);
+        vAPI.storage.preferences.set(µb.restoreBackupSettings);
 
         getLocalData(callback);
     };
@@ -1093,8 +1093,7 @@ var backupUserData = function(callback) {
         µb.extractSelectedFilterLists(onSelectedListsReady);
     };
 
-
+    µb.loadUserFilters(onUserFiltersReady);
-    µb.assets.get('assets/user/filters.txt', onUserFiltersReady);
 };
 
 /******************************************************************************/
@@ -1111,19 +1110,19 @@ var restoreUserData = function(request) {
 
     var onAllRemoved = function() {
         // Be sure to adjust `countdown` if adding/removing anything below
-        µb.keyvalSetOne('version', userData.version);
+        µb.keyvalSetOnePref('version', userData.version);
         µBlock.saveLocalSettings(true);
-        vAPI.storage.set(userData.userSettings, onCountdown);
+        vAPI.storage.preferences.set(userData.userSettings, onCountdown);
-        µb.keyvalSetOne('remoteBlacklists', userData.filterLists, onCountdown);
+        µb.keyvalSetOnePref('remoteBlacklists', userData.filterLists, onCountdown);
-        µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown);
+        µb.keyvalSetOnePref('netWhitelist', userData.netWhitelist || '', onCountdown);
 
         // With versions 0.9.2.4-, dynamic rules were saved within the
         // `userSettings` object. No longer the case.
         var s = userData.dynamicFilteringString || userData.userSettings.dynamicFilteringString || '';
-        µb.keyvalSetOne('dynamicFilteringString', s, onCountdown);
+        µb.keyvalSetOnePref('dynamicFilteringString', s, onCountdown);
 
-        µb.assets.put('assets/user/filters.txt', userData.userFilters, onCountdown);
+        µb.keyvalSetOnePref('userFilters', userData.userFilters, onCountdown);
-        vAPI.storage.set({
+        vAPI.storage.preferences.set({
             lastRestoreFile: request.file || '',
             lastRestoreTime: Date.now(),
             lastBackupFile: '',

src/js/start.js

@@ -169,12 +169,16 @@ var onSystemSettingsReady = function(fetched) {
     if ( mustSaveSystemSettings ) {
         fetched.selfie = null;
         µb.destroySelfie();
-        vAPI.storage.set(µb.systemSettings, µb.noopFunc);
+        vAPI.storage.preferences.set(µb.systemSettings, µb.noopFunc);
     }
 };
 
 /******************************************************************************/
 
+var onUserFiltersReady = function(userFilters) {
+    µb.saveUserFilters(userFilters); // we need this because of migration
+};
+
 var onFirstFetchReady = function(fetched) {
     // Order is important -- do not change:
     onSystemSettingsReady(fetched);
@@ -182,6 +186,7 @@ var onFirstFetchReady = function(fetched) {
     onUserSettingsReady(fetched);
     fromFetch(µb.restoreBackupSettings, fetched);
     onNetWhitelistReady(fetched.netWhitelist);
+    onUserFiltersReady(fetched.userFilters);
     onVersionReady(fetched.version);
 
     // If we have a selfie, skip loading PSL, filters
@@ -195,6 +200,16 @@ var onFirstFetchReady = function(fetched) {
 
 /******************************************************************************/
 
+var onPrefFetchReady = function(fetched) {
+    fetched.userFilters = fetched.userFilters || fetched["cached_asset_content://assets/user/filters.txt"];
+    vAPI.storage.get({"selfie": null}, function(res) {
+        fetched["selfie"] = res["selfie"];
+        onFirstFetchReady(fetched);
+    });
+};
+
+/******************************************************************************/
+
 var toFetch = function(from, fetched) {
     for ( var k in from ) {
         if ( from.hasOwnProperty(k) === false ) {
@@ -230,7 +245,8 @@ return function() {
         'lastBackupFile': '',
         'lastBackupTime': 0,
         'netWhitelist': '',
-        'selfie': null,
+        'userFilters': '',
+        'cached_asset_content://assets/user/filters.txt': '',
         'selfieMagic': '',
         'version': '0.0.0.0'
     };
@@ -239,7 +255,7 @@ return function() {
     toFetch(µb.userSettings, fetchableProps);
     toFetch(µb.restoreBackupSettings, fetchableProps);
 
-    vAPI.storage.get(fetchableProps, onFirstFetchReady);
+    vAPI.storage.preferences.get(fetchableProps, onPrefFetchReady);
 };
 
 /******************************************************************************/

src/js/storage.js

@@ -38,10 +38,10 @@
 
 /******************************************************************************/
 
-µBlock.keyvalSetOne = function(key, val, callback) {
+µBlock.keyvalSetOnePref = function(key, val, callback) {
     var bin = {};
     bin[key] = val;
-    vAPI.storage.set(bin, callback || this.noopFunc);
+    vAPI.storage.preferences.set(bin, callback || this.noopFunc);
 };
 
 /******************************************************************************/
@@ -54,7 +54,7 @@
         return;
     }
     this.localSettingsSaveTime = Date.now();
-    vAPI.storage.set(this.localSettings);
+    vAPI.storage.preferences.set(this.localSettings);
 };
 
 /******************************************************************************/
@@ -72,13 +72,13 @@
 /******************************************************************************/
 
 µBlock.saveUserSettings = function() {
-    vAPI.storage.set(this.userSettings);
+    vAPI.storage.preferences.set(this.userSettings);
 };
 
 /******************************************************************************/
 
 µBlock.savePermanentFirewallRules = function() {
-    this.keyvalSetOne('dynamicFilteringString', this.permanentFirewall.toString());
+    this.keyvalSetOnePref('dynamicFilteringString', this.permanentFirewall.toString());
 };
 
 /******************************************************************************/
@@ -87,7 +87,7 @@
     var bin = {
         'netWhitelist': this.stringFromWhitelist(this.netWhitelist)
     };
-    vAPI.storage.set(bin);
+    vAPI.storage.preferences.set(bin);
     this.netWhitelistModifyTime = Date.now();
 };
 
@@ -135,7 +135,9 @@
 /******************************************************************************/
 
 µBlock.saveUserFilters = function(content, callback) {
-    return this.assets.put(this.userFiltersPath, content, callback);
+    vAPI.storage.preferences.set({"userFilters": content});
+    this.assets.put(this.userFiltersPath, content, callback);
+    return;
 };
 
 /******************************************************************************/
@@ -165,15 +167,12 @@
         var deltaEntryUsedCount = deltaEntryCount - (snfe.duplicateCount + cfe.duplicateCount - duplicateCount);
         entry.entryCount += deltaEntryCount;
         entry.entryUsedCount += deltaEntryUsedCount;
-        vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists });
+        vAPI.storage.preferences.set({ 'remoteBlacklists': µb.remoteBlacklists });
         µb.staticNetFilteringEngine.freeze();
         µb.cosmeticFilteringEngine.freeze();
     };
 
     var onLoaded = function(details) {
-        if ( details.error ) {
-            return;
-        }
         // https://github.com/chrisaljoudi/uBlock/issues/976
         // If we reached this point, the filter quite probably needs to be
         // added for sure: do not try to be too smart, trying to avoid
@@ -270,7 +269,7 @@
         }
 
         // Now get user's selection of lists
-        vAPI.storage.get(
+        vAPI.storage.preferences.get(
             { 'remoteBlacklists': availableLists },
             onSelectedListsLoaded
         );
@@ -342,7 +341,7 @@
 
         µb.staticNetFilteringEngine.freeze();
         µb.cosmeticFilteringEngine.freeze();
-        vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists });
+        vAPI.storage.preferences.set({ 'remoteBlacklists': µb.remoteBlacklists });
 
         //quickProfiler.stop(0);
 
@@ -600,7 +599,7 @@
         filterLists[location].off = state;
     }
 
-    vAPI.storage.set({ 'remoteBlacklists': filterLists });
+    vAPI.storage.preferences.set({ 'remoteBlacklists': filterLists });
 };
 
 /******************************************************************************/