From 288ea7b4cc0fdcf3e5e42c38f2e59be8add34c62 Mon Sep 17 00:00:00 2001 From: Chris Date: Thu, 7 May 2015 23:47:18 -0600 Subject: [PATCH] Storage: distinguish preferences from caches (addresses #1366) --- platform/chromium/vapi-background.js | 1 + platform/firefox/vapi-background.js | 2 + platform/safari/vapi-background.js | 132 ++++++++++++++++++++++++--- src/js/background.js | 5 +- src/js/messaging.js | 23 +++-- src/js/start.js | 22 ++++- src/js/storage.js | 27 +++--- 7 files changed, 170 insertions(+), 42 deletions(-) diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index 095fb4738..baa02f618 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -71,6 +71,7 @@ vAPI.app.restart = function() { // chrome.storage.local.get(null, function(bin){ console.debug('%o', bin); }); vAPI.storage = chrome.storage.local; +vAPI.storage.preferences = vAPI.storage; /******************************************************************************/ diff --git a/platform/firefox/vapi-background.js b/platform/firefox/vapi-background.js index d6817b21e..2e366d513 100644 --- a/platform/firefox/vapi-background.js +++ b/platform/firefox/vapi-background.js @@ -276,6 +276,8 @@ vAPI.storage = { } }; +vAPI.storage.preferences = vAPI.storage; + /******************************************************************************/ var windowWatcher = { diff --git a/platform/safari/vapi-background.js b/platform/safari/vapi-background.js index 6baece097..f9bf50b90 100644 --- a/platform/safari/vapi-background.js +++ b/platform/safari/vapi-background.js @@ -105,16 +105,7 @@ size: storageQuota, storeName: "keyvaluepairs" }); - var oldSettings = safari.extension.settings; // To smoothly transition users - if(oldSettings.hasOwnProperty("version")) { // Old 'storage'! - for(var key in oldSettings) { - if(!oldSettings.hasOwnProperty(key) || key === "open_prefs") { - continue; - } - localforage.setItem(key, oldSettings[key]); - } - oldSettings.clear(); - } + vAPI.storage = { QUOTA_BYTES: storageQuota, // copied from Info.plist @@ -212,6 +203,7 @@ }, clear: function(callback) { + this.preferences.clear(); localforage.clear(function() { callback(); }); @@ -229,6 +221,124 @@ }); } }; + /******************************************************************************/ + + if(!safari.extension.settings.migratedStorage) { + var migrationMap = { + "cached_asset_content://assets/user/filters.txt": "userFilters" + }; + + var delayed = []; + + vAPI.storage.preferences = { + get: function(a, b) { + delayed.push(settingsStorage.get.bind(settingsStorage, a, b)); + }, + set: function(a, b) { + delayed.push(settingsStorage.set.bind(settingsStorage, a, b)); + }, + remove: function(a, b) { + delayed.push(settingsStorage.remove.bind(settingsStorage, a, b)); + }, + clear: function() { + delayed.push(settingsStorage.clear.bind(settingsStorage)); + }, + }; + + localforage.iterate(function(value, key) { + if(migrationMap[key]) { + safari.extension.settings[migrationMap[key]] = value; + return; + } + if(key.lastIndexOf("cached_asset", 0) === 0) { + return; + } + safari.extension.settings[key] = value; + localforage.removeItem(key); + }, function() { + var func; + while(func = delayed.pop()) { + func(); + } + delayed = null; + vAPI.storage.preferences = settingsStorage; + }); + safari.extension.settings.migratedStorage = true; + } + + var settingsStorage = { + _storage: safari.extension.settings, + get: function(keys, callback) { + if(typeof callback !== "function") { + return; + } + + var i, value, result = {}; + + if(keys === null) { + for(i in this._storage) { + if(!this._storage.hasOwnProperty(i)) continue; + value = this._storage[i]; + if(typeof value === "string") { + result[i] = JSON.parse(value); + } + } + } + else if(typeof keys === "string") { + value = this._storage[keys]; + if(typeof value === "string") { + result[keys] = JSON.parse(value); + } + } + else if(Array.isArray(keys)) { + for(i = 0; i < keys.length; i++) { + value = this._storage[i]; + + if(typeof value === "string") { + result[keys[i]] = JSON.parse(value); + } + } + } + else if(typeof keys === "object") { + for(i in keys) { + if(!keys.hasOwnProperty(i)) { + continue; + } + value = this._storage[i]; + + if(typeof value === "string") { + result[i] = JSON.parse(value); + } + else { + result[i] = keys[i]; + } + } + } + callback(result); + }, + set: function(details, callback) { + for(var key in details) { + if(!details.hasOwnProperty(key)) { + continue; + } + this._storage.setItem(key, JSON.stringify(details[key])); + } + if(typeof callback === "function") { + callback(); + } + }, + remove: function(keys) { + if(typeof keys === "string") { + keys = [keys]; + } + for(var i = 0; i < keys.length; i++) { + this._storage.removeItem(keys[i]); + } + }, + clear: function() { + this._storage.clear(); + } + }; /******************************************************************************/ @@ -567,7 +677,7 @@ if(iconState.dirty & 2) { icon.badge = iconState.badge; } - if(iconState.dirty & 1 && icon.image !== ICON_URLS[iconState.img]) { + if((iconState.dirty & 1) && icon.image !== ICON_URLS[iconState.img]) { icon.image = ICON_URLS[iconState.img]; } iconState.dirty = 0; diff --git a/src/js/background.js b/src/js/background.js index 0638be3c3..f022bb2eb 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -63,7 +63,7 @@ return { firewallPaneMinimized: true, parseAllABPHideFilters: true, requestLogMaxEntries: 1000, - showIconBadge: true + showIconBadge: true, }, // https://github.com/chrisaljoudi/uBlock/issues/180 @@ -78,6 +78,8 @@ return { 'loopconversation.about-scheme', 'opera-scheme' ].join('\n').trim(), + + userFiltersPath: "assets/user/filters.txt", localSettings: { blockedRequestCount: 0, @@ -103,7 +105,6 @@ return { // as per list headers. updateAssetsEvery: 97 * oneHour, projectServerRoot: 'https://raw.githubusercontent.com/chrisaljoudi/uBlock/master/', - userFiltersPath: 'assets/user/filters.txt', pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', // permanent lists diff --git a/src/js/messaging.js b/src/js/messaging.js index 424b62487..97570631b 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -797,10 +797,10 @@ var onMessage = function(request, sender, callback) { // Async switch ( request.what ) { case 'readUserFilters': - return µb.assets.get(µb.userFiltersPath, callback); + return µb.loadUserFilters(callback); case 'writeUserFilters': - return µb.assets.put(µb.userFiltersPath, request.content, callback); + return µb.saveUserFilters(request.content, callback); default: break; @@ -1083,7 +1083,7 @@ var backupUserData = function(callback) { µb.restoreBackupSettings.lastBackupFile = filename; µb.restoreBackupSettings.lastBackupTime = Date.now(); - vAPI.storage.set(µb.restoreBackupSettings); + vAPI.storage.preferences.set(µb.restoreBackupSettings); getLocalData(callback); }; @@ -1093,8 +1093,7 @@ var backupUserData = function(callback) { µb.extractSelectedFilterLists(onSelectedListsReady); }; - - µb.assets.get('assets/user/filters.txt', onUserFiltersReady); + µb.loadUserFilters(onUserFiltersReady); }; /******************************************************************************/ @@ -1111,19 +1110,19 @@ var restoreUserData = function(request) { var onAllRemoved = function() { // Be sure to adjust `countdown` if adding/removing anything below - µb.keyvalSetOne('version', userData.version); + µb.keyvalSetOnePref('version', userData.version); µBlock.saveLocalSettings(true); - vAPI.storage.set(userData.userSettings, onCountdown); - µb.keyvalSetOne('remoteBlacklists', userData.filterLists, onCountdown); - µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown); + vAPI.storage.preferences.set(userData.userSettings, onCountdown); + µb.keyvalSetOnePref('remoteBlacklists', userData.filterLists, onCountdown); + µb.keyvalSetOnePref('netWhitelist', userData.netWhitelist || '', onCountdown); // With versions 0.9.2.4-, dynamic rules were saved within the // `userSettings` object. No longer the case. var s = userData.dynamicFilteringString || userData.userSettings.dynamicFilteringString || ''; - µb.keyvalSetOne('dynamicFilteringString', s, onCountdown); + µb.keyvalSetOnePref('dynamicFilteringString', s, onCountdown); - µb.assets.put('assets/user/filters.txt', userData.userFilters, onCountdown); - vAPI.storage.set({ + µb.keyvalSetOnePref('userFilters', userData.userFilters, onCountdown); + vAPI.storage.preferences.set({ lastRestoreFile: request.file || '', lastRestoreTime: Date.now(), lastBackupFile: '', diff --git a/src/js/start.js b/src/js/start.js index b660644db..6337708c5 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -169,12 +169,16 @@ var onSystemSettingsReady = function(fetched) { if ( mustSaveSystemSettings ) { fetched.selfie = null; µb.destroySelfie(); - vAPI.storage.set(µb.systemSettings, µb.noopFunc); + vAPI.storage.preferences.set(µb.systemSettings, µb.noopFunc); } }; /******************************************************************************/ +var onUserFiltersReady = function(userFilters) { + µb.saveUserFilters(userFilters); // we need this because of migration +}; + var onFirstFetchReady = function(fetched) { // Order is important -- do not change: onSystemSettingsReady(fetched); @@ -182,6 +186,7 @@ var onFirstFetchReady = function(fetched) { onUserSettingsReady(fetched); fromFetch(µb.restoreBackupSettings, fetched); onNetWhitelistReady(fetched.netWhitelist); + onUserFiltersReady(fetched.userFilters); onVersionReady(fetched.version); // If we have a selfie, skip loading PSL, filters @@ -195,6 +200,16 @@ var onFirstFetchReady = function(fetched) { /******************************************************************************/ +var onPrefFetchReady = function(fetched) { + fetched.userFilters = fetched.userFilters || fetched["cached_asset_content://assets/user/filters.txt"]; + vAPI.storage.get({"selfie": null}, function(res) { + fetched["selfie"] = res["selfie"]; + onFirstFetchReady(fetched); + }); +}; + +/******************************************************************************/ + var toFetch = function(from, fetched) { for ( var k in from ) { if ( from.hasOwnProperty(k) === false ) { @@ -230,7 +245,8 @@ return function() { 'lastBackupFile': '', 'lastBackupTime': 0, 'netWhitelist': '', - 'selfie': null, + 'userFilters': '', + 'cached_asset_content://assets/user/filters.txt': '', 'selfieMagic': '', 'version': '0.0.0.0' }; @@ -239,7 +255,7 @@ return function() { toFetch(µb.userSettings, fetchableProps); toFetch(µb.restoreBackupSettings, fetchableProps); - vAPI.storage.get(fetchableProps, onFirstFetchReady); + vAPI.storage.preferences.get(fetchableProps, onPrefFetchReady); }; /******************************************************************************/ diff --git a/src/js/storage.js b/src/js/storage.js index fbd26eeb1..d97bc911b 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -38,10 +38,10 @@ /******************************************************************************/ -µBlock.keyvalSetOne = function(key, val, callback) { +µBlock.keyvalSetOnePref = function(key, val, callback) { var bin = {}; bin[key] = val; - vAPI.storage.set(bin, callback || this.noopFunc); + vAPI.storage.preferences.set(bin, callback || this.noopFunc); }; /******************************************************************************/ @@ -54,7 +54,7 @@ return; } this.localSettingsSaveTime = Date.now(); - vAPI.storage.set(this.localSettings); + vAPI.storage.preferences.set(this.localSettings); }; /******************************************************************************/ @@ -72,13 +72,13 @@ /******************************************************************************/ µBlock.saveUserSettings = function() { - vAPI.storage.set(this.userSettings); + vAPI.storage.preferences.set(this.userSettings); }; /******************************************************************************/ µBlock.savePermanentFirewallRules = function() { - this.keyvalSetOne('dynamicFilteringString', this.permanentFirewall.toString()); + this.keyvalSetOnePref('dynamicFilteringString', this.permanentFirewall.toString()); }; /******************************************************************************/ @@ -87,7 +87,7 @@ var bin = { 'netWhitelist': this.stringFromWhitelist(this.netWhitelist) }; - vAPI.storage.set(bin); + vAPI.storage.preferences.set(bin); this.netWhitelistModifyTime = Date.now(); }; @@ -135,7 +135,9 @@ /******************************************************************************/ µBlock.saveUserFilters = function(content, callback) { - return this.assets.put(this.userFiltersPath, content, callback); + vAPI.storage.preferences.set({"userFilters": content}); + this.assets.put(this.userFiltersPath, content, callback); + return; }; /******************************************************************************/ @@ -165,15 +167,12 @@ var deltaEntryUsedCount = deltaEntryCount - (snfe.duplicateCount + cfe.duplicateCount - duplicateCount); entry.entryCount += deltaEntryCount; entry.entryUsedCount += deltaEntryUsedCount; - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.preferences.set({ 'remoteBlacklists': µb.remoteBlacklists }); µb.staticNetFilteringEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); }; var onLoaded = function(details) { - if ( details.error ) { - return; - } // https://github.com/chrisaljoudi/uBlock/issues/976 // If we reached this point, the filter quite probably needs to be // added for sure: do not try to be too smart, trying to avoid @@ -270,7 +269,7 @@ } // Now get user's selection of lists - vAPI.storage.get( + vAPI.storage.preferences.get( { 'remoteBlacklists': availableLists }, onSelectedListsLoaded ); @@ -342,7 +341,7 @@ µb.staticNetFilteringEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.preferences.set({ 'remoteBlacklists': µb.remoteBlacklists }); //quickProfiler.stop(0); @@ -600,7 +599,7 @@ filterLists[location].off = state; } - vAPI.storage.set({ 'remoteBlacklists': filterLists }); + vAPI.storage.preferences.set({ 'remoteBlacklists': filterLists }); }; /******************************************************************************/