this fixes #376

src/js/assets.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     µBlock - a Chromium browser extension to block requests.
-    Copyright (C) 2014 Raymond Hill
+    Copyright (C) 2014-2015 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -274,7 +274,7 @@ var getTextFileFromURL = function(url, onLoad, onError) {
     // console.log('µBlock> getTextFileFromURL("%s"):', url);
     var xhr = new XMLHttpRequest();
     xhr.open('get', url, true);
-    xhr.timeout = 15000;
+    xhr.timeout = 30000;
     xhr.onload = onResponseReceived;
     xhr.onerror = onError;
     xhr.ontimeout = onError;
@@ -1044,7 +1044,194 @@ exports.purgeAll = function(callback) {
 
 return exports;
 
+})();
+
 /******************************************************************************/
+/******************************************************************************/
+
+µBlock.assetUpdater = (function() {
+
+'use strict';
+
+/******************************************************************************/
+
+var µb = µBlock;
+
+var updateDaemonTimerPeriod =      11 * 60 * 1000; // 11 minutes
+var updateCycleFirstPeriod  =       7 * 60 * 1000; //  7 minutes
+var updateCycleNextPeriod   = 11 * 60 * 60 * 1000; // 11 hours
+var updateCycleTime = 0;
+
+var toUpdate = {};
+var toUpdateCount = 0;
+var updated = {};
+var updatedCount = 0;
+var metadata = null;
+
+var onStart = null;
+var onCompleted = null;
+
+var exports = {};
+
+/******************************************************************************/
+
+var onAssetUpdated = function(details) {
+    var path = details.path;
+    if ( details.error ) {
+        console.debug('assets.js > µBlock.assetUpdater/onAssetUpdated: "%s" failed', path);
+        return;
+    }
+    console.debug('assets.js > µBlock.assetUpdater/onAssetUpdated: "%s"', path);
+    updated[path] = true;
+    updatedCount += 1;
+
+    // New data available: selfie is now invalid
+    µb.destroySelfie();
+};
+
+/******************************************************************************/
+
+var updateOne = function() {
+    var metaEntry;
+    for ( var path in toUpdate ) {
+        if ( toUpdate.hasOwnProperty(path) === false ) {
+            continue;
+        }
+        if ( toUpdate[path] !== true ) {
+            continue;
+        }
+        toUpdate[path] = false;
+        toUpdateCount -= 1;
+        if ( metadata.hasOwnProperty(path) === false ) {
+            continue;
+        }
+        metaEntry = metadata[path];
+        if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) {
+            continue;
+        }
+        console.debug('assets.js > µBlock.assetUpdater/updateOne: assets.get("%s")', path);
+        µb.assets.get(path, onAssetUpdated);
+        break;
+    }
+};
+
+/******************************************************************************/
+
+var onMetadataReady = function(response) {
+    metadata = response;
+    updateOne();
+};
+
+/******************************************************************************/
+
+var updateDaemon = function() {
+    setTimeout(updateDaemon, updateDaemonTimerPeriod);
+
+    µb.assets.autoUpdate = µb.userSettings.autoUpdate;
+
+    if ( µb.assets.autoUpdate !== true ) {
+        return;
+    }
+
+    // Start an update cycle?
+    if ( updateCycleTime !== 0 ) {
+        if ( Date.now() >= updateCycleTime ) {
+            console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle started');
+            reset();
+            if ( onStart !== null ) {
+                onStart();
+            }
+        }
+        return;
+    }
+
+    // Any asset to update?
+    if ( toUpdateCount !== 0 ) {
+        if ( metadata === null ) {
+            µb.assets.metadata(onMetadataReady);
+        } else {
+            updateOne();
+        }
+        return;
+    }
+    // Nothing left to update
+
+    // If anything was updated, notify listener
+    if ( updatedCount !== 0 ) {
+        if ( onCompleted !== null ) {
+            console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle completed');
+            onCompleted({
+                updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy
+                updatedCount: updatedCount
+            });
+        }
+    }
+
+    // Schedule next update cycle
+    if ( updateCycleTime === 0 ) {
+        reset();
+        console.debug('assets.js > µBlock.assetUpdater/updateDaemon: update cycle re-scheduled');
+        updateCycleTime = Date.now() + updateCycleNextPeriod;
+    }
+};
+
+setTimeout(updateDaemon, updateDaemonTimerPeriod);
+
+/******************************************************************************/
+
+var reset = function() {
+    toUpdate = {};
+    toUpdateCount = 0;
+    updated = {};
+    updatedCount = 0;
+    updateCycleTime = 0;
+    metadata = null;
+};
+
+/******************************************************************************/
+
+exports.onStart = {
+    addEventListener: function(callback) {
+        onStart = callback || null;
+        if ( onStart !== null ) {
+            updateCycleTime = Date.now() + updateCycleFirstPeriod;
+        }
+    }
+};
+
+/******************************************************************************/
+
+exports.onCompleted = {
+    addEventListener: function(callback) {
+        onCompleted = callback || null;
+    }
+};
+
+/******************************************************************************/
+
+exports.add = function(path) {
+    if ( toUpdate.hasOwnProperty(path) ) {
+        return;
+    }
+
+    console.debug('assets.js > µBlock.assetUpdater.add("%s")', path);
+
+    toUpdate[path] = true;
+    toUpdateCount += 1;
+};
+
+/******************************************************************************/
+
+// Typically called when an update has been forced.
+
+exports.restart = function() {
+    reset();
+    updateCycleTime = Date.now() + updateCycleNextPeriod;
+};
+
+/******************************************************************************/
+
+return exports;
 
 })();
 

src/js/background.js

@@ -67,7 +67,6 @@ return {
 
     // https://github.com/gorhill/uBlock/issues/180
     // Whitelist directives need to be loaded once the PSL is available
-    netExceptionList: {}, // TODO: remove once all users are up to date
     netWhitelist: {},
     netWhitelistModifyTime: 0,
     netWhitelistDefault: [
@@ -85,7 +84,7 @@ return {
 
     // EasyList, EasyPrivacy and many others have an 4-day update period,
     // as per list headers.
-    updateAssetsEvery: 75 * oneHour + 23 * oneMinute + 53 * oneSecond + 605,
+    updateAssetsEvery: 97 * oneHour,
     projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/',
     userFiltersPath: 'assets/user/filters.txt',
     pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat',
@@ -111,11 +110,8 @@ return {
     remoteBlacklists: {
     },
 
-    firstUpdateAfter: 5 * oneMinute,
-    nextUpdateAfter: 7 * oneHour,
-
     selfieMagic: 'bizhviclttie',
-    selfieAfter: 7 * oneMinute,
+    selfieAfter: 23 * oneMinute,
 
     pageStores: {},
 

src/js/mirrors.js

@@ -498,6 +498,9 @@ var parseMirrorCandidates = function(rawText) {
 /******************************************************************************/
 
 var load = function() {
+    if ( loaded ) {
+        return;
+    }
     loaded = true;
 
     var onMirrorCandidatesReady = function(details) {

src/js/start.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     µBlock - a Chromium browser extension to block requests.
-    Copyright (C) 2014 Raymond Hill
+    Copyright (C) 2014-2015 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,97 +19,148 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-/* global µBlock */
+/* global vAPI, µBlock */
 
 /******************************************************************************/
 
-// Automatic update of non-user assets
+// Load all: executed once.
-// https://github.com/gorhill/httpswitchboard/issues/334
 
-µBlock.updater = (function() {
+(function() {
-
-'use strict';
 
 /******************************************************************************/
 
-var µb = µBlock;
+// Final initialization steps after all needed assets are in memory.
+// - Initialize internal state with maybe already existing tabs.
+// - Schedule next update operation.
 
-var jobCallback = function() {
+var onAllReady = function() {
-    // Simpler to fire restart here, and safe given how far this will happen
+    var µb = µBlock;
-    // in the future.
-    restart();
 
-    // If auto-update is disabled, check again in a while.
+    // https://github.com/gorhill/uBlock/issues/184
-    if ( µb.userSettings.autoUpdate !== true ) {
+    // Check for updates not too far in the future.
+    µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb));
+    µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb));
+
+    // Important: remove barrier to remote fetching, this was useful only
+    // for launch time.
+    µb.assets.allowRemoteFetch = true;
+
+    vAPI.onLoadAllCompleted();
+};
+
+/******************************************************************************/
+
+// To bring older versions up to date
+
+var onVersionReady = function(bin) {
+    var µb = µBlock;
+    var lastVersion = bin.version || '0.0.0.0';
+
+    // Whitelist some key scopes by default
+    if ( lastVersion.localeCompare('0.8.6.0') < 0 ) {
+        µb.netWhitelist = µb.whitelistFromString(
+            µb.stringFromWhitelist(µb.netWhitelist) + 
+            '\n' + 
+            µb.netWhitelistDefault
+        );
+        µb.saveWhitelist();
+    }
+
+    vAPI.storage.set({ version: vAPI.app.version });
+    onAllReady();
+};
+
+/******************************************************************************/
+
+// Filter lists
+// Whitelist
+
+var countdown = 2;
+var doCountdown = function() {
+    countdown -= 1;
+    if ( countdown !== 0 ) {
         return;
     }
-
+    // Last step: do whatever is necessary when version changes
-    var onMetadataReady = function(metadata) {
+    vAPI.storage.get('version', onVersionReady);
-        // Check PSL
-        var mdEntry = metadata[µb.pslPath];
-        if ( mdEntry.repoObsolete ) {
-            // console.log('µBlock.updater> updating all updatable assets');
-            µb.loadUpdatableAssets({ update: true });
-            return;
-        }
-        // Check used filter lists
-        var lists = µb.remoteBlacklists;
-        for ( var path in lists ) {
-            if ( lists.hasOwnProperty(path) === false ) {
-                continue;
-            }
-            if ( lists[path].off ) {
-                continue;
-            }
-            if ( metadata.hasOwnProperty(path) === false ) {
-                continue;
-            }
-            mdEntry = metadata[path];
-            if ( mdEntry.cacheObsolete || mdEntry.repoObsolete ) {
-                // console.log('µBlock.updater> updating only filter lists');
-                µb.loadUpdatableAssets({ update: true, psl: false });
-                return;
-            }
-        }
-
-        // console.log('µBlock.updater> all is up to date');
-    };
-
-    µb.assets.metadata(onMetadataReady);
 };
 
-// https://www.youtube.com/watch?v=cIrGQD84F1g
-
 /******************************************************************************/
 
-var restart = function(after) {
+// Filters are in memory.
-    if ( after === undefined ) {
+// Filter engines need PSL to be ready.
-        after = µb.nextUpdateAfter;
+
+var onFiltersReady = function() {
+    doCountdown();
+};
+
+/******************************************************************************/
+
+// https://github.com/gorhill/uBlock/issues/226
+// Whitelist in memory.
+// Whitelist parser needs PSL to be ready.
+// gorhill 2014-12-15: not anymore
+
+var onWhitelistReady = function() {
+    doCountdown();
+};
+
+/******************************************************************************/
+
+// Load order because dependencies:
+// User settings -> PSL -> [filter lists]
+
+var onPSLReady = function() {
+    µBlock.loadFilterLists(onFiltersReady);
+};
+
+/******************************************************************************/
+
+// If no selfie available, take the long way, i.e. load and parse
+// raw data.
+
+var onSelfieReady = function(success) {
+    if ( success === true ) {
+        onFiltersReady();
+        return;
     }
-
+    µBlock.loadPublicSuffixList(onPSLReady);
-    µb.asyncJobs.add(
-        'autoUpdateAssets',
-        null,
-        jobCallback,
-        after,
-        false
-    );
 };
 
 /******************************************************************************/
 
-return {
+// User settings are in memory
-    restart: restart
+
+var onUserSettingsReady = function(userSettings) {
+    var µb = µBlock;
+
+    // https://github.com/gorhill/uBlock/issues/426
+    // Important: block remote fetching for when loading assets at launch
+    // time.
+    µb.assets.allowRemoteFetch = false;
+    µb.assets.autoUpdate = userSettings.autoUpdate;
+    µb.fromSelfie(onSelfieReady);
+
+    // https://github.com/gorhill/uBlock/issues/540
+    // Disabling local mirroring for the time being
+    userSettings.experimentalEnabled = false;
+    µb.mirrors.toggle(false /* userSettings.experimentalEnabled */);
+
+    µb.contextMenu.toggle(userSettings.contextMenuEnabled);
+    µb.permanentFirewall.fromString(userSettings.dynamicFilteringString);
+    µb.sessionFirewall.assign(µb.permanentFirewall);
+
+    // Remove obsolete setting
+    delete userSettings.logRequests;
+    µb.XAL.keyvalRemoveOne('logRequests');
 };
 
+µBlock.loadUserSettings(onUserSettingsReady);
+µBlock.loadWhitelist(onWhitelistReady);
+µBlock.loadLocalSettings();
+
 /******************************************************************************/
 
 })();
 
 /******************************************************************************/
-
-// Load everything
-
-µBlock.load();
-
-/******************************************************************************/

src/js/storage.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     µBlock - a Chromium browser extension to block requests.
-    Copyright (C) 2014 Raymond Hill
+    Copyright (C) 2014-2015 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -104,17 +104,6 @@
 µBlock.loadWhitelist = function(callback) {
     var onWhitelistLoaded = function(store) {
         var µb = µBlock;
-        // Backward compatibility after fix to #5
-        // TODO: remove once all users are up to date with latest version.
-        if ( store.netExceptionList ) {
-            if ( store.netWhitelist === '' ) {
-                store.netWhitelist = Object.keys(store.netExceptionList).join('\n');
-                if ( store.netWhitelist !== '' ) {
-                    vAPI.storage.set({ 'netWhitelist': store.netWhitelist });
-                }
-            }
-            vAPI.storage.remove('netExceptionList');
-        }
         µb.netWhitelist = µb.whitelistFromString(store.netWhitelist);
         µb.netWhitelistModifyTime = Date.now();
 
@@ -124,8 +113,7 @@
     };
 
     var bin = {
-        'netWhitelist': this.netWhitelistDefault,
+        'netWhitelist': this.netWhitelistDefault
-        'netExceptionList': ''
     };
     vAPI.storage.get(bin, onWhitelistLoaded);
 };
@@ -377,7 +365,6 @@
     var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/;
     var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/;
     //var reAsciiSegment = /^[\x21-\x7e]+$/;
-    var matches;
     var lineBeg = 0, lineEnd, currentLineBeg;
     var line, lineRaw, c, pos;
 
@@ -534,7 +521,7 @@
 
     var onFiltersReady = function() {
         if ( update ) {
-            µb.updater.restart();
+            µb.assetUpdater.restart();
         }
     };
 
@@ -560,7 +547,7 @@
         cosmeticFilteringEngine: this.cosmeticFilteringEngine.toSelfie()
     };
     vAPI.storage.set({ selfie: selfie });
-    // console.log('µBlock.toSelfie> made a selfie!');
+    console.log('storage.js > µBlock.toSelfie()');
 };
 
 // This is to be sure the selfie is generated in a sane manner: the selfie will
@@ -613,117 +600,62 @@
 
 µBlock.destroySelfie = function() {
     vAPI.storage.remove('selfie');
+    this.asyncJobs.remove('toSelfie');
+    console.log('storage.js > µBlock.destroySelfie()');
 };
 
 /******************************************************************************/
 
-// Load all
+µBlock.updateStartHandler = function() {
-
-µBlock.load = function() {
     var µb = this;
-    var fromSelfie = false;
+    var onListsReady = function(lists) {
-
+        for ( var location in lists ) {
-    // Final initialization steps after all needed assets are in memory.
+            if ( lists.hasOwnProperty(location) === false ) {
-    // - Initialize internal state with maybe already existing tabs.
+                continue;
-    // - Schedule next update operation.
+            }
-    var onAllReady = function() {
+            if ( lists[location].off ) {
-        // https://github.com/gorhill/uBlock/issues/426
+                continue;
-        // Important: remove barrier to remote fetching, this was useful only
+            }
-        // for launch time.
+            µb.assetUpdater.add(location);
-        µb.assets.allowRemoteFetch = true;
-
-        // https://github.com/gorhill/uBlock/issues/184
-        // Check for updates not too far in the future.
-        µb.updater.restart(µb.firstUpdateAfter);
-
-        vAPI.onLoadAllCompleted();
-    };
-
-    // To bring older versions up to date
-    var onVersionReady = function(bin) {
-        var lastVersion = bin.version || '0.0.0.0';
-
-        // Whitelist some key scopes by default
-        if ( lastVersion.localeCompare('0.8.6.0') < 0 ) {
-            µb.netWhitelist = µb.whitelistFromString(
-                µb.stringFromWhitelist(µb.netWhitelist) + 
-                '\n' + 
-                µb.netWhitelistDefault
-            );
-            µb.saveWhitelist();
         }
-
-        vAPI.storage.set({ version: vAPI.app.version });
-        onAllReady();
     };
-
+    this.getAvailableLists(onListsReady);
-    // Filter lists
+    this.assetUpdater.add(this.pslPath);
-    // Whitelist
+    this.assetUpdater.add('assets/ublock/mirror-candidates.txt');
-    var countdown = 2;
+};
-    var doCountdown = function() {
+
-        countdown -= 1;
+/******************************************************************************/
-        if ( countdown !== 0 ) {
+
-            return;
+µBlock.updateCompleteHandler = function(details) {
-        }
+    var µb = this;
-        // Last step: do whatever is necessary when version changes
+    var updatedCount = details.updatedCount;
-        vAPI.storage.get('version', onVersionReady);
+
-    };
+    // Assets are supposed to have been all updated, avoid fetching from
-
+    // remote servers.
-    // Filters are in memory.
+    µb.assets.allowRemoteFetch = false;
-    // Filter engines need PSL to be ready.
+
-    var onFiltersReady = function() {
+    var onFiltersReady = function() {
-        doCountdown();
+        µb.assets.allowRemoteFetch = true;
-    };
+    };
-
+
-    // https://github.com/gorhill/uBlock/issues/226
+    var onPSLReady = function() {
-    // Whitelist in memory.
+        if ( updatedCount !== 0 ) {
-    // Whitelist parser needs PSL to be ready.
+            console.debug('storage.js > µBlock.updateCompleteHandler: reloading filter lists');
-    // gorhill 2014-12-15: not anymore
+            µb.loadFilterLists(onFiltersReady);
-    var onWhitelistReady = function() {
+        } else {
-        doCountdown();
+            onFiltersReady();
-    };
+        }
-
+    };
-    // Load order because dependencies:
+
-    // User settings -> PSL -> [filter lists]
+    if ( details.hasOwnProperty('assets/ublock/mirror-candidates.txt') ) {
-    var onPSLReady = function() {
+        /* TODO */
-        µb.loadFilterLists(onFiltersReady);
+    }
-    };
+
-
+    if ( details.hasOwnProperty(this.pslPath) ) {
-    // If no selfie available, take the long way, i.e. load and parse
+        console.debug('storage.js > µBlock.updateCompleteHandler: reloading PSL');
-    // raw data.
+        this.loadPublicSuffixList(onPSLReady);
-    var onSelfieReady = function(success) {
+        updatedCount -= 1;
-        if ( success === true ) {
+    } else {
-            fromSelfie = true;
+        onPSLReady();
-            onFiltersReady();
+    }
-            return;
-        }
-        µb.loadPublicSuffixList(onPSLReady);
-    };
-
-    // User settings are in memory
-    var onUserSettingsReady = function(userSettings) {
-        // https://github.com/gorhill/uBlock/issues/426
-        // Important: block remote fetching for when loading assets at launch
-        // time.
-        µb.assets.allowRemoteFetch = false;
-        µb.assets.autoUpdate = userSettings.autoUpdate;
-        µb.fromSelfie(onSelfieReady);
-
-        // https://github.com/gorhill/uBlock/issues/540
-        // Disabling local mirroring for the time being
-        userSettings.experimentalEnabled = false;
-        µb.mirrors.toggle(false /* userSettings.experimentalEnabled */);
-
-        µb.contextMenu.toggle(userSettings.contextMenuEnabled);
-        µb.permanentFirewall.fromString(userSettings.dynamicFilteringString);
-        µb.sessionFirewall.assign(µb.permanentFirewall);
-
-        // Remove obsolete setting
-        delete userSettings.logRequests;
-        µb.XAL.keyvalRemoveOne('logRequests');
-    };
-
-    this.loadUserSettings(onUserSettingsReady);
-    this.loadWhitelist(onWhitelistReady);
-    this.loadLocalSettings();
 };