uBlock/src/js/storage.js

/*******************************************************************************

    µBlock - a Chromium browser extension to block requests.
    Copyright (C) 2014 Raymond Hill

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see {http://www.gnu.org/licenses/}.

    Home: https://github.com/gorhill/uBlock
*/

/* global µBlock, vAPI, punycode, publicSuffixList */

'use strict';

/******************************************************************************/

µBlock.getBytesInUse = function() {
    var getBytesInUseHandler = function(bytesInUse) {
        µBlock.storageUsed = bytesInUse;
    };
    vAPI.storage.getBytesInUse(null, getBytesInUseHandler);
};

/******************************************************************************/

µBlock.saveLocalSettings = function(callback) {
    if ( typeof callback !== 'function' ) {
        callback = this.noopFunc;
    }
    vAPI.storage.set(this.localSettings, callback);
};

/******************************************************************************/

µBlock.loadLocalSettings = function() {
    var settingsLoaded = function(store) {
        µBlock.localSettings = store;
    };

    vAPI.storage.get(this.localSettings, settingsLoaded);
};

/******************************************************************************/

// Save local settings regularly. Not critical.

µBlock.asyncJobs.add(
    'autoSaveLocalSettings',
    null,
    µBlock.saveLocalSettings.bind(µBlock),
    2 * 60 * 1000,
    true
);

/******************************************************************************/

µBlock.saveUserSettings = function() {
    vAPI.storage.set(this.userSettings);
};

/******************************************************************************/

µBlock.loadUserSettings = function(callback) {
    var settingsLoaded = function(store) {
        µBlock.userSettings = store;
        if ( typeof callback === 'function' ) {
            callback(µBlock.userSettings);
        }
    };

    vAPI.storage.get(this.userSettings, settingsLoaded);
};

/******************************************************************************/

µBlock.saveDynamicRules = function() {
    this.userSettings.dynamicFilteringString = this.dynamicNetFilteringEngine.toString();
    this.XAL.keyvalSetOne('dynamicFilteringString', this.userSettings.dynamicFilteringString);
};

/******************************************************************************/

µBlock.saveWhitelist = function() {
    var bin = {
        'netWhitelist': this.stringFromWhitelist(this.netWhitelist)
    };
    vAPI.storage.set(bin);
    this.netWhitelistModifyTime = Date.now();
};

/******************************************************************************/

µBlock.loadWhitelist = function(callback) {
    var onWhitelistLoaded = function(store) {
        var µb = µBlock;
        // Backward compatibility after fix to #5
        // TODO: remove once all users are up to date with latest version.
        if ( store.netExceptionList ) {
            if ( store.netWhitelist === '' ) {
                store.netWhitelist = Object.keys(store.netExceptionList).join('\n');
                if ( store.netWhitelist !== '' ) {
                    vAPI.storage.set({ 'netWhitelist': store.netWhitelist });
                }
            }
            vAPI.storage.remove('netExceptionList');
        }
        µb.netWhitelist = µb.whitelistFromString(store.netWhitelist);
        µb.netWhitelistModifyTime = Date.now();

        if ( typeof callback === 'function' ) {
            callback();
        }
    };

    var bin = {
        'netWhitelist': this.netWhitelistDefault,
        'netExceptionList': ''
    };
    vAPI.storage.get(bin, onWhitelistLoaded);
};

/******************************************************************************/

µBlock.saveUserFilters = function(content, callback) {
    return this.assets.put(this.userFiltersPath, content, callback);
};

/******************************************************************************/

µBlock.loadUserFilters = function(callback) {
    return this.assets.get(this.userFiltersPath, callback);
};

/******************************************************************************/

µBlock.appendUserFilters = function(content) {
    var µb = this;

    var onSaved = function(details) {
        if ( details.error ) {
            return;
        }
        µb.mergeFilterText(content);
        µb.staticNetFilteringEngine.freeze();
        µb.cosmeticFilteringEngine.freeze();
        µb.destroySelfie();
        µb.toSelfieAsync();
    };

    var onLoaded = function(details) {
        if ( details.error ) {
            return;
        }
        if ( details.content.indexOf(content.trim()) !== -1 ) {
            return;
        }
        µb.saveUserFilters(details.content + '\n' + content, onSaved);
    };

    if ( content.length > 0 ) {
        this.loadUserFilters(onLoaded);
    }
};

/******************************************************************************/

µBlock.getAvailableLists = function(callback) {
    var availableLists = {};

    // selected lists
    var onSelectedListsLoaded = function(store) {
        var µb = µBlock;
        var lists = store.remoteBlacklists;
        var locations = Object.keys(lists);
        var location, availableEntry, storedEntry;

        while ( location = locations.pop() ) {
            availableEntry = availableLists[location];
            if ( availableEntry === undefined ) {
                continue;
            }
            storedEntry = lists[location];
            availableEntry.off = storedEntry.off || false;
            µb.assets.setHomeURL(location, availableEntry.homeURL);
            if ( storedEntry.entryCount !== undefined ) {
                availableEntry.entryCount = storedEntry.entryCount;
            }
            if ( storedEntry.entryUsedCount !== undefined ) {
                availableEntry.entryUsedCount = storedEntry.entryUsedCount;
            }
            // This may happen if the list name was pulled from the list content
            if ( availableEntry.title === '' && storedEntry.title !== '' ) {
                availableEntry.title = storedEntry.title;
            }
        }
        callback(availableLists);
    };

    // built-in lists
    var onBuiltinListsLoaded = function(details) {
        var location, locations;
        try {
            locations = JSON.parse(details.content);
        } catch (e) {
            locations = {};
        }
        var entry;
        for ( location in locations ) {
            if ( locations.hasOwnProperty(location) === false ) {
                continue;
            }
            entry = locations[location];
            // https://github.com/gorhill/uBlock/issues/418
            // We now support built-in external filter lists
            if ( /^https?:/.test(location) === false ) {
                location = 'assets/thirdparties/' + location;
            }
            availableLists[location] = entry;
        }

        // Now get user's selection of lists
        vAPI.storage.get(
            { 'remoteBlacklists': availableLists },
            onSelectedListsLoaded
        );
    };

    // permanent lists
    var location;
    var lists = this.permanentLists;
    for ( location in lists ) {
        if ( lists.hasOwnProperty(location) === false ) {
            continue;
        }
        availableLists[location] = lists[location];
    }

    // custom lists
    var c;
    var locations = this.userSettings.externalLists.split('\n');
    for ( var i = 0; i < locations.length; i++ ) {
        location = locations[i].trim();
        c = location.charAt(0);
        if ( location === '' || c === '!' || c === '#' ) {
            continue;
        }
        // Coarse validation
        if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) {
            continue;
        }
        availableLists[location] = {
            title: '',
            group: 'custom',
            external: true
        };
    }

    // get built-in block lists.
    this.assets.get('assets/ublock/filter-lists.json', onBuiltinListsLoaded);
};

/******************************************************************************/

µBlock.loadFilterLists = function(callback) {
    var µb = this;
    var filterlistCount;

    if ( typeof callback !== 'function' ) {
        callback = this.noopFunc;
    }

    var loadBlacklistsEnd = function() {
        µb.staticNetFilteringEngine.freeze();
        µb.cosmeticFilteringEngine.freeze();
        vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists });
        vAPI.messaging.broadcast({ what: 'loadUbiquitousBlacklistCompleted' });
        µb.toSelfieAsync();
        callback();
    };

    var mergeBlacklist = function(details) {
        µb.mergeFilterList(details);
        filterlistCount -= 1;
        if ( filterlistCount === 0 ) {
            loadBlacklistsEnd();
        }
    };

    var loadBlacklistsStart = function(lists) {
        µb.remoteBlacklists = lists;
        µb.staticNetFilteringEngine.reset();
        µb.cosmeticFilteringEngine.reset();
        µb.destroySelfie();
        var locations = Object.keys(lists);
        filterlistCount = locations.length;

        // Load all filter lists which are not disabled
        var location;
        var filterlistLoadingCount = 0;
        while ( filterlistCount !== 0 ) {
            location = locations.pop()
            if ( !location ) {
                break;
            }
            // rhill 2013-12-09:
            // Ignore list if disabled
            // https://github.com/gorhill/httpswitchboard/issues/78
            if ( lists[location].off === false ) {
                filterlistLoadingCount += 1;
                µb.assets.get(location, mergeBlacklist);
                continue;
            }
            filterlistCount -= 1;
        }
        // https://github.com/gorhill/uBlock/issues/695
        // It's may happen not a single filter list is selected
        if ( filterlistLoadingCount === 0 ) {
            loadBlacklistsEnd();
        }
    };

    this.getAvailableLists(loadBlacklistsStart);
};

/******************************************************************************/

µBlock.mergeFilterList = function(details) {
    // console.log('µBlock > mergeFilterList from "%s": "%s..."', details.path, details.content.slice(0, 40));

    var staticNetFilteringEngine = this.staticNetFilteringEngine;
    var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
    var duplicateCount = staticNetFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount;
    var acceptedCount = staticNetFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount;

    this.mergeFilterText(details.content);

    // For convenience, store the number of entries for this
    // blacklist, user might be happy to know this information.
    duplicateCount = staticNetFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount - duplicateCount;
    acceptedCount = staticNetFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount - acceptedCount;

    var filterListMeta = this.remoteBlacklists[details.path];

    filterListMeta.entryCount = acceptedCount;
    filterListMeta.entryUsedCount = acceptedCount - duplicateCount;

    // Try to extract a human-friendly name (works only for
    // ABP-compatible filter lists)
    if ( filterListMeta.title === '' ) {
        var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i);
        if ( matches !== null ) {
            filterListMeta.title = matches[1].trim();
        }
    }
};

/******************************************************************************/

µBlock.mergeFilterText = function(rawText) {
    var rawEnd = rawText.length;

    // Useful references:
    //    https://adblockplus.org/en/filter-cheatsheet
    //    https://adblockplus.org/en/filters
    var staticNetFilteringEngine = this.staticNetFilteringEngine;
    var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
    var parseCosmeticFilters = this.userSettings.parseAllABPHideFilters;

    var reIsCosmeticFilter = /#[@#]/;
    var reIsWhitespaceChar = /\s/;
    var reMaybeLocalIp = /^[\d:f]/;
    var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/;
    var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/;
    //var reAsciiSegment = /^[\x21-\x7e]+$/;
    var matches;
    var lineBeg = 0, lineEnd, currentLineBeg;
    var line, lineRaw, c, pos;

    while ( lineBeg < rawEnd ) {
        lineEnd = rawText.indexOf('\n', lineBeg);
        if ( lineEnd === -1 ) {
            lineEnd = rawText.indexOf('\r', lineBeg);
            if ( lineEnd === -1 ) {
                lineEnd = rawEnd;
            }
        }

        // rhill 2014-04-18: The trim is important here, as without it there
        // could be a lingering `\r` which would cause problems in the
        // following parsing code.
        line = lineRaw = rawText.slice(lineBeg, lineEnd).trim();
        currentLineBeg = lineBeg;
        lineBeg = lineEnd + 1;

        if ( line.length === 0 ) {
            continue;
        }

        // Strip comments
        c = line.charAt(0);
        if ( c === '!' || c === '[' ) {
            continue;
        }

        // Parse or skip cosmetic filters
        // All cosmetic filters are caught here
        if ( parseCosmeticFilters ) {
            if ( cosmeticFilteringEngine.add(line) ) {
                continue;
            }
        } else if ( reIsCosmeticFilter.test(line) ) {
            continue;
        }

        // Whatever else is next can be assumed to not be a cosmetic filter

        // Most comments start in first column
        if ( c === '#' ) {
            continue;
        }

        // Catch comments somewhere on the line
        // Remove:
        //   ... #blah blah blah
        //   ... # blah blah blah
        // Don't remove:
        //   ...#blah blah blah
        // because some ABP filters uses the `#` character (URL fragment)
        pos = line.indexOf('#');
        if ( pos !== -1 && reIsWhitespaceChar.test(line.charAt(pos - 1)) ) {
            line = line.slice(0, pos).trim();
        }

        // https://github.com/gorhill/httpswitchboard/issues/15
        // Ensure localhost et al. don't end up in the ubiquitous blacklist.
        // With hosts files, we need to remove local IP redirection
        if ( reMaybeLocalIp.test(c) ) {
            // Ignore hosts file redirect configuration
            // 127.0.0.1 localhost
            // 255.255.255.255 broadcasthost
            if ( reIsLocalhostRedirect.test(line) ) {
                continue;
            }
            line = line.replace(reLocalIp, '').trim();
        }
    
        if ( line.length === 0 ) {
            continue;
        }

        // The filter is whatever sequence of printable ascii character without
        // whitespaces
        //matches = reAsciiSegment.exec(line);
        //if ( matches === null ) {
        //    console.debug('storage.js > µBlock.mergeFilterList(): skipping "%s"', lineRaw);
        //    continue;
        //}

        // Bypass anomalies
        // For example, when a filter contains whitespace characters, or
        // whatever else outside the range of printable ascii characters.
        //if ( matches[0] !== line ) {
        //    console.error('"%s" !== "%s"', matches[0], line);
        //    continue;
        //}

        staticNetFilteringEngine.add(line);
    }
};

/******************************************************************************/

// `switches` contains the preset blacklists for which the switch must be
// revisited.

µBlock.reloadPresetBlacklists = function(switches, update) {
    var µb = µBlock;

    var onFilterListsReady = function() {
        µb.loadUpdatableAssets({ update: update, psl: update });
    };

    // Toggle switches, if any
    if ( switches !== undefined ) {
        var filterLists = this.remoteBlacklists;
        var i = switches.length;
        while ( i-- ) {
            if ( filterLists.hasOwnProperty(switches[i].location) === false ) {
                continue;
            }
            filterLists[switches[i].location].off = !!switches[i].off;
        }
        // Save switch states
        vAPI.storage.set({ 'remoteBlacklists': filterLists }, onFilterListsReady);
    } else {
        onFilterListsReady();
    }
};

/******************************************************************************/

µBlock.loadPublicSuffixList = function(callback) {
    if ( typeof callback !== 'function' ) {
        callback = this.noopFunc;
    }
    var applyPublicSuffixList = function(details) {
        // TODO: Not getting proper suffix list is a bit serious, I think
        // the extension should be force-restarted if it occurs..
        if ( !details.error ) {
            publicSuffixList.parse(details.content, punycode.toASCII);
        }
        callback();
    };
    this.assets.get(this.pslPath, applyPublicSuffixList);
};

/******************************************************************************/

// Load updatable assets

µBlock.loadUpdatableAssets = function(details) {
    var µb = this;

    details = details || {};
    var update = details.update !== false;

    this.assets.autoUpdate = update || this.userSettings.autoUpdate;
    this.assets.autoUpdateDelay = this.updateAssetsEvery;

    var onFiltersReady = function() {
        if ( update ) {
            µb.updater.restart();
        }
    };

    var onPSLReady = function() {
        µb.loadFilterLists(onFiltersReady);
    };

    if ( details.psl !== false ) {
        this.loadPublicSuffixList(onPSLReady);
    } else {
        this.loadFilterLists(onFiltersReady);
    }
};

/******************************************************************************/

µBlock.toSelfie = function() {
    var selfie = {
        magic: this.selfieMagic,
        publicSuffixList: publicSuffixList.toSelfie(),
        filterLists: this.remoteBlacklists,
        staticNetFilteringEngine: this.staticNetFilteringEngine.toSelfie(),
        cosmeticFilteringEngine: this.cosmeticFilteringEngine.toSelfie()
    };
    vAPI.storage.set({ selfie: selfie });
    // console.log('µBlock.toSelfie> made a selfie!');
};

// This is to be sure the selfie is generated in a sane manner: the selfie will
// be generated if the user doesn't change his filter lists selection for
// some set time.

µBlock.toSelfieAsync = function(after) {
    if ( typeof after !== 'number' ) {
        after = this.selfieAfter;
    }
    this.asyncJobs.add(
        'toSelfie',
        null,
        this.toSelfie.bind(this),
        after,
        false
    );
};

/******************************************************************************/

µBlock.fromSelfie = function(callback) {
    var µb = this;

    if ( typeof callback !== 'function' ) {
        callback = this.noopFunc;
    }

    var onSelfieReady = function(store) {
        var selfie = store.selfie;
        if ( typeof selfie !== 'object' || selfie.magic !== µb.selfieMagic ) {
            callback(false);
            return;
        }
        if ( publicSuffixList.fromSelfie(selfie.publicSuffixList) !== true ) {
            callback(false);
            return;
        }
        // console.log('µBlock.fromSelfie> selfie looks good');
        µb.remoteBlacklists = selfie.filterLists;
        µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine);
        µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine);
        callback(true);
    };

    vAPI.storage.get('selfie', onSelfieReady);
};

/******************************************************************************/

µBlock.destroySelfie = function() {
    vAPI.storage.remove('selfie');
};

/******************************************************************************/

// Load all

µBlock.load = function() {
    var µb = this;
    var fromSelfie = false;

    // Final initialization steps after all needed assets are in memory.
    // - Initialize internal state with maybe already existing tabs.
    // - Schedule next update operation.
    var onAllReady = function() {
        // https://github.com/gorhill/uBlock/issues/426
        // Important: remove barrier to remote fetching, this was useful only
        // for launch time.
        µb.assets.allowRemoteFetch = true;

        // https://github.com/gorhill/uBlock/issues/184
        // Check for updates not too far in the future.
        µb.updater.restart(µb.firstUpdateAfter);

        vAPI.onLoadAllCompleted();
    };

    // To bring older versions up to date
    var onVersionReady = function(bin) {
        var lastVersion = bin.version || '0.0.0.0';

        // Whitelist some key scopes by default
        if ( lastVersion.localeCompare('0.8.6.0') < 0 ) {
            µb.netWhitelist = µb.whitelistFromString(
                µb.stringFromWhitelist(µb.netWhitelist) + 
                '\n' + 
                µb.netWhitelistDefault
            );
            µb.saveWhitelist();
        }

        vAPI.storage.set({ version: vAPI.app.version });
        onAllReady();
    };

    // Filter lists
    // Whitelist
    var countdown = 2;
    var doCountdown = function() {
        countdown -= 1;
        if ( countdown !== 0 ) {
            return;
        }
        // Last step: do whatever is necessary when version changes
        vAPI.storage.get('version', onVersionReady);
    };

    // Filters are in memory.
    // Filter engines need PSL to be ready.
    var onFiltersReady = function() {
        doCountdown();
    };

    // https://github.com/gorhill/uBlock/issues/226
    // Whitelist in memory.
    // Whitelist parser needs PSL to be ready.
    // gorhill 2014-12-15: not anymore
    var onWhitelistReady = function() {
        doCountdown();
    };

    // Load order because dependencies:
    // User settings -> PSL -> [filter lists]
    var onPSLReady = function() {
        µb.loadFilterLists(onFiltersReady);
    };

    // If no selfie available, take the long way, i.e. load and parse
    // raw data.
    var onSelfieReady = function(success) {
        if ( success === true ) {
            fromSelfie = true;
            onFiltersReady();
            return;
        }
        µb.loadPublicSuffixList(onPSLReady);
    };

    // User settings are in memory
    var onUserSettingsReady = function(userSettings) {
        // https://github.com/gorhill/uBlock/issues/426
        // Important: block remote fetching for when loading assets at launch
        // time.
        µb.assets.allowRemoteFetch = false;
        µb.assets.autoUpdate = userSettings.autoUpdate;
        µb.fromSelfie(onSelfieReady);

        // https://github.com/gorhill/uBlock/issues/540
        // Disabling local mirroring for the time being
        userSettings.experimentalEnabled = false;
        µb.mirrors.toggle(false /* userSettings.experimentalEnabled */);

        µb.contextMenu.toggle(userSettings.contextMenuEnabled);
        µb.dynamicNetFilteringEngine.fromString(userSettings.dynamicFilteringString);

        // Remove obsolete setting
        delete userSettings.logRequests;
        µb.XAL.keyvalRemoveOne('logRequests');

        if ( typeof userSettings.dynamicFilteringSelfie === 'string' ) {
            if ( userSettings.dynamicFilteringString === '' && userSettings.dynamicFilteringSelfie !== '' ) {
                µb.dynamicNetFilteringEngine.fromObsoleteSelfie(userSettings.dynamicFilteringSelfie);
                userSettings.dynamicFilteringString = µb.dynamicNetFilteringEngine.toString();
                µb.XAL.keyvalSetOne('dynamicFilteringString', userSettings.dynamicFilteringString);

                // Auto-enable advanced user if there were dynamic rules
                userSettings.advancedUserEnabled = userSettings.dynamicFilteringString !== '';
                µb.XAL.keyvalSetOne('advancedUserEnabled', userSettings.advancedUserEnabled);
            }
            delete userSettings.dynamicFilteringSelfie;
            µb.XAL.keyvalRemoveOne('dynamicFilteringSelfie');
        }
    };

    this.loadUserSettings(onUserSettingsReady);
    this.loadWhitelist(onWhitelistReady);
    this.loadLocalSettings();
};