From 5d7b2918efbf927c59e6c0a4812b8ccf1b029adf Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Tue, 8 Dec 2020 10:00:47 -0500 Subject: [PATCH] Harden processing of changes in compiled list format Related issue: - https://github.com/uBlockOrigin/uBlock-issues/issues/1365 This commit adds the compiled magic version number to the compiled data itself, and consequently this allows uBO to no longer require that any given compiled list with a mismatched format to be detected and discarded at launch time. Given this change, uBO no longer needs to rely on the deletion of cached data at launch time to ensure it won't use no longer valid compiled lists. --- src/js/background.js | 7 +++++++ src/js/cosmetic-filtering.js | 12 ++++-------- src/js/html-filtering.js | 6 ++---- src/js/reverselookup.js | 4 ++-- src/js/scriptlet-filtering.js | 6 ++---- src/js/start.js | 8 +++----- src/js/static-net-filtering.js | 14 +++++++------- src/js/storage.js | 18 +++++++++++++++--- src/js/utils.js | 3 ++- 9 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/js/background.js b/src/js/background.js index af21783a2..d73551cd9 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -154,6 +154,13 @@ const µBlock = (( ) => { // jshint ignore:line compiledFormatChanged: false, selfieIsInvalid: false, + compiledNetworkSection: 100, + compiledCosmeticSection: 200, + compiledScriptletSection: 300, + compiledHTMLSection: 400, + compiledSentinelSection: 1000, + compiledBadSubsection: 1, + restoreBackupSettings: { lastRestoreFile: '', lastRestoreTime: 0, diff --git a/src/js/cosmetic-filtering.js b/src/js/cosmetic-filtering.js index 91155a627..8c1943f7a 100644 --- a/src/js/cosmetic-filtering.js +++ b/src/js/cosmetic-filtering.js @@ -340,8 +340,7 @@ FilterContainer.prototype.keyFromSelector = function(selector) { /******************************************************************************/ FilterContainer.prototype.compile = function(parser, writer) { - // 1000 = cosmetic filtering - writer.select(1000); + writer.select(µb.compiledCosmeticSection); if ( parser.hasOptions() === false ) { this.compileGenericSelector(parser, writer); @@ -551,8 +550,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader, options) { return; } - // 1000 = cosmetic filtering - reader.select(1000); + reader.select(µb.compiledCosmeticSection); let db, bucket; @@ -643,8 +641,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader, options) { /******************************************************************************/ FilterContainer.prototype.skipGenericCompiledContent = function(reader) { - // 1000 = cosmetic filtering - reader.select(1000); + reader.select(µb.compiledCosmeticSection); while ( reader.next() ) { this.acceptedCount += 1; @@ -685,8 +682,7 @@ FilterContainer.prototype.skipGenericCompiledContent = function(reader) { /******************************************************************************/ FilterContainer.prototype.skipCompiledContent = function(reader) { - // 1000 = cosmetic filtering - reader.select(1000); + reader.select(µb.compiledCosmeticSection); while ( reader.next() ) { this.acceptedCount += 1; diff --git a/src/js/html-filtering.js b/src/js/html-filtering.js index 91cf126af..4ca03ce50 100644 --- a/src/js/html-filtering.js +++ b/src/js/html-filtering.js @@ -304,8 +304,7 @@ return; } - // 1002 = html filtering - writer.select(1002); + writer.select(µb.compiledHTMLSection); // TODO: Mind negated hostnames, they are currently discarded. @@ -327,8 +326,7 @@ // Don't bother loading filters if stream filtering is not supported. if ( µb.canFilterResponseData === false ) { return; } - // 1002 = html filtering - reader.select(1002); + reader.select(µb.compiledHTMLSection); while ( reader.next() ) { acceptedCount += 1; diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 5557f599f..7a228231a 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -64,7 +64,7 @@ if ( for ( const assetKey in listEntries ) { const entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } - const content = extractBlocks(entry.content, 0, 1); + const content = extractBlocks(entry.content, 100, 101); let pos = 0; for (;;) { pos = content.indexOf(compiledFilter, pos); @@ -165,7 +165,7 @@ if ( for ( const assetKey in listEntries ) { const entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } - let content = extractBlocks(entry.content, 1000, 2000), + let content = extractBlocks(entry.content, 200, 1000), isProcedural, found; let pos = 0; diff --git a/src/js/scriptlet-filtering.js b/src/js/scriptlet-filtering.js index c05341935..f348b0ee3 100644 --- a/src/js/scriptlet-filtering.js +++ b/src/js/scriptlet-filtering.js @@ -230,8 +230,7 @@ }; api.compile = function(parser, writer) { - // 1001 = scriptlet injection - writer.select(1001); + writer.select(µb.compiledScriptletSection); // Only exception filters are allowed to be global. const { raw, exception } = parser.result; @@ -270,8 +269,7 @@ // 4 -1 api.fromCompiledContent = function(reader) { - // 1001 = scriptlet injection - reader.select(1001); + reader.select(µb.compiledScriptletSection); while ( reader.next() ) { acceptedCount += 1; diff --git a/src/js/start.js b/src/js/start.js index a1298677a..efe9bbd6f 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -190,7 +190,6 @@ const onUserSettingsReady = function(fetched) { const onCacheSettingsReady = async function(fetched) { if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { - await µb.assets.remove(/^compiled\//); µb.compiledFormatChanged = true; µb.selfieIsInvalid = true; } @@ -302,10 +301,9 @@ try { }), µb.cacheStorage.get( { compiledMagic: 0, selfieMagic: 0 } - ).then(fetched => - onCacheSettingsReady(fetched) - ).then(( ) => { - log.info(`Integrity of cached data processed ${Date.now()-vAPI.T0} ms after launch`); + ).then(fetched => { + log.info(`Cache magic numbers ready ${Date.now()-vAPI.T0} ms after launch`); + onCacheSettingsReady(fetched); }), vAPI.storage.get(createDefaultProps()).then(fetched => { log.info(`First fetch ready ${Date.now()-vAPI.T0} ms after launch`); diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 2f0a494cd..f52c4c45b 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -3639,9 +3639,11 @@ FilterContainer.prototype.compile = function(parser, writer) { return false; } - // 0 = network filters - // 1 = network filters: bad filters - writer.select(parsed.badFilter ? 1 : 0); + writer.select( + parsed.badFilter + ? µb.compiledNetworkSection + µb.compiledBadSubsection + : µb.compiledNetworkSection + ); // Reminder: // `redirect=` is a combination of a `redirect-rule` filter and a @@ -3808,8 +3810,7 @@ FilterContainer.prototype.compileToAtomicFilter = function( /******************************************************************************/ FilterContainer.prototype.fromCompiledContent = function(reader) { - // 0 = network filters - reader.select(0); + reader.select(µb.compiledNetworkSection); while ( reader.next() ) { this.acceptedCount += 1; if ( this.goodFilters.has(reader.line) ) { @@ -3819,8 +3820,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader) { } } - // 1 = network filters: bad filter directives - reader.select(1); + reader.select(µb.compiledNetworkSection + µb.compiledBadSubsection); while ( reader.next() ) { this.badFilters.add(reader.line); } diff --git a/src/js/storage.js b/src/js/storage.js index 3ff2359bb..e86d02cd9 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -733,12 +733,18 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µBlock.getCompiledFilterList = async function(assetKey) { const compiledPath = 'compiled/' + assetKey; + // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 + // Verify that the list version matches that of the current compiled + // format. if ( this.compiledFormatChanged === false && this.badLists.has(assetKey) === false ) { const compiledDetails = await this.assets.get(compiledPath); - if ( compiledDetails.content !== '' ) { + if ( + parseInt(compiledDetails.content, 10) === + this.systemSettings.compiledMagic + ) { compiledDetails.assetKey = assetKey; return compiledDetails; } @@ -878,7 +884,13 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { staticNetFilteringEngine.compile(parser, writer); } - return writer.toString(); + // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 + // Embed version into compiled list itself: it is encoded in as the + // first digits followed by a whitespace. + const compiledContent + = `${this.systemSettings.compiledMagic}\n` + writer.toString(); + + return compiledContent; }; /******************************************************************************/ @@ -889,7 +901,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µBlock.applyCompiledFilters = function(rawText, firstparty) { if ( rawText === '' ) { return; } - let reader = new this.CompiledLineIO.Reader(rawText); + const reader = new this.CompiledLineIO.Reader(rawText); this.staticNetFilteringEngine.fromCompiledContent(reader); this.staticExtFilteringEngine.fromCompiledContent(reader, { skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters, diff --git a/src/js/utils.js b/src/js/utils.js index 983b75120..57f7370a1 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -153,6 +153,7 @@ if ( this.block === undefined ) { this.blocks.set(blockId, (this.block = [])); } + return this; } toString() { let result = []; @@ -179,7 +180,7 @@ this.blocks = new Map(); this.properties = new Map(); let reBlockStart = new RegExp( - '^' + this.io.blockStartPrefix + '(\\d+)\\n', + `^${this.io.blockStartPrefix}(\\d+)\\n`, 'gm' ); let match = reBlockStart.exec(raw);