From d3fe0ccfe060933704e741e9442e9ff61ec63f93 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Mon, 6 Dec 2021 07:01:39 -0500 Subject: [PATCH] Fix regression with `csp=`, deprecate `queryprune`, etc Fixed serious regression in previous dev build in applying `csp=` filters. Reported internally by uBO team. Promote usage of `removeparam` in code instead of `queryprune`, which is to be deprecated. Removed test against previously tested hostname in FilterHostnameDict since as per various benchmark, the test does not really help. Remove serialization API in Node.js code as the API is now present in SNFE itself. --- platform/nodejs/index.js | 30 +++-------------- src/js/static-filtering-parser.js | 18 +++++----- src/js/static-net-filtering.js | 56 ++++++++++++++++--------------- src/logger-ui.html | 2 +- 4 files changed, 43 insertions(+), 63 deletions(-) diff --git a/platform/nodejs/index.js b/platform/nodejs/index.js index 477602be2..87cc45123 100644 --- a/platform/nodejs/index.js +++ b/platform/nodejs/index.js @@ -191,25 +191,6 @@ useLists.promise = null; /******************************************************************************/ -class MockStorage { - constructor(serialized) { - this.map = new Map(serialized); - } - - async put(assetKey, content) { - this.map.set(assetKey, content); - return ({ assetKey, content }); - } - - async get(assetKey) { - return ({ assetKey, content: this.map.get(assetKey) }); - } - - *[Symbol.iterator]() { - yield* this.map; - } -} - const fctx = new FilteringContext(); let snfeProxyInstance = null; @@ -253,15 +234,12 @@ class StaticNetFilteringEngine { return compileList(...args); } - async serialize() { - const storage = new MockStorage(); - await snfe.toSelfie(storage, 'path'); - return JSON.stringify([...storage]); + serialize() { + return snfe.serialize(); } - async deserialize(serialized) { - const storage = new MockStorage(JSON.parse(serialized)); - await snfe.fromSelfie(storage, 'path'); + deserialize(serialized) { + return snfe.unserialize(serialized); } static async create({ noPSL = false } = {}) { diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index b022fae2f..f4a48f745 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -1308,7 +1308,7 @@ Parser.prototype.SelectorCompiler = class { [ 'matches-css-after', ':matches-css-after' ], [ 'matches-css-before', ':matches-css-before' ], ]); - this.reSimpleSelector = /^[#.][A-Za-z_][\w-]*$/; + this.reSimpleSelector = /^[#.]?[A-Za-z_][\w-]*$/; // https://developer.mozilla.org/en-US/docs/Web/API/CSSStyleSheet#browser_compatibility // Firefox does not support constructor for CSSStyleSheet this.stylesheet = (( ) => { @@ -2100,7 +2100,7 @@ const OPTTokenPopunder = 31; const OPTTokenPopup = 32; const OPTTokenRedirect = 33; const OPTTokenRedirectRule = 34; -const OPTTokenQueryprune = 35; +const OPTTokenRemoveparam = 35; const OPTTokenScript = 36; const OPTTokenShide = 37; const OPTTokenXhr = 38; @@ -2200,7 +2200,7 @@ Parser.prototype.OPTTokenOther = OPTTokenOther; Parser.prototype.OPTTokenPing = OPTTokenPing; Parser.prototype.OPTTokenPopunder = OPTTokenPopunder; Parser.prototype.OPTTokenPopup = OPTTokenPopup; -Parser.prototype.OPTTokenQueryprune = OPTTokenQueryprune; +Parser.prototype.OPTTokenRemoveparam = OPTTokenRemoveparam; Parser.prototype.OPTTokenRedirect = OPTTokenRedirect; Parser.prototype.OPTTokenRedirectRule = OPTTokenRedirectRule; Parser.prototype.OPTTokenScript = OPTTokenScript; @@ -2265,11 +2265,11 @@ const netOptionTokenDescriptors = new Map([ /* synonym */ [ 'beacon', OPTTokenPing | OPTCanNegate | OPTNetworkType | OPTModifiableType | OPTNonCspableType | OPTNonRedirectableType ], [ 'popunder', OPTTokenPopunder | OPTNonNetworkType | OPTNonCspableType | OPTNonRedirectableType ], [ 'popup', OPTTokenPopup | OPTNonNetworkType | OPTCanNegate | OPTNonCspableType | OPTNonRedirectableType ], - [ 'queryprune', OPTTokenQueryprune | OPTMayAssign | OPTModifierType | OPTNonCspableType | OPTNonRedirectableType ], - /* synonym */ [ 'removeparam', OPTTokenQueryprune | OPTMayAssign | OPTModifierType | OPTNonCspableType | OPTNonRedirectableType ], [ 'redirect', OPTTokenRedirect | OPTMustAssign | OPTAllowMayAssign | OPTModifierType ], /* synonym */ [ 'rewrite', OPTTokenRedirect | OPTMustAssign | OPTAllowMayAssign | OPTModifierType ], [ 'redirect-rule', OPTTokenRedirectRule | OPTMustAssign | OPTAllowMayAssign | OPTModifierType | OPTNonCspableType ], + [ 'removeparam', OPTTokenRemoveparam | OPTMayAssign | OPTModifierType | OPTNonCspableType | OPTNonRedirectableType ], + /* synonym */ [ 'queryprune', OPTTokenRemoveparam | OPTMayAssign | OPTModifierType | OPTNonCspableType | OPTNonRedirectableType ], [ 'script', OPTTokenScript | OPTCanNegate | OPTNetworkType | OPTModifiableType | OPTRedirectableType | OPTNonCspableType ], [ 'shide', OPTTokenShide | OPTNonNetworkType | OPTNonCspableType | OPTNonRedirectableType ], /* synonym */ [ 'specifichide', OPTTokenShide | OPTNonNetworkType | OPTNonCspableType | OPTNonRedirectableType ], @@ -2324,11 +2324,11 @@ Parser.netOptionTokenIds = new Map([ /* synonym */ [ 'beacon', OPTTokenPing ], [ 'popunder', OPTTokenPopunder ], [ 'popup', OPTTokenPopup ], - [ 'queryprune', OPTTokenQueryprune ], - /* synonym */ [ 'removeparam', OPTTokenQueryprune ], [ 'redirect', OPTTokenRedirect ], /* synonym */ [ 'rewrite', OPTTokenRedirect ], [ 'redirect-rule', OPTTokenRedirectRule ], + [ 'removeparam', OPTTokenRemoveparam ], + /* synonym */ [ 'queryprune', OPTTokenRemoveparam ], [ 'script', OPTTokenScript ], [ 'shide', OPTTokenShide ], /* synonym */ [ 'specifichide', OPTTokenShide ], @@ -2371,7 +2371,7 @@ Parser.netOptionTokenNames = new Map([ [ OPTTokenPing, 'ping' ], [ OPTTokenPopunder, 'popunder' ], [ OPTTokenPopup, 'popup' ], - [ OPTTokenQueryprune, 'removeparam' ], + [ OPTTokenRemoveparam, 'removeparam' ], [ OPTTokenRedirect, 'redirect' ], [ OPTTokenRedirectRule, 'redirect-rule' ], [ OPTTokenScript, 'script' ], @@ -2592,7 +2592,7 @@ const NetOptionsIterator = class { } // `removeparam=`: only for network requests. { - const i = this.tokenPos[OPTTokenQueryprune]; + const i = this.tokenPos[OPTTokenRemoveparam]; if ( i !== -1 ) { if ( hasBits(allBits, OPTNonNetworkType) ) { optSlices[i] = OPTTokenInvalid; diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 0eaa7eaf1..8555327e2 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -1794,6 +1794,10 @@ const FilterModifierResult = class { this.bits = (env.bits & ~RealmBitsMask) | filterData[imodifierunit+1]; } + get result() { + return (this.bits & AllowAction) === 0 ? 1 : 2; + } + get value() { return this.refs.value; } @@ -1808,7 +1812,7 @@ const FilterModifierResult = class { logData() { const r = new LogData(this.bits, this.th, this.ireportedunit); - r.result = (this.bits & AllowAction) === 0 ? 1 : 2; + r.result = this.result; r.modifier = true; return r; } @@ -2005,26 +2009,28 @@ registerFilterClass(FilterCompositeAll); const FilterHostnameDict = class { static getCount(idata) { + const itrie = filterData[idata+1]; + if ( itrie === 0 ) { + return filterRefs[filterData[idata+3]].length; + } return Array.from( destHNTrieContainer.trieIterator(filterData[idata+1]) ).length; } static match(idata) { - const refs = filterRefs[filterData[idata+3]]; - if ( $requestHostname !== refs.$last ) { - const itrie = filterData[idata+1] || this.optimize(idata); + const itrie = filterData[idata+1] || this.optimize(idata); + return ( filterData[idata+2] = destHNTrieContainer - .setNeedle(refs.$last = $requestHostname) - .matches(itrie); - } - return filterData[idata+2] !== -1; + .setNeedle($requestHostname) + .matches(itrie) + ) !== -1; } static add(idata, hn) { const itrie = filterData[idata+1]; if ( itrie === 0 ) { - filterRefs[filterData[idata+3]].hostnames.push(hn); + filterRefs[filterData[idata+3]].push(hn); } else { destHNTrieContainer.setNeedle(hn).add(itrie); } @@ -2033,9 +2039,9 @@ const FilterHostnameDict = class { static optimize(idata) { const itrie = filterData[idata+1]; if ( itrie !== 0 ) { return itrie; } - const refs = filterRefs[filterData[idata+3]]; - filterData[idata+1] = destHNTrieContainer.createTrie(refs.hostnames); - refs.hostnames = []; + const hostnames = filterRefs[filterData[idata+3]]; + filterData[idata+1] = destHNTrieContainer.createTrie(hostnames); + filterRefs[filterData[idata+3]] = null; return filterData[idata+1]; } @@ -2044,16 +2050,12 @@ const FilterHostnameDict = class { filterData[idata+0] = FilterHostnameDict.fid; // fid filterData[idata+1] = 0; // itrie filterData[idata+2] = -1; // lastResult - filterData[idata+3] = filterRefAdd({ - hostnames: [], - $last: '', - }); + filterData[idata+3] = filterRefAdd([]); // []: hostnames return idata; } static logData(idata, details) { - const refs = filterRefs[filterData[idata+3]]; - const hostname = refs.$last.slice(filterData[idata+2]); + const hostname = $requestHostname.slice(filterData[idata+2]); details.pattern.push('||', hostname, '^'); details.regex.push( restrFromPlainPattern(hostname), @@ -3031,11 +3033,11 @@ class FilterCompiler { break; case this.parser.OPTTokenNoop: break; - case this.parser.OPTTokenQueryprune: + case this.parser.OPTTokenRemoveparam: if ( this.processModifierOption(id, val) === false ) { return false; } - this.optionUnitBits |= this.QUERYPRUNE_BIT; + this.optionUnitBits |= this.REMOVEPARAM_BIT; break; case this.parser.OPTTokenRedirect: if ( this.action === AllowAction ) { @@ -3192,12 +3194,12 @@ class FilterCompiler { // are not good. Avoid if possible. This has a significant positive // impact on performance. // - // For pattern-less queryprune filters, try to derive a pattern from - // the queryprune value. + // For pattern-less removeparam filters, try to derive a pattern from + // the removeparam value. makeToken() { if ( this.pattern === '*' ) { - if ( this.modifyType !== this.parser.OPTTokenQueryprune ) { + if ( this.modifyType !== this.parser.OPTTokenRemoveparam ) { return; } return this.extractTokenFromQuerypruneValue(); @@ -3529,7 +3531,7 @@ FilterCompiler.prototype.DENYALLOW_BIT = 0b000000010; FilterCompiler.prototype.HEADER_BIT = 0b000000100; FilterCompiler.prototype.STRICT_PARTY_BIT = 0b000001000; FilterCompiler.prototype.CSP_BIT = 0b000010000; -FilterCompiler.prototype.QUERYPRUNE_BIT = 0b000100000; +FilterCompiler.prototype.REMOVEPARAM_BIT = 0b000100000; FilterCompiler.prototype.REDIRECT_BIT = 0b001000000; FilterCompiler.prototype.NOT_TYPE_BIT = 0b010000000; FilterCompiler.prototype.IMPORTANT_BIT = 0b100000000; @@ -3543,7 +3545,7 @@ FilterCompiler.prototype.FILTER_UNSUPPORTED = 2; const FilterContainer = function() { this.compilerVersion = '2'; - this.selfieVersion = '2'; + this.selfieVersion = '3'; this.MAX_TOKEN_LENGTH = MAX_TOKEN_LENGTH; this.optimizeTaskId = undefined; @@ -3898,8 +3900,8 @@ FilterContainer.prototype.fromSelfie = async function(storage, path) { FilterContainer.prototype.unserialize = async function(s) { const selfie = new Map(JSON.parse(s)); const storage = { - get(name) { - return Promise.resolve(selfie.get(name)); + async get(name) { + return { content: selfie.get(name) }; } }; return this.fromSelfie(storage, ''); diff --git a/src/logger-ui.html b/src/logger-ui.html index 7ea8f46ad..8b2f57368 100644 --- a/src/logger-ui.html +++ b/src/logger-ui.html @@ -58,7 +58,7 @@ angle-up
-
+
css/fontimagemediascript