From 51c2e22c7a882d67d1ec431022b915a52b6b8d3a Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Tue, 27 Sep 2022 07:46:24 -0400 Subject: [PATCH] [mv3] Fix procedural operator matches-media() The failure was caused by the fact that there is no window.matchMedia() API available in Nodejs. The validation is now done using cssTree. --- platform/mv3/extension/js/background.js | 22 ++-- platform/mv3/extension/js/ruleset-manager.js | 32 +++-- .../mv3/extension/js/scripting-manager.js | 40 +++--- platform/mv3/extension/js/trusted-sites.js | 12 +- platform/mv3/make-rulesets.js | 62 ++++----- .../mv3/scriptlets/css-specific-procedural.js | 27 +++- src/js/static-filtering-parser.js | 46 +++---- src/js/static-net-filtering.js | 121 ++++++++++-------- 8 files changed, 201 insertions(+), 161 deletions(-) diff --git a/platform/mv3/extension/js/background.js b/platform/mv3/extension/js/background.js index 857b5080e..5bfbd17c3 100644 --- a/platform/mv3/extension/js/background.js +++ b/platform/mv3/extension/js/background.js @@ -72,7 +72,7 @@ async function loadRulesetConfig() { return; } - const match = /^\|\|example.invalid\/([^\/]+)\/(?:([^\/]+)\/)?/.exec( + const match = /^\|\|(?:example|ubolite)\.invalid\/([^\/]+)\/(?:([^\/]+)\/)?/.exec( configRule.condition.urlFilter ); if ( match === null ) { return; } @@ -101,7 +101,7 @@ async function saveRulesetConfig() { const version = rulesetConfig.version; const enabledRulesets = encodeURIComponent(rulesetConfig.enabledRulesets.join(' ')); - const urlFilter = `||example.invalid/${version}/${enabledRulesets}/`; + const urlFilter = `||ubolite.invalid/${version}/${enabledRulesets}/`; if ( urlFilter === configRule.condition.urlFilter ) { return; } configRule.condition.urlFilter = urlFilter; @@ -115,26 +115,26 @@ async function saveRulesetConfig() { async function hasGreatPowers(origin) { return browser.permissions.contains({ - origins: [ `${origin}/*` ] + origins: [ `${origin}/*` ], }); } function grantGreatPowers(hostname) { return browser.permissions.request({ - origins: [ - `*://${hostname}/*`, - ] + origins: [ `*://${hostname}/*` ], }); } function revokeGreatPowers(hostname) { return browser.permissions.remove({ - origins: [ - `*://${hostname}/*`, - ] + origins: [ `*://${hostname}/*` ], }); } +function onPermissionsChanged() { + registerInjectable(); +} + /******************************************************************************/ function onMessage(request, sender, callback) { @@ -213,10 +213,6 @@ function onMessage(request, sender, callback) { } } -async function onPermissionsChanged() { - await registerInjectable(); -} - /******************************************************************************/ async function start() { diff --git a/platform/mv3/extension/js/ruleset-manager.js b/platform/mv3/extension/js/ruleset-manager.js index faf9bd521..933468226 100644 --- a/platform/mv3/extension/js/ruleset-manager.js +++ b/platform/mv3/extension/js/ruleset-manager.js @@ -205,26 +205,42 @@ async function defaultRulesetsFromLanguage() { async function enableRulesets(ids) { const afterIds = new Set(ids); const beforeIds = new Set(await dnr.getEnabledRulesets()); - const enableRulesetIds = []; - const disableRulesetIds = []; + const enableRulesetSet = new Set(); + const disableRulesetSet = new Set(); for ( const id of afterIds ) { if ( beforeIds.has(id) ) { continue; } - enableRulesetIds.push(id); + enableRulesetSet.add(id); } for ( const id of beforeIds ) { if ( afterIds.has(id) ) { continue; } - disableRulesetIds.push(id); + disableRulesetSet.add(id); } - + + if ( enableRulesetSet.size === 0 && disableRulesetSet.size === 0 ) { + return; + } + + // Be sure the rulesets to enable/disable do exist in the current version, + // otherwise the API throws. + const rulesetDetails = await getRulesetDetails(); + for ( const id of enableRulesetSet ) { + if ( rulesetDetails.has(id) ) { continue; } + enableRulesetSet.delete(id); + } + for ( const id of disableRulesetSet ) { + if ( rulesetDetails.has(id) ) { continue; } + disableRulesetSet.delete(id); + } + const enableRulesetIds = Array.from(enableRulesetSet); + const disableRulesetIds = Array.from(disableRulesetSet); + if ( enableRulesetIds.length !== 0 ) { console.info(`Enable rulesets: ${enableRulesetIds}`); } if ( disableRulesetIds.length !== 0 ) { console.info(`Disable ruleset: ${disableRulesetIds}`); } - if ( enableRulesetIds.length !== 0 || disableRulesetIds.length !== 0 ) { - return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds }); - } + return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds }); } /******************************************************************************/ diff --git a/platform/mv3/extension/js/scripting-manager.js b/platform/mv3/extension/js/scripting-manager.js index d37ee8231..d003031fe 100644 --- a/platform/mv3/extension/js/scripting-manager.js +++ b/platform/mv3/extension/js/scripting-manager.js @@ -27,7 +27,7 @@ import { browser, dnr } from './ext.js'; import { fetchJSON } from './fetch.js'; -import { matchesTrustedSiteDirective } from './trusted-sites.js'; +import { getAllTrustedSiteDirectives } from './trusted-sites.js'; import { parsedURLromOrigin, @@ -94,7 +94,6 @@ const arrayEq = (a, b) => { const toRegisterable = (fname, entry) => { const directive = { id: fname, - allFrames: true, }; if ( entry.matches ) { directive.matches = matchesFromHostnames(entry.matches); @@ -173,37 +172,36 @@ async function getInjectableCount(origin) { async function registerInjectable() { + if ( browser.scripting === undefined ) { return false; } + const [ hostnames, + trustedSites, rulesetIds, registered, scriptingDetails, ] = await Promise.all([ browser.permissions.getAll(), + getAllTrustedSiteDirectives(), dnr.getEnabledRulesets(), browser.scripting.getRegisteredContentScripts(), getScriptingDetails(), ]).then(results => { - results[0] = new Map( - hostnamesFromMatches(results[0].origins).map(hn => [ hn, false ]) - ); + results[0] = new Set(hostnamesFromMatches(results[0].origins)); + results[1] = new Set(results[1]); return results; }); - if ( hostnames.has('*') && hostnames.size > 1 ) { - hostnames.clear(); - hostnames.set('*', false); - } - - await Promise.all( - Array.from(hostnames.keys()).map( - hn => matchesTrustedSiteDirective({ hostname: hn }) - .then(trusted => hostnames.set(hn, trusted)) - ) - ); - const toRegister = new Map(); + const isTrustedHostname = hn => { + while ( hn ) { + if ( trustedSites.has(hn) ) { return true; } + hn = toBroaderHostname(hn); + } + return false; + }; + const checkMatches = (details, hn) => { let fids = details.matches?.get(hn); if ( fids === undefined ) { return; } @@ -222,9 +220,9 @@ async function registerInjectable() { for ( const rulesetId of rulesetIds ) { const details = scriptingDetails.get(rulesetId); if ( details === undefined ) { continue; } - for ( let [ hn, trusted ] of hostnames ) { - if ( trusted ) { continue; } - while ( hn !== '' ) { + for ( let hn of hostnames ) { + if ( isTrustedHostname(hn) ) { continue; } + while ( hn ) { checkMatches(details, hn); hn = toBroaderHostname(hn); } @@ -251,7 +249,7 @@ async function registerInjectable() { const details = scriptingDetails.get(rulesetId); if ( details === undefined ) { continue; } for ( let hn of hostnames.keys() ) { - while ( hn !== '' ) { + while ( hn ) { checkExcludeMatches(details, hn); hn = toBroaderHostname(hn); } diff --git a/platform/mv3/extension/js/trusted-sites.js b/platform/mv3/extension/js/trusted-sites.js index bb56c0b81..bfed7f780 100644 --- a/platform/mv3/extension/js/trusted-sites.js +++ b/platform/mv3/extension/js/trusted-sites.js @@ -39,6 +39,15 @@ import { /******************************************************************************/ +async function getAllTrustedSiteDirectives() { + const dynamicRuleMap = await getDynamicRules(); + const rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID); + if ( rule === undefined ) { return []; } + return rule.condition.requestDomains; +} + +/******************************************************************************/ + async function matchesTrustedSiteDirective(details) { const hostname = details.hostname || @@ -47,7 +56,7 @@ async function matchesTrustedSiteDirective(details) { if ( hostname === undefined ) { return false; } const dynamicRuleMap = await getDynamicRules(); - let rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID); + const rule = dynamicRuleMap.get(TRUSTED_DIRECTIVE_BASE_RULE_ID); if ( rule === undefined ) { return false; } const domainSet = new Set(rule.condition.requestDomains); @@ -155,6 +164,7 @@ async function toggleTrustedSiteDirective(details) { /******************************************************************************/ export { + getAllTrustedSiteDirectives, matchesTrustedSiteDirective, toggleTrustedSiteDirective, }; diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 188ac6cf0..e1454de0e 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -74,35 +74,6 @@ const uidint32 = (s) => { /******************************************************************************/ -const isUnsupported = rule => - rule._error !== undefined; - -const isRegex = rule => - rule.condition !== undefined && - rule.condition.regexFilter !== undefined; - -const isRedirect = rule => - rule.action !== undefined && - rule.action.type === 'redirect' && - rule.action.redirect.extensionPath !== undefined; - -const isCsp = rule => - rule.action !== undefined && - rule.action.type === 'modifyHeaders'; - -const isRemoveparam = rule => - rule.action !== undefined && - rule.action.type === 'redirect' && - rule.action.redirect.transform !== undefined; - -const isGood = rule => - isUnsupported(rule) === false && - isRedirect(rule) === false && - isCsp(rule) === false && - isRemoveparam(rule) === false; - -/******************************************************************************/ - const stdOutput = []; const log = (text, silent = false) => { @@ -217,6 +188,35 @@ async function fetchAsset(assetDetails) { /******************************************************************************/ +const isUnsupported = rule => + rule._error !== undefined; + +const isRegex = rule => + rule.condition !== undefined && + rule.condition.regexFilter !== undefined; + +const isRedirect = rule => + rule.action !== undefined && + rule.action.type === 'redirect' && + rule.action.redirect.extensionPath !== undefined; + +const isCsp = rule => + rule.action !== undefined && + rule.action.type === 'modifyHeaders'; + +const isRemoveparam = rule => + rule.action !== undefined && + rule.action.type === 'redirect' && + rule.action.redirect.transform !== undefined; + +const isGood = rule => + isUnsupported(rule) === false && + isRedirect(rule) === false && + isCsp(rule) === false && + isRemoveparam(rule) === false; + +/******************************************************************************/ + async function processNetworkFilters(assetDetails, network) { const replacer = (k, v) => { if ( k.startsWith('__') ) { return; } @@ -993,7 +993,9 @@ async function main() { ); // Log results - await fs.writeFile(`${outputDir}/log.txt`, stdOutput.join('\n') + '\n'); + const logContent = stdOutput.join('\n') + '\n'; + await fs.writeFile(`${outputDir}/log.txt`, logContent); + await fs.writeFile(`${cacheDir}/log.txt`, logContent); } main(); diff --git a/platform/mv3/scriptlets/css-specific-procedural.js b/platform/mv3/scriptlets/css-specific-procedural.js index acf1c3482..7563e4c21 100644 --- a/platform/mv3/scriptlets/css-specific-procedural.js +++ b/platform/mv3/scriptlets/css-specific-procedural.js @@ -69,6 +69,15 @@ class PSelectorTask { } } +class PSelectorVoidTask extends PSelectorTask { + constructor(task) { + super(); + console.info(`uBO: :${task[0]}() operator does not exist`); + } + transpose() { + } +} + class PSelectorHasTextTask extends PSelectorTask { constructor(task) { super(); @@ -120,6 +129,19 @@ class PSelectorMatchesCSSTask extends PSelectorTask { } } } +class PSelectorMatchesCSSAfterTask extends PSelectorMatchesCSSTask { + constructor(task) { + super(task); + this.pseudo = '::after'; + } +} + +class PSelectorMatchesCSSBeforeTask extends PSelectorMatchesCSSTask { + constructor(task) { + super(task); + this.pseudo = '::before'; + } +} class PSelectorMatchesMediaTask extends PSelectorTask { constructor(task) { @@ -370,6 +392,8 @@ class PSelector { [ 'if', PSelectorIfTask ], [ 'if-not', PSelectorIfNotTask ], [ 'matches-css', PSelectorMatchesCSSTask ], + [ 'matches-css-after', PSelectorMatchesCSSAfterTask ], + [ 'matches-css-before', PSelectorMatchesCSSBeforeTask ], [ 'matches-media', PSelectorMatchesMediaTask ], [ 'matches-path', PSelectorMatchesPathTask ], [ 'min-text-length', PSelectorMinTextLengthTask ], @@ -387,8 +411,7 @@ class PSelector { const tasks = []; if ( Array.isArray(o.tasks) === false ) { return; } for ( const task of o.tasks ) { - const ctor = this.operatorToTaskMap.get(task[0]); - if ( ctor === undefined ) { return; } + const ctor = this.operatorToTaskMap.get(task[0]) || PSelectorVoidTask; tasks.push(new ctor(task)); } // Initialize only after all tasks have been successfully instantiated diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index 01809e6fc..ffa3a8492 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -1499,46 +1499,33 @@ Parser.prototype.SelectorCompiler = class { case 'ClassSelector': case 'Combinator': case 'IdSelector': + case 'MediaFeature': + case 'Nth': + case 'Raw': case 'TypeSelector': out.push({ data }); break; - case 'Declaration': { + case 'Declaration': if ( data.value ) { this.astFlatten(data.value, args = []); } out.push({ data, args }); args = undefined; break; - } case 'DeclarationList': - args = out; - out.push({ data }); - break; case 'Identifier': + case 'MediaQueryList': + case 'Selector': + case 'SelectorList': args = out; out.push({ data }); break; - case 'Nth': { - out.push({ data }); - break; - } + case 'MediaQuery': case 'PseudoClassSelector': case 'PseudoElementSelector': if ( head ) { args = []; } out.push({ data, args }); break; - case 'Raw': - if ( head ) { args = []; } - out.push({ data, args }); - break; - case 'Selector': - args = out; - out.push({ data }); - break; - case 'SelectorList': - args = out; - out.push({ data }); - break; case 'Value': args = out; break; @@ -1552,7 +1539,7 @@ Parser.prototype.SelectorCompiler = class { } let next = head.next; while ( next ) { - this.astFlatten(next.data, out); + this.astFlatten(next.data, args); next = next.next; } } @@ -1923,15 +1910,12 @@ Parser.prototype.SelectorCompiler = class { } compileMediaQuery(s) { - if ( typeof self !== 'object' ) { return; } - if ( self === null ) { return; } - if ( typeof self.matchMedia !== 'function' ) { return; } - try { - const mql = self.matchMedia(s); - if ( mql instanceof self.MediaQueryList === false ) { return; } - if ( mql.media !== 'not all' ) { return s; } - } catch(ex) { - } + const parts = this.astFromRaw(s, 'mediaQueryList'); + if ( parts === undefined ) { return; } + if ( this.astHasType(parts, 'Raw') ) { return; } + if ( this.astHasType(parts, 'MediaQuery') === false ) { return; } + // TODO: normalize by serializing resulting AST + return s; } compileUpwardArgument(s) { diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 02757d7a8..3f16c89d2 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -3903,65 +3903,65 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { const bucket = buckets.get(bits); switch ( tokenHash ) { - case DOT_TOKEN_HASH: { - if ( bucket.has(DOT_TOKEN_HASH) === false ) { - bucket.set(DOT_TOKEN_HASH, [{ - condition: { - requestDomains: [] - } - }]); - } - const rule = bucket.get(DOT_TOKEN_HASH)[0]; - rule.condition.requestDomains.push(fdata); - break; + case DOT_TOKEN_HASH: { + if ( bucket.has(DOT_TOKEN_HASH) === false ) { + bucket.set(DOT_TOKEN_HASH, [{ + condition: { + requestDomains: [] + } + }]); } - case ANY_TOKEN_HASH: { - if ( bucket.has(ANY_TOKEN_HASH) === false ) { - bucket.set(ANY_TOKEN_HASH, [{ - condition: { - initiatorDomains: [] - } - }]); - } - const rule = bucket.get(ANY_TOKEN_HASH)[0]; - rule.condition.initiatorDomains.push(fdata); - break; + const rule = bucket.get(DOT_TOKEN_HASH)[0]; + rule.condition.requestDomains.push(fdata); + break; + } + case ANY_TOKEN_HASH: { + if ( bucket.has(ANY_TOKEN_HASH) === false ) { + bucket.set(ANY_TOKEN_HASH, [{ + condition: { + initiatorDomains: [] + } + }]); } - case ANY_HTTPS_TOKEN_HASH: { - if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) { - bucket.set(ANY_HTTPS_TOKEN_HASH, [{ - condition: { - urlFilter: '|https://', - initiatorDomains: [] - } - }]); - } - const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0]; - rule.condition.initiatorDomains.push(fdata); - break; + const rule = bucket.get(ANY_TOKEN_HASH)[0]; + rule.condition.initiatorDomains.push(fdata); + break; + } + case ANY_HTTPS_TOKEN_HASH: { + if ( bucket.has(ANY_HTTPS_TOKEN_HASH) === false ) { + bucket.set(ANY_HTTPS_TOKEN_HASH, [{ + condition: { + urlFilter: '|https://', + initiatorDomains: [] + } + }]); } - case ANY_HTTP_TOKEN_HASH: { - if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) { - bucket.set(ANY_HTTP_TOKEN_HASH, [{ - condition: { - urlFilter: '|http://', - initiatorDomains: [] - } - }]); - } - const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0]; - rule.condition.initiatorDomains.push(fdata); - break; + const rule = bucket.get(ANY_HTTPS_TOKEN_HASH)[0]; + rule.condition.initiatorDomains.push(fdata); + break; + } + case ANY_HTTP_TOKEN_HASH: { + if ( bucket.has(ANY_HTTP_TOKEN_HASH) === false ) { + bucket.set(ANY_HTTP_TOKEN_HASH, [{ + condition: { + urlFilter: '|http://', + initiatorDomains: [] + } + }]); } - default: { - if ( bucket.has(EMPTY_TOKEN_HASH) === false ) { - bucket.set(EMPTY_TOKEN_HASH, []); - } - const rule = {}; - dnrRuleFromCompiled(fdata, rule); - bucket.get(EMPTY_TOKEN_HASH).push(rule); - break; + const rule = bucket.get(ANY_HTTP_TOKEN_HASH)[0]; + rule.condition.initiatorDomains.push(fdata); + break; + } + default: { + if ( bucket.has(EMPTY_TOKEN_HASH) === false ) { + bucket.set(EMPTY_TOKEN_HASH, []); } + const rule = {}; + dnrRuleFromCompiled(fdata, rule); + bucket.get(EMPTY_TOKEN_HASH).push(rule); + break; + } } } @@ -4066,7 +4066,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } } }; - if ( /^\/.+\/$/.test(rule.__modifierValue) ) { + if ( /^~?\/.+\/$/.test(rule.__modifierValue) ) { dnrAddRuleError(rule, `Unsupported regex-based removeParam: ${rule.__modifierValue}`); } } else { @@ -4076,6 +4076,17 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } }; } + if ( rule.condition === undefined ) { + rule.condition = { + }; + } + if ( rule.condition.resourceTypes === undefined ) { + rule.condition.resourceTypes = [ + 'main_frame', + 'sub_frame', + 'xmlhttprequest', + ]; + } if ( rule.__modifierAction === AllowAction ) { dnrAddRuleError(rule, 'Unhandled modifier exception'); }