diff --git a/platform/mv3/extension/js/popup.js b/platform/mv3/extension/js/popup.js index ffa096006..f1aad0f6e 100644 --- a/platform/mv3/extension/js/popup.js +++ b/platform/mv3/extension/js/popup.js @@ -291,9 +291,9 @@ async function init() { const div = qs$('#templates .rulesetDetails').cloneNode(true); dom.text(qs$('h1', div), details.name); const { rules, filters, css } = details; - let ruleCount = rules.plain + rules.regexes; + let ruleCount = rules.plain + rules.regex; if ( popupPanelData.hasOmnipotence ) { - ruleCount += rules.removeparams; + ruleCount += rules.removeparam + rules.redirect; } dom.text( qs$('p', div), diff --git a/platform/mv3/extension/js/ruleset-manager.js b/platform/mv3/extension/js/ruleset-manager.js index 4b5e692d2..3c1353b69 100644 --- a/platform/mv3/extension/js/ruleset-manager.js +++ b/platform/mv3/extension/js/ruleset-manager.js @@ -33,8 +33,10 @@ import { fetchJSON } from './fetch.js'; const RULE_REALM_SIZE = 1000000; const REGEXES_REALM_START = 1000000; const REGEXES_REALM_END = REGEXES_REALM_START + RULE_REALM_SIZE; -const REMOVEPARAMS_REALM_START = 2000000; +const REMOVEPARAMS_REALM_START = REGEXES_REALM_END; const REMOVEPARAMS_REALM_END = REMOVEPARAMS_REALM_START + RULE_REALM_SIZE; +const REDIRECT_REALM_START = REMOVEPARAMS_REALM_END; +const REDIRECT_REALM_END = REDIRECT_REALM_START + RULE_REALM_SIZE; const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000; const BLOCKING_MODES_RULE_ID = TRUSTED_DIRECTIVE_BASE_RULE_ID + 1; const CURRENT_CONFIG_BASE_RULE_ID = 9000000; @@ -100,8 +102,8 @@ async function updateRegexRules() { // Fetch regexes for all enabled rulesets const toFetch = []; for ( const details of rulesetDetails ) { - if ( details.rules.regexes === 0 ) { continue; } - toFetch.push(fetchJSON(`/rulesets/regex/${details.id}.regexes`)); + if ( details.rules.regex === 0 ) { continue; } + toFetch.push(fetchJSON(`/rulesets/regex/${details.id}`)); } const regexRulesets = await Promise.all(toFetch); @@ -195,8 +197,8 @@ async function updateRemoveparamRules() { // Fetch removeparam rules for all enabled rulesets const toFetch = []; for ( const details of rulesetDetails ) { - if ( details.rules.removeparams === 0 ) { continue; } - toFetch.push(fetchJSON(`/rulesets/removeparam/${details.id}.removeparams`)); + if ( details.rules.removeparam === 0 ) { continue; } + toFetch.push(fetchJSON(`/rulesets/removeparam/${details.id}`)); } const removeparamRulesets = await Promise.all(toFetch); @@ -253,17 +255,90 @@ async function updateRemoveparamRules() { /******************************************************************************/ +async function updateRedirectRules() { + const [ + hasOmnipotence, + rulesetDetails, + dynamicRuleMap, + ] = await Promise.all([ + browser.permissions.contains({ origins: [ '' ] }), + getEnabledRulesetsDetails(), + getDynamicRules(), + ]); + + // Fetch redirect rules for all enabled rulesets + const toFetch = []; + for ( const details of rulesetDetails ) { + if ( details.rules.redirect === 0 ) { continue; } + toFetch.push(fetchJSON(`/rulesets/redirect/${details.id}`)); + } + const redirectRulesets = await Promise.all(toFetch); + + // Redirect rules can only be enforced with omnipotence + const newRules = []; + if ( hasOmnipotence ) { + let redirectRuleId = REDIRECT_REALM_START; + for ( const rules of redirectRulesets ) { + if ( Array.isArray(rules) === false ) { continue; } + for ( const rule of rules ) { + rule.id = redirectRuleId++; + newRules.push(rule); + } + } + } + + // Add redirect rules to dynamic ruleset without affecting rules + // outside redirect rules realm. + const newRuleMap = new Map(newRules.map(rule => [ rule.id, rule ])); + const addRules = []; + const removeRuleIds = []; + + for ( const oldRule of dynamicRuleMap.values() ) { + if ( oldRule.id < REDIRECT_REALM_START ) { continue; } + if ( oldRule.id >= REDIRECT_REALM_END ) { continue; } + const newRule = newRuleMap.get(oldRule.id); + if ( newRule === undefined ) { + removeRuleIds.push(oldRule.id); + dynamicRuleMap.delete(oldRule.id); + } else if ( JSON.stringify(oldRule) !== JSON.stringify(newRule) ) { + removeRuleIds.push(oldRule.id); + addRules.push(newRule); + dynamicRuleMap.set(oldRule.id, newRule); + } + } + + for ( const newRule of newRuleMap.values() ) { + if ( dynamicRuleMap.has(newRule.id) ) { continue; } + addRules.push(newRule); + dynamicRuleMap.set(newRule.id, newRule); + } + + if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; } + + if ( removeRuleIds.length !== 0 ) { + console.info(`Remove ${removeRuleIds.length} DNR redirect rules`); + } + if ( addRules.length !== 0 ) { + console.info(`Add ${addRules.length} DNR redirect rules`); + } + + return dnr.updateDynamicRules({ addRules, removeRuleIds }); +} + +/******************************************************************************/ + async function updateDynamicRules() { return Promise.all([ updateRegexRules(), updateRemoveparamRules(), + updateRedirectRules(), ]); } /******************************************************************************/ async function defaultRulesetsFromLanguage() { - const out = [ 'default' ]; + const out = [ 'default', 'cname-trackers' ]; const dropCountry = lang => { const pos = lang.indexOf('-'); @@ -335,10 +410,7 @@ async function enableRulesets(ids) { } await dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds }); - return Promise.all([ - updateRegexRules(), - updateRemoveparamRules(), - ]); + return updateDynamicRules(); } /******************************************************************************/ diff --git a/platform/mv3/extension/js/settings.js b/platform/mv3/extension/js/settings.js index b36395124..b41d2b457 100644 --- a/platform/mv3/extension/js/settings.js +++ b/platform/mv3/extension/js/settings.js @@ -47,9 +47,9 @@ function rulesetStats(rulesetId) { const rulesetDetails = rulesetMap.get(rulesetId); if ( rulesetDetails === undefined ) { return; } const { rules, filters } = rulesetDetails; - let ruleCount = rules.plain + rules.regexes; + let ruleCount = rules.plain + rules.regex; if ( canRemoveParams ) { - ruleCount += rules.removeparams; + ruleCount += rules.removeparam + rules.redirect; } const filterCount = filters.accepted; return { ruleCount, filterCount }; diff --git a/platform/mv3/extension/manifest.json b/platform/mv3/extension/manifest.json index 765b5df72..03722b3b2 100644 --- a/platform/mv3/extension/manifest.json +++ b/platform/mv3/extension/manifest.json @@ -37,5 +37,6 @@ "scripting" ], "short_name": "uBO Lite", - "version": "0.1" + "version": "0.1", + "web_accessible_resources": [] } diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 629267fb1..8ebab566f 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -28,6 +28,7 @@ import https from 'https'; import path from 'path'; import process from 'process'; import { createHash } from 'crypto'; +import redirectResourcesMap from './js/redirect-resources.js'; import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js'; import { StaticFilteringParser } from './js/static-filtering-parser.js'; import { fnameFromFileId } from './js/utils.js'; @@ -142,6 +143,14 @@ const writeFile = async (fname, data) => { return promise; }; +const copyFile = async (from, to) => { + const dir = path.dirname(to); + await fs.mkdir(dir, { recursive: true }); + const promise = fs.copyFile(from, to); + writeOps.push(promise); + return promise; +}; + const writeOps = []; /******************************************************************************/ @@ -153,6 +162,7 @@ const proceduralDetails = new Map(); const scriptletStats = new Map(); const specificDetails = new Map(); const genericDetails = new Map(); +const requiredRedirectResources = new Set(); /******************************************************************************/ @@ -265,7 +275,12 @@ async function processNetworkFilters(assetDetails, network) { isUnsupported(rule) === false && isRedirect(rule) ); - log(`\tredirect-rule= (discarded): ${redirects.length}`); + redirects.forEach(rule => { + requiredRedirectResources.add( + rule.action.redirect.extensionPath.replace(/^\/+/, '') + ); + }); + log(`\tredirect=: ${redirects.length}`); const headers = rules.filter(rule => isUnsupported(rule) === false && @@ -294,25 +309,33 @@ async function processNetworkFilters(assetDetails, network) { if ( regexes.length !== 0 ) { writeFile( - `${rulesetDir}/regex/${assetDetails.id}.regexes.json`, + `${rulesetDir}/regex/${assetDetails.id}.json`, `${JSON.stringify(regexes, replacer)}\n` ); } if ( removeparamsGood.length !== 0 ) { writeFile( - `${rulesetDir}/removeparam/${assetDetails.id}.removeparams.json`, + `${rulesetDir}/removeparam/${assetDetails.id}.json`, `${JSON.stringify(removeparamsGood, replacer)}\n` ); } + if ( redirects.length !== 0 ) { + writeFile( + `${rulesetDir}/redirect/${assetDetails.id}.json`, + `${JSON.stringify(redirects, replacer)}\n` + ); + } + return { total: rules.length, plain: plainGood.length, discarded: redirects.length + headers.length + removeparamsBad.length, rejected: bad.length, - regexes: regexes.length, - removeparams: removeparamsGood.length, + regex: regexes.length, + removeparam: removeparamsGood.length, + redirect: redirects.length, }; } @@ -902,9 +925,25 @@ async function rulesetFromURLs(assetDetails) { assetDetails.text = text; } + + const extensionPaths = []; + for ( const [ fname, details ] of redirectResourcesMap ) { + const path = `/web_accessible_resources/${fname}`; + extensionPaths.push([ fname, path ]); + if ( details.alias === undefined ) { continue; } + if ( typeof details.alias === 'string' ) { + extensionPaths.push([ details.alias, path ]); + continue; + } + if ( Array.isArray(details.alias) === false ) { continue; } + for ( const alias of details.alias ) { + extensionPaths.push([ alias, path ]); + } + } + const results = await dnrRulesetFromRawLists( [ { name: assetDetails.id, text: assetDetails.text } ], - { env } + { env, extensionPaths } ); const netStats = await processNetworkFilters( @@ -972,8 +1011,9 @@ async function rulesetFromURLs(assetDetails) { rules: { total: netStats.total, plain: netStats.plain, - regexes: netStats.regexes, - removeparams: netStats.removeparams, + regex: netStats.regex, + removeparam: netStats.removeparam, + redirect: netStats.redirect, discarded: netStats.discarded, rejected: netStats.rejected, }, @@ -1113,6 +1153,7 @@ async function main() { urls: [ 'https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers.txt' ], homeURL: 'https://github.com/AdguardTeam/cname-trackers#cname-cloaked-trackers', }); + await rulesetFromURLs({ id: 'stevenblack-hosts', name: 'Steven Black\'s hosts file', @@ -1159,16 +1200,30 @@ async function main() { `${JSON.stringify(genericDetails, jsonSetMapReplacer, 1)}\n` ); + // Copy required redirect resources + for ( const path of requiredRedirectResources ) { + copyFile(`./${path}`, `${outputDir}/${path}`); + } + await Promise.all(writeOps); // Patch manifest + // Patch declarative_net_request key manifest.declarative_net_request = { rule_resources: ruleResources }; + // Patch web_accessible_resources key + manifest.web_accessible_resources = [{ + resources: Array.from(requiredRedirectResources).map(path => `/${path}`), + matches: [ '' ], + use_dynamic_url: true, + }]; + // Patch version key const now = new Date(); const yearPart = now.getUTCFullYear() - 2000; const monthPart = (now.getUTCMonth() + 1) * 1000; const dayPart = now.getUTCDate() * 10; const hourPart = Math.floor(now.getUTCHours() / 3) + 1; manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`; + // Commit changes await fs.writeFile( `${outputDir}/manifest.json`, JSON.stringify(manifest, null, 2) + '\n' diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 269899413..455de0d37 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -23,6 +23,8 @@ /******************************************************************************/ +import redirectableResources from './redirect-resources.js'; + import { LineIterator, orphanizeString, @@ -30,165 +32,6 @@ import { /******************************************************************************/ -// The resources referenced below are found in ./web_accessible_resources/ -// -// The content of the resources which declare a `data` property will be loaded -// in memory, and converted to a suitable internal format depending on the -// type of the loaded data. The `data` property allows for manual injection -// through `+js(...)`, or for redirection to a data: URI when a redirection -// to a web accessible resource is not desirable. - -const redirectableResources = new Map([ - [ '1x1.gif', { - alias: '1x1-transparent.gif', - data: 'blob', - } ], - [ '2x2.png', { - alias: '2x2-transparent.png', - data: 'blob', - } ], - [ '3x2.png', { - alias: '3x2-transparent.png', - data: 'blob', - } ], - [ '32x32.png', { - alias: '32x32-transparent.png', - data: 'blob', - } ], - [ 'addthis_widget.js', { - alias: 'addthis.com/addthis_widget.js', - } ], - [ 'amazon_ads.js', { - alias: 'amazon-adsystem.com/aax2/amzn_ads.js', - data: 'text', - } ], - [ 'amazon_apstag.js', { - } ], - [ 'ampproject_v0.js', { - alias: 'ampproject.org/v0.js', - } ], - [ 'chartbeat.js', { - alias: 'static.chartbeat.com/chartbeat.js', - } ], - [ 'click2load.html', { - params: [ 'aliasURL', 'url' ], - } ], - [ 'doubleclick_instream_ad_status.js', { - alias: 'doubleclick.net/instream/ad_status.js', - data: 'text', - } ], - [ 'empty', { - data: 'text', // Important! - } ], - [ 'fingerprint2.js', { - data: 'text', - } ], - [ 'fingerprint3.js', { - data: 'text', - } ], - [ 'google-analytics_analytics.js', { - alias: [ - 'google-analytics.com/analytics.js', - 'googletagmanager_gtm.js', - 'googletagmanager.com/gtm.js' - ], - data: 'text', - } ], - [ 'google-analytics_cx_api.js', { - alias: 'google-analytics.com/cx/api.js', - } ], - [ 'google-analytics_ga.js', { - alias: 'google-analytics.com/ga.js', - data: 'text', - } ], - [ 'google-analytics_inpage_linkid.js', { - alias: 'google-analytics.com/inpage_linkid.js', - } ], - [ 'google-ima.js', { - } ], - [ 'googlesyndication_adsbygoogle.js', { - alias: 'googlesyndication.com/adsbygoogle.js', - data: 'text', - } ], - [ 'googletagservices_gpt.js', { - alias: 'googletagservices.com/gpt.js', - data: 'text', - } ], - [ 'hd-main.js', { - } ], - [ 'ligatus_angular-tag.js', { - alias: 'ligatus.com/*/angular-tag.js', - } ], - [ 'mxpnl_mixpanel.js', { - } ], - [ 'monkeybroker.js', { - alias: 'd3pkae9owd2lcf.cloudfront.net/mb105.js', - } ], - [ 'noeval.js', { - data: 'text', - } ], - [ 'noeval-silent.js', { - alias: 'silent-noeval.js', - data: 'text', - } ], - [ 'nobab.js', { - alias: 'bab-defuser.js', - data: 'text', - } ], - [ 'nobab2.js', { - data: 'text', - } ], - [ 'nofab.js', { - alias: 'fuckadblock.js-3.2.0', - data: 'text', - } ], - [ 'noop-0.1s.mp3', { - alias: [ 'noopmp3-0.1s', 'abp-resource:blank-mp3' ], - data: 'blob', - } ], - [ 'noop-0.5s.mp3', { - } ], - [ 'noop-1s.mp4', { - alias: 'noopmp4-1s', - data: 'blob', - } ], - [ 'noop.html', { - alias: 'noopframe', - } ], - [ 'noop.js', { - alias: [ 'noopjs', 'abp-resource:blank-js' ], - data: 'text', - } ], - [ 'noop.txt', { - alias: 'nooptext', - data: 'text', - } ], - [ 'noop-vmap1.0.xml', { - alias: 'noopvmap-1.0', - data: 'text', - } ], - [ 'outbrain-widget.js', { - alias: 'widgets.outbrain.com/outbrain.js', - } ], - [ 'popads.js', { - alias: 'popads.net.js', - data: 'text', - } ], - [ 'popads-dummy.js', { - data: 'text', - } ], - [ 'prebid-ads.js', { - data: 'text', - } ], - [ 'scorecardresearch_beacon.js', { - alias: 'scorecardresearch.com/beacon.js', - } ], - [ 'window.open-defuser.js', { - alias: 'nowoif.js', - data: 'text', - } ], -]); - const extToMimeMap = new Map([ [ 'gif', 'image/gif' ], [ 'html', 'text/html' ], diff --git a/src/js/redirect-resources.js b/src/js/redirect-resources.js new file mode 100644 index 000000000..76e93304c --- /dev/null +++ b/src/js/redirect-resources.js @@ -0,0 +1,183 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// The resources referenced below are found in ./web_accessible_resources/ +// +// The content of the resources which declare a `data` property will be loaded +// in memory, and converted to a suitable internal format depending on the +// type of the loaded data. The `data` property allows for manual injection +// through `+js(...)`, or for redirection to a data: URI when a redirection +// to a web accessible resource is not desirable. + +export default new Map([ + [ '1x1.gif', { + alias: '1x1-transparent.gif', + data: 'blob', + } ], + [ '2x2.png', { + alias: '2x2-transparent.png', + data: 'blob', + } ], + [ '3x2.png', { + alias: '3x2-transparent.png', + data: 'blob', + } ], + [ '32x32.png', { + alias: '32x32-transparent.png', + data: 'blob', + } ], + [ 'addthis_widget.js', { + alias: 'addthis.com/addthis_widget.js', + } ], + [ 'amazon_ads.js', { + alias: 'amazon-adsystem.com/aax2/amzn_ads.js', + data: 'text', + } ], + [ 'amazon_apstag.js', { + } ], + [ 'ampproject_v0.js', { + alias: 'ampproject.org/v0.js', + } ], + [ 'chartbeat.js', { + alias: 'static.chartbeat.com/chartbeat.js', + } ], + [ 'click2load.html', { + params: [ 'aliasURL', 'url' ], + } ], + [ 'doubleclick_instream_ad_status.js', { + alias: 'doubleclick.net/instream/ad_status.js', + data: 'text', + } ], + [ 'empty', { + data: 'text', // Important! + } ], + [ 'fingerprint2.js', { + data: 'text', + } ], + [ 'fingerprint3.js', { + data: 'text', + } ], + [ 'google-analytics_analytics.js', { + alias: [ + 'google-analytics.com/analytics.js', + 'googletagmanager_gtm.js', + 'googletagmanager.com/gtm.js' + ], + data: 'text', + } ], + [ 'google-analytics_cx_api.js', { + alias: 'google-analytics.com/cx/api.js', + } ], + [ 'google-analytics_ga.js', { + alias: 'google-analytics.com/ga.js', + data: 'text', + } ], + [ 'google-analytics_inpage_linkid.js', { + alias: 'google-analytics.com/inpage_linkid.js', + } ], + [ 'google-ima.js', { + } ], + [ 'googlesyndication_adsbygoogle.js', { + alias: 'googlesyndication.com/adsbygoogle.js', + data: 'text', + } ], + [ 'googletagservices_gpt.js', { + alias: 'googletagservices.com/gpt.js', + data: 'text', + } ], + [ 'hd-main.js', { + } ], + [ 'ligatus_angular-tag.js', { + alias: 'ligatus.com/*/angular-tag.js', + } ], + [ 'mxpnl_mixpanel.js', { + } ], + [ 'monkeybroker.js', { + alias: 'd3pkae9owd2lcf.cloudfront.net/mb105.js', + } ], + [ 'noeval.js', { + data: 'text', + } ], + [ 'noeval-silent.js', { + alias: 'silent-noeval.js', + data: 'text', + } ], + [ 'nobab.js', { + alias: 'bab-defuser.js', + data: 'text', + } ], + [ 'nobab2.js', { + data: 'text', + } ], + [ 'nofab.js', { + alias: 'fuckadblock.js-3.2.0', + data: 'text', + } ], + [ 'noop-0.1s.mp3', { + alias: [ 'noopmp3-0.1s', 'abp-resource:blank-mp3' ], + data: 'blob', + } ], + [ 'noop-0.5s.mp3', { + } ], + [ 'noop-1s.mp4', { + alias: 'noopmp4-1s', + data: 'blob', + } ], + [ 'noop.html', { + alias: 'noopframe', + } ], + [ 'noop.js', { + alias: [ 'noopjs', 'abp-resource:blank-js' ], + data: 'text', + } ], + [ 'noop.txt', { + alias: 'nooptext', + data: 'text', + } ], + [ 'noop-vmap1.0.xml', { + alias: 'noopvmap-1.0', + data: 'text', + } ], + [ 'outbrain-widget.js', { + alias: 'widgets.outbrain.com/outbrain.js', + } ], + [ 'popads.js', { + alias: 'popads.net.js', + data: 'text', + } ], + [ 'popads-dummy.js', { + data: 'text', + } ], + [ 'prebid-ads.js', { + data: 'text', + } ], + [ 'scorecardresearch_beacon.js', { + alias: 'scorecardresearch.com/beacon.js', + } ], + [ 'window.open-defuser.js', { + alias: 'nowoif.js', + data: 'text', + } ], +]); diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index 9347a825b..a453ecc81 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -219,6 +219,9 @@ function addToDNR(context, list) { }); const compiler = staticNetFilteringEngine.createCompiler(parser); + // Can't enforce `redirect-rule=` with DNR + compiler.excludeOptions([ parser.OPTTokenRedirectRule ]); + writer.properties.set('name', list.name); compiler.start(writer); diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 6e102a2eb..ad7970c18 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -1742,8 +1742,7 @@ const FilterOriginEntityHit = class extends FilterOriginHit { } static dnrFromCompiled(args, rule) { - dnrAddRuleError(rule, `Entity not supported: ${args[1]}`); - super.dnrFromCompiled(args, rule); + dnrAddRuleError(rule, `FilterOriginEntityHit: Entity ${args[1]} not supported`); } }; @@ -1761,8 +1760,7 @@ const FilterOriginEntityMiss = class extends FilterOriginMiss { } static dnrFromCompiled(args, rule) { - dnrAddRuleError(rule, `Entity not supported: ${args[1]}`); - super.dnrFromCompiled(args, rule); + dnrAddRuleError(rule, `FilterOriginEntityMiss: Entity ${args[1]} not supported`); } }; @@ -2623,7 +2621,7 @@ const FilterStrictParty = class { static dnrFromCompiled(args, rule) { const partyness = args[1] === 0 ? 1 : 3; - dnrAddRuleError(rule, `Strict partyness not supported: strict${partyness}p`); + dnrAddRuleError(rule, `FilterStrictParty: Strict partyness strict${partyness}p not supported`); } static keyFromArgs(args) { @@ -2891,6 +2889,7 @@ class FilterCompiler { [ parser.OPTTokenWebrtc, bitFromType('unsupported') ], [ parser.OPTTokenWebsocket, bitFromType('websocket') ], ]); + this.excludedOptionSet = new Set(); // These top 100 "bad tokens" are collated using the "miss" histogram // from tokenHistograms(). The "score" is their occurrence among the // 200K+ URLs used in the benchmark and executed against default @@ -3053,6 +3052,12 @@ class FilterCompiler { return ''; } + excludeOptions(options) { + for ( const option of options ) { + this.excludedOptionSet.add(option); + } + } + // https://github.com/chrisaljoudi/uBlock/issues/589 // Be ready to handle multiple negated types @@ -3109,30 +3114,31 @@ class FilterCompiler { } processOptions() { - for ( let { id, val, not } of this.parser.netOptions() ) { + const { parser } = this; + for ( let { id, val, not } of parser.netOptions() ) { switch ( id ) { - case this.parser.OPTToken1p: + case parser.OPTToken1p: this.processPartyOption(true, not); break; - case this.parser.OPTToken1pStrict: + case parser.OPTToken1pStrict: this.strictParty = this.strictParty === -1 ? 0 : 1; this.optionUnitBits |= this.STRICT_PARTY_BIT; break; - case this.parser.OPTToken3p: + case parser.OPTToken3p: this.processPartyOption(false, not); break; - case this.parser.OPTToken3pStrict: + case parser.OPTToken3pStrict: this.strictParty = this.strictParty === 1 ? 0 : -1; this.optionUnitBits |= this.STRICT_PARTY_BIT; break; - case this.parser.OPTTokenAll: + case parser.OPTTokenAll: this.processTypeOption(-1); break; // https://github.com/uBlockOrigin/uAssets/issues/192 - case this.parser.OPTTokenBadfilter: + case parser.OPTTokenBadfilter: this.badFilter = true; break; - case this.parser.OPTTokenCsp: + case parser.OPTTokenCsp: if ( this.processModifierOption(id, val) === false ) { return false; } @@ -3144,7 +3150,7 @@ class FilterCompiler { // https://github.com/gorhill/uBlock/issues/2294 // Detect and discard filter if domain option contains // nonsensical characters. - case this.parser.OPTTokenDomain: + case parser.OPTTokenDomain: this.domainOpt = this.processHostnameList( val, 0b1010, @@ -3153,73 +3159,76 @@ class FilterCompiler { if ( this.domainOpt === '' ) { return false; } this.optionUnitBits |= this.DOMAIN_BIT; break; - case this.parser.OPTTokenDenyAllow: + case parser.OPTTokenDenyAllow: this.denyallowOpt = this.processHostnameList(val, 0b0000); if ( this.denyallowOpt === '' ) { return false; } this.optionUnitBits |= this.DENYALLOW_BIT; break; // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ // Add support for `elemhide`. Rarely used but it happens. - case this.parser.OPTTokenEhide: - this.processTypeOption(this.parser.OPTTokenShide, not); - this.processTypeOption(this.parser.OPTTokenGhide, not); + case parser.OPTTokenEhide: + this.processTypeOption(parser.OPTTokenShide, not); + this.processTypeOption(parser.OPTTokenGhide, not); break; - case this.parser.OPTTokenHeader: + case parser.OPTTokenHeader: this.headerOpt = val !== undefined ? val : ''; this.optionUnitBits |= this.HEADER_BIT; break; - case this.parser.OPTTokenImportant: + case parser.OPTTokenImportant: if ( this.action === AllowAction ) { return false; } this.optionUnitBits |= this.IMPORTANT_BIT; this.action = BlockImportant; break; // Used by Adguard: // https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#empty-modifier - case this.parser.OPTTokenEmpty: + case parser.OPTTokenEmpty: id = this.action === AllowAction - ? this.parser.OPTTokenRedirectRule - : this.parser.OPTTokenRedirect; + ? parser.OPTTokenRedirectRule + : parser.OPTTokenRedirect; if ( this.processModifierOption(id, 'empty') === false ) { return false; } this.optionUnitBits |= this.REDIRECT_BIT; break; - case this.parser.OPTTokenMatchCase: + case parser.OPTTokenMatchCase: this.patternMatchCase = true; break; - case this.parser.OPTTokenMp4: + case parser.OPTTokenMp4: id = this.action === AllowAction - ? this.parser.OPTTokenRedirectRule - : this.parser.OPTTokenRedirect; + ? parser.OPTTokenRedirectRule + : parser.OPTTokenRedirect; if ( this.processModifierOption(id, 'noopmp4-1s') === false ) { return false; } this.optionUnitBits |= this.REDIRECT_BIT; break; - case this.parser.OPTTokenNoop: + case parser.OPTTokenNoop: break; - case this.parser.OPTTokenRemoveparam: + case parser.OPTTokenRemoveparam: if ( this.processModifierOption(id, val) === false ) { return false; } this.optionUnitBits |= this.REMOVEPARAM_BIT; break; - case this.parser.OPTTokenRedirect: + case parser.OPTTokenRedirect: if ( this.action === AllowAction ) { - id = this.parser.OPTTokenRedirectRule; + id = parser.OPTTokenRedirectRule; } if ( this.processModifierOption(id, val) === false ) { return false; } this.optionUnitBits |= this.REDIRECT_BIT; break; - case this.parser.OPTTokenRedirectRule: + case parser.OPTTokenRedirectRule: + if ( this.excludedOptionSet.has(parser.OPTTokenRedirectRule) ) { + return false; + } if ( this.processModifierOption(id, val) === false ) { return false; } this.optionUnitBits |= this.REDIRECT_BIT; break; - case this.parser.OPTTokenInvalid: + case parser.OPTTokenInvalid: return false; default: if ( this.tokenIdToNormalizedType.has(id) === false ) { @@ -4051,6 +4060,34 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } } + // Try to recover from errors for when the rule is still useful despite not + // being complete. + for ( const rule of ruleset ) { + if ( rule._error === undefined ) { continue; } + let i = rule._error.length; + while ( i-- ) { + const error = rule._error[i]; + if ( error.startsWith('FilterOriginEntityHit:') ) { + if ( + Array.isArray(rule.condition.initiatorDomains) && + rule.condition.initiatorDomains.length > 0 + ) { + rule._error.splice(i, 1); + } + } else if ( error.startsWith('FilterOriginEntityMiss:') ) { + if ( + Array.isArray(rule.condition.excludedInitiatorDomains) && + rule.condition.excludedInitiatorDomains.length > 0 + ) { + rule._error.splice(i, 1); + } + } + } + if ( rule._error.length === 0 ) { + delete rule._error; + } + } + // Patch modifier filters for ( const rule of ruleset ) { if ( rule.__modifierType === undefined ) { continue; } @@ -4067,10 +4104,12 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } break; case 'redirect-rule': { + let priority = rule.priority || 0; let token = rule.__modifierValue; if ( token !== '' ) { - const match = /:\d+$/.exec(token); + const match = /:(\d+)$/.exec(token); if ( match !== null ) { + priority += parseInt(match[1], 10); token = token.slice(0, match.index); } } @@ -4078,14 +4117,14 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { if ( rule.__modifierValue !== '' && resource === undefined ) { dnrAddRuleError(rule, `Unpatchable redirect filter: ${rule.__modifierValue}`); } - const extensionPath = resource && resource.extensionPath || token; if ( rule.__modifierAction !== AllowAction ) { + const extensionPath = resource || token; rule.action.type = 'redirect'; rule.action.redirect = { extensionPath }; - rule.priority = (rule.priority || 1) + 1; + rule.priority = priority + 1; } else { rule.action.type = 'block'; - rule.priority = (rule.priority || 1) + 2; + rule.priority = priority + 2; } break; } diff --git a/tools/make-mv3.sh b/tools/make-mv3.sh index 675d761e8..f9f34ccd7 100755 --- a/tools/make-mv3.sh +++ b/tools/make-mv3.sh @@ -51,6 +51,8 @@ if [ "$1" != "quick" ]; then cp platform/mv3/extension/js/utils.js $TMPDIR/js/ cp assets/assets.json $TMPDIR/ cp -R platform/mv3/scriptlets $TMPDIR/ + mkdir -p $TMPDIR/web_accessible_resources + cp src/web_accessible_resources/* $TMPDIR/web_accessible_resources/ cd $TMPDIR node --no-warnings make-rulesets.js output=$DES cd - > /dev/null diff --git a/tools/make-nodejs.sh b/tools/make-nodejs.sh index 623a6a570..1e38ba143 100755 --- a/tools/make-nodejs.sh +++ b/tools/make-nodejs.sh @@ -13,6 +13,7 @@ cp src/js/dynamic-net-filtering.js $DES/js cp src/js/filtering-context.js $DES/js cp src/js/hnswitches.js $DES/js cp src/js/hntrie.js $DES/js +cp src/js/redirect-resources.js $DES/js cp src/js/static-dnr-filtering.js $DES/js cp src/js/static-filtering-parser.js $DES/js cp src/js/static-net-filtering.js $DES/js