From 857abb380b477337a0e38d1f312158c75d12aae9 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sat, 19 Aug 2023 07:48:14 -0400 Subject: [PATCH] [mv3] Add support for converting `^responseheader()` filters to DNR Additionally, finalize versioning scheme for uBOL. Since most updates will be simply related to update rulesets, the version will from now on reflects the date at which the extension package was created: year.month.day.minutes So for example: 2023.8.19.690 --- platform/mv3/make-rulesets.js | 57 +++++------ src/js/static-dnr-filtering.js | 180 ++++++++++++++++++++++++++++++++- src/js/static-net-filtering.js | 141 +------------------------- 3 files changed, 208 insertions(+), 170 deletions(-) diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 65e6bde45..de8a8a9f6 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -320,18 +320,19 @@ async function processNetworkFilters(assetDetails, network) { log(`Output rule count: ${rules.length}`); // https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/RuleCondition#browser_compatibility - // isUrlFilterCaseSensitive is false by default in Firefox. It will be + // isUrlFilterCaseSensitive is true by default in Chromium. It will be // false by default in Chromium 118+. - if ( platform === 'firefox' ) { + if ( platform !== 'firefox' ) { for ( const rule of rules ) { - if ( rule.condition === undefined ) { continue; } - if ( rule.condition.urlFilter === undefined ) { - if ( rule.condition.regexFilter === undefined ) { continue; } + const { condition } = rule; + if ( condition === undefined ) { continue; } + if ( condition.urlFilter === undefined ) { + if ( condition.regexFilter === undefined ) { continue; } } - if ( rule.condition.isUrlFilterCaseSensitive === undefined ) { - rule.condition.isUrlFilterCaseSensitive = true; - } else if ( rule.condition.isUrlFilterCaseSensitive === false ) { - rule.condition.isUrlFilterCaseSensitive = undefined; + if ( condition.isUrlFilterCaseSensitive === undefined ) { + condition.isUrlFilterCaseSensitive = false; + } else if ( condition.isUrlFilterCaseSensitive === true ) { + condition.isUrlFilterCaseSensitive = undefined; } } } @@ -1098,23 +1099,15 @@ async function rulesetFromURLs(assetDetails) { async function main() { - // Get manifest content - const manifest = await fs.readFile( - `${outputDir}/manifest.json`, - { encoding: 'utf8' } - ).then(text => - JSON.parse(text) - ); - - // Create unique version number according to build time - let version = manifest.version; + let version = ''; { const now = new Date(); - const yearPart = now.getUTCFullYear() - 2000; - const monthPart = (now.getUTCMonth() + 1) * 1000; - const dayPart = now.getUTCDate() * 10; - const hourPart = Math.floor(now.getUTCHours() / 3) + 1; - version += `.${yearPart}.${monthPart + dayPart + hourPart}`; + const yearPart = now.getUTCFullYear(); + const monthPart = now.getUTCMonth() + 1; + const dayPart = now.getUTCDate(); + const hourPart = Math.floor(now.getUTCHours()); + const minutePart = Math.floor(now.getUTCMinutes()); + version = `${yearPart}.${monthPart}.${dayPart}.${hourPart * 60 + minutePart}`; } log(`Version: ${version}`); @@ -1300,6 +1293,13 @@ async function main() { await Promise.all(writeOps); // Patch manifest + // Get manifest content + const manifest = await fs.readFile( + `${outputDir}/manifest.json`, + { encoding: 'utf8' } + ).then(text => + JSON.parse(text) + ); // Patch declarative_net_request key manifest.declarative_net_request = { rule_resources: ruleResources }; // Patch web_accessible_resources key @@ -1312,13 +1312,8 @@ async function main() { } manifest.web_accessible_resources = [ web_accessible_resources ]; - // Patch version key - const now = new Date(); - const yearPart = now.getUTCFullYear() - 2000; - const monthPart = (now.getUTCMonth() + 1) * 1000; - const dayPart = now.getUTCDate() * 10; - const hourPart = Math.floor(now.getUTCHours() / 3) + 1; - manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`; + // Patch manifest version property + manifest.version = version; // Commit changes await fs.writeFile( `${outputDir}/manifest.json`, diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index df3126c71..ec6ba4562 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -134,7 +134,54 @@ function addExtendedToDNR(context, parser) { } // Response header filtering - if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { + if ( parser.isResponseheaderFilter() ) { + if ( parser.hasError() ) { return; } + if ( parser.hasOptions() === false ) { return; } + if ( parser.isException() ) { return; } + const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER); + if ( node === 0 ) { return; } + const header = parser.getNodeString(node); + if ( context.responseHeaderRules === undefined ) { + context.responseHeaderRules = []; + } + const rule = { + action: { + responseHeaders: [ + { + header, + operation: 'remove', + } + ], + type: 'modifyHeaders' + }, + condition: { + resourceTypes: [ + 'main_frame', + 'sub_frame' + ] + }, + }; + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + if ( not ) { + if ( rule.condition.excludedInitiatorDomains === undefined ) { + rule.condition.excludedInitiatorDomains = []; + } + rule.condition.excludedInitiatorDomains.push(hn); + continue; + } + if ( hn === '*' ) { + if ( rule.condition.initiatorDomains !== undefined ) { + rule.condition.initiatorDomains = undefined; + } + continue; + } + if ( rule.condition.initiatorDomains === undefined ) { + rule.condition.initiatorDomains = []; + } + rule.condition.initiatorDomains.push(hn); + } + context.responseHeaderRules.push(rule); return; } @@ -286,6 +333,129 @@ function addToDNR(context, list) { /******************************************************************************/ +function finalizeRuleset(context, network) { + const ruleset = network.ruleset; + + // Assign rule ids + const rulesetMap = new Map(); + { + let ruleId = 1; + for ( const rule of ruleset ) { + rulesetMap.set(ruleId++, rule); + } + } + // Merge rules where possible by merging arrays of a specific property. + // + // https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579 + // Do not merge rules which have errors. + const mergeRules = (rulesetMap, mergeTarget) => { + const mergeMap = new Map(); + const sorter = (_, v) => { + if ( Array.isArray(v) ) { + return typeof v[0] === 'string' ? v.sort() : v; + } + if ( v instanceof Object ) { + const sorted = {}; + for ( const kk of Object.keys(v).sort() ) { + sorted[kk] = v[kk]; + } + return sorted; + } + return v; + }; + const ruleHasher = (rule, target) => { + return JSON.stringify(rule, (k, v) => { + if ( k.startsWith('_') ) { return; } + if ( k === target ) { return; } + return sorter(k, v); + }); + }; + const extractTargetValue = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return v; } + if ( v instanceof Object ) { + const r = extractTargetValue(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + const extractTargetOwner = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return obj; } + if ( v instanceof Object ) { + const r = extractTargetOwner(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + for ( const [ id, rule ] of rulesetMap ) { + if ( rule._error !== undefined ) { continue; } + const hash = ruleHasher(rule, mergeTarget); + if ( mergeMap.has(hash) === false ) { + mergeMap.set(hash, []); + } + mergeMap.get(hash).push(id); + } + for ( const ids of mergeMap.values() ) { + if ( ids.length === 1 ) { continue; } + const leftHand = rulesetMap.get(ids[0]); + const leftHandSet = new Set( + extractTargetValue(leftHand, mergeTarget) || [] + ); + for ( let i = 1; i < ids.length; i++ ) { + const rightHandId = ids[i]; + const rightHand = rulesetMap.get(rightHandId); + const rightHandArray = extractTargetValue(rightHand, mergeTarget); + if ( rightHandArray !== undefined ) { + if ( leftHandSet.size !== 0 ) { + for ( const item of rightHandArray ) { + leftHandSet.add(item); + } + } + } else { + leftHandSet.clear(); + } + rulesetMap.delete(rightHandId); + } + const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); + if ( leftHandSet.size > 1 ) { + //if ( leftHandOwner === undefined ) { debugger; } + leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); + } else if ( leftHandSet.size === 0 ) { + if ( leftHandOwner !== undefined ) { + leftHandOwner[mergeTarget] = undefined; + } + } + } + }; + mergeRules(rulesetMap, 'resourceTypes'); + mergeRules(rulesetMap, 'initiatorDomains'); + mergeRules(rulesetMap, 'requestDomains'); + mergeRules(rulesetMap, 'removeParams'); + mergeRules(rulesetMap, 'responseHeaders'); + + // Patch id + const rulesetFinal = []; + { + let ruleId = 1; + for ( const rule of rulesetMap.values() ) { + if ( rule._error === undefined ) { + rule.id = ruleId++; + } else { + rule.id = 0; + } + rulesetFinal.push(rule); + } + for ( const invalid of context.invalid ) { + rulesetFinal.push({ _error: [ invalid ] }); + } + } + + network.ruleset = rulesetFinal; +} + +/******************************************************************************/ + async function dnrRulesetFromRawLists(lists, options = {}) { const context = Object.assign({}, options); staticNetFilteringEngine.dnrFromCompiled('begin', context); @@ -300,8 +470,7 @@ async function dnrRulesetFromRawLists(lists, options = {}) { } } await Promise.all(toLoad); - - return { + const result = { network: staticNetFilteringEngine.dnrFromCompiled('end', context), genericCosmetic: context.genericCosmeticFilters, genericHighCosmetic: context.genericHighCosmeticFilters, @@ -309,6 +478,11 @@ async function dnrRulesetFromRawLists(lists, options = {}) { specificCosmetic: context.specificCosmeticFilters, scriptlet: context.scriptletFilters, }; + if ( context.responseHeaderRules ) { + result.network.ruleset.push(...context.responseHeaderRules); + } + finalizeRuleset(context, result.network); + return result; } /******************************************************************************/ diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index ab14f3829..e7035daea 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -1274,7 +1274,9 @@ class FilterRegex { dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`); } rule.condition.regexFilter = args[1]; - rule.condition.isUrlFilterCaseSensitive = args[2] === 1; + if ( args[2] === 1 ) { + rule.condition.isUrlFilterCaseSensitive = true; + } } static keyFromArgs(args) { @@ -4349,7 +4351,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { 'ping', 'other', ]); - let ruleset = []; + const ruleset = []; for ( const [ realmBits, realmName ] of realms ) { for ( const [ partyBits, partyName ] of partyness ) { for ( const typeName in typeNameToTypeValue ) { @@ -4521,141 +4523,8 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } } - // Assign rule ids - const rulesetMap = new Map(); - { - let ruleId = 1; - for ( const rule of ruleset ) { - rulesetMap.set(ruleId++, rule); - } - } - - // Merge rules where possible by merging arrays of a specific property. - // - // https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579 - // Do not merge rules which have errors. - const mergeRules = (rulesetMap, mergeTarget) => { - const mergeMap = new Map(); - const sorter = (_, v) => { - if ( Array.isArray(v) ) { - return typeof v[0] === 'string' ? v.sort() : v; - } - if ( v instanceof Object ) { - const sorted = {}; - for ( const kk of Object.keys(v).sort() ) { - sorted[kk] = v[kk]; - } - return sorted; - } - return v; - }; - const ruleHasher = (rule, target) => { - return JSON.stringify(rule, (k, v) => { - if ( k.startsWith('_') ) { return; } - if ( k === target ) { return; } - return sorter(k, v); - }); - }; - const extractTargetValue = (obj, target) => { - for ( const [ k, v ] of Object.entries(obj) ) { - if ( Array.isArray(v) && k === target ) { return v; } - if ( v instanceof Object ) { - const r = extractTargetValue(v, target); - if ( r !== undefined ) { return r; } - } - } - }; - const extractTargetOwner = (obj, target) => { - for ( const [ k, v ] of Object.entries(obj) ) { - if ( Array.isArray(v) && k === target ) { return obj; } - if ( v instanceof Object ) { - const r = extractTargetOwner(v, target); - if ( r !== undefined ) { return r; } - } - } - }; - for ( const [ id, rule ] of rulesetMap ) { - if ( rule._error !== undefined ) { continue; } - const hash = ruleHasher(rule, mergeTarget); - if ( mergeMap.has(hash) === false ) { - mergeMap.set(hash, []); - } - mergeMap.get(hash).push(id); - } - for ( const ids of mergeMap.values() ) { - if ( ids.length === 1 ) { continue; } - const leftHand = rulesetMap.get(ids[0]); - const leftHandSet = new Set( - extractTargetValue(leftHand, mergeTarget) || [] - ); - for ( let i = 1; i < ids.length; i++ ) { - const rightHandId = ids[i]; - const rightHand = rulesetMap.get(rightHandId); - const rightHandArray = extractTargetValue(rightHand, mergeTarget); - if ( rightHandArray !== undefined ) { - if ( leftHandSet.size !== 0 ) { - for ( const item of rightHandArray ) { - leftHandSet.add(item); - } - } - } else { - leftHandSet.clear(); - } - rulesetMap.delete(rightHandId); - } - const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); - if ( leftHandSet.size > 1 ) { - //if ( leftHandOwner === undefined ) { debugger; } - leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); - } else if ( leftHandSet.size === 0 ) { - if ( leftHandOwner !== undefined ) { - leftHandOwner[mergeTarget] = undefined; - } - } - } - }; - mergeRules(rulesetMap, 'resourceTypes'); - mergeRules(rulesetMap, 'initiatorDomains'); - mergeRules(rulesetMap, 'requestDomains'); - mergeRules(rulesetMap, 'removeParams'); - mergeRules(rulesetMap, 'responseHeaders'); - - // Patch case-sensitiveness - for ( const rule of rulesetMap.values() ) { - const { condition } = rule; - if ( - condition === undefined || - condition.urlFilter === undefined && - condition.regexFilter === undefined - ) { - continue; - } - if ( condition.isUrlFilterCaseSensitive === undefined ) { - condition.isUrlFilterCaseSensitive = false; - } else if ( condition.isUrlFilterCaseSensitive === true ) { - condition.isUrlFilterCaseSensitive = undefined; - } - } - - // Patch id - const rulesetFinal = []; - { - let ruleId = 1; - for ( const rule of rulesetMap.values() ) { - if ( rule._error === undefined ) { - rule.id = ruleId++; - } else { - rule.id = 0; - } - rulesetFinal.push(rule); - } - for ( const invalid of context.invalid ) { - rulesetFinal.push({ _error: [ invalid ] }); - } - } - return { - ruleset: rulesetFinal, + ruleset, filterCount: context.filterCount, acceptedFilterCount: context.acceptedFilterCount, rejectedFilterCount: context.rejectedFilterCount,