diff --git a/platform/mv3/make-rulesets.js b/platform/mv3/make-rulesets.js index 65e6bde45..de8a8a9f6 100644 --- a/platform/mv3/make-rulesets.js +++ b/platform/mv3/make-rulesets.js @@ -320,18 +320,19 @@ async function processNetworkFilters(assetDetails, network) { log(`Output rule count: ${rules.length}`); // https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/RuleCondition#browser_compatibility - // isUrlFilterCaseSensitive is false by default in Firefox. It will be + // isUrlFilterCaseSensitive is true by default in Chromium. It will be // false by default in Chromium 118+. - if ( platform === 'firefox' ) { + if ( platform !== 'firefox' ) { for ( const rule of rules ) { - if ( rule.condition === undefined ) { continue; } - if ( rule.condition.urlFilter === undefined ) { - if ( rule.condition.regexFilter === undefined ) { continue; } + const { condition } = rule; + if ( condition === undefined ) { continue; } + if ( condition.urlFilter === undefined ) { + if ( condition.regexFilter === undefined ) { continue; } } - if ( rule.condition.isUrlFilterCaseSensitive === undefined ) { - rule.condition.isUrlFilterCaseSensitive = true; - } else if ( rule.condition.isUrlFilterCaseSensitive === false ) { - rule.condition.isUrlFilterCaseSensitive = undefined; + if ( condition.isUrlFilterCaseSensitive === undefined ) { + condition.isUrlFilterCaseSensitive = false; + } else if ( condition.isUrlFilterCaseSensitive === true ) { + condition.isUrlFilterCaseSensitive = undefined; } } } @@ -1098,23 +1099,15 @@ async function rulesetFromURLs(assetDetails) { async function main() { - // Get manifest content - const manifest = await fs.readFile( - `${outputDir}/manifest.json`, - { encoding: 'utf8' } - ).then(text => - JSON.parse(text) - ); - - // Create unique version number according to build time - let version = manifest.version; + let version = ''; { const now = new Date(); - const yearPart = now.getUTCFullYear() - 2000; - const monthPart = (now.getUTCMonth() + 1) * 1000; - const dayPart = now.getUTCDate() * 10; - const hourPart = Math.floor(now.getUTCHours() / 3) + 1; - version += `.${yearPart}.${monthPart + dayPart + hourPart}`; + const yearPart = now.getUTCFullYear(); + const monthPart = now.getUTCMonth() + 1; + const dayPart = now.getUTCDate(); + const hourPart = Math.floor(now.getUTCHours()); + const minutePart = Math.floor(now.getUTCMinutes()); + version = `${yearPart}.${monthPart}.${dayPart}.${hourPart * 60 + minutePart}`; } log(`Version: ${version}`); @@ -1300,6 +1293,13 @@ async function main() { await Promise.all(writeOps); // Patch manifest + // Get manifest content + const manifest = await fs.readFile( + `${outputDir}/manifest.json`, + { encoding: 'utf8' } + ).then(text => + JSON.parse(text) + ); // Patch declarative_net_request key manifest.declarative_net_request = { rule_resources: ruleResources }; // Patch web_accessible_resources key @@ -1312,13 +1312,8 @@ async function main() { } manifest.web_accessible_resources = [ web_accessible_resources ]; - // Patch version key - const now = new Date(); - const yearPart = now.getUTCFullYear() - 2000; - const monthPart = (now.getUTCMonth() + 1) * 1000; - const dayPart = now.getUTCDate() * 10; - const hourPart = Math.floor(now.getUTCHours() / 3) + 1; - manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`; + // Patch manifest version property + manifest.version = version; // Commit changes await fs.writeFile( `${outputDir}/manifest.json`, diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index df3126c71..ec6ba4562 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -134,7 +134,54 @@ function addExtendedToDNR(context, parser) { } // Response header filtering - if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { + if ( parser.isResponseheaderFilter() ) { + if ( parser.hasError() ) { return; } + if ( parser.hasOptions() === false ) { return; } + if ( parser.isException() ) { return; } + const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER); + if ( node === 0 ) { return; } + const header = parser.getNodeString(node); + if ( context.responseHeaderRules === undefined ) { + context.responseHeaderRules = []; + } + const rule = { + action: { + responseHeaders: [ + { + header, + operation: 'remove', + } + ], + type: 'modifyHeaders' + }, + condition: { + resourceTypes: [ + 'main_frame', + 'sub_frame' + ] + }, + }; + for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) { + if ( bad ) { continue; } + if ( not ) { + if ( rule.condition.excludedInitiatorDomains === undefined ) { + rule.condition.excludedInitiatorDomains = []; + } + rule.condition.excludedInitiatorDomains.push(hn); + continue; + } + if ( hn === '*' ) { + if ( rule.condition.initiatorDomains !== undefined ) { + rule.condition.initiatorDomains = undefined; + } + continue; + } + if ( rule.condition.initiatorDomains === undefined ) { + rule.condition.initiatorDomains = []; + } + rule.condition.initiatorDomains.push(hn); + } + context.responseHeaderRules.push(rule); return; } @@ -286,6 +333,129 @@ function addToDNR(context, list) { /******************************************************************************/ +function finalizeRuleset(context, network) { + const ruleset = network.ruleset; + + // Assign rule ids + const rulesetMap = new Map(); + { + let ruleId = 1; + for ( const rule of ruleset ) { + rulesetMap.set(ruleId++, rule); + } + } + // Merge rules where possible by merging arrays of a specific property. + // + // https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579 + // Do not merge rules which have errors. + const mergeRules = (rulesetMap, mergeTarget) => { + const mergeMap = new Map(); + const sorter = (_, v) => { + if ( Array.isArray(v) ) { + return typeof v[0] === 'string' ? v.sort() : v; + } + if ( v instanceof Object ) { + const sorted = {}; + for ( const kk of Object.keys(v).sort() ) { + sorted[kk] = v[kk]; + } + return sorted; + } + return v; + }; + const ruleHasher = (rule, target) => { + return JSON.stringify(rule, (k, v) => { + if ( k.startsWith('_') ) { return; } + if ( k === target ) { return; } + return sorter(k, v); + }); + }; + const extractTargetValue = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return v; } + if ( v instanceof Object ) { + const r = extractTargetValue(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + const extractTargetOwner = (obj, target) => { + for ( const [ k, v ] of Object.entries(obj) ) { + if ( Array.isArray(v) && k === target ) { return obj; } + if ( v instanceof Object ) { + const r = extractTargetOwner(v, target); + if ( r !== undefined ) { return r; } + } + } + }; + for ( const [ id, rule ] of rulesetMap ) { + if ( rule._error !== undefined ) { continue; } + const hash = ruleHasher(rule, mergeTarget); + if ( mergeMap.has(hash) === false ) { + mergeMap.set(hash, []); + } + mergeMap.get(hash).push(id); + } + for ( const ids of mergeMap.values() ) { + if ( ids.length === 1 ) { continue; } + const leftHand = rulesetMap.get(ids[0]); + const leftHandSet = new Set( + extractTargetValue(leftHand, mergeTarget) || [] + ); + for ( let i = 1; i < ids.length; i++ ) { + const rightHandId = ids[i]; + const rightHand = rulesetMap.get(rightHandId); + const rightHandArray = extractTargetValue(rightHand, mergeTarget); + if ( rightHandArray !== undefined ) { + if ( leftHandSet.size !== 0 ) { + for ( const item of rightHandArray ) { + leftHandSet.add(item); + } + } + } else { + leftHandSet.clear(); + } + rulesetMap.delete(rightHandId); + } + const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); + if ( leftHandSet.size > 1 ) { + //if ( leftHandOwner === undefined ) { debugger; } + leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); + } else if ( leftHandSet.size === 0 ) { + if ( leftHandOwner !== undefined ) { + leftHandOwner[mergeTarget] = undefined; + } + } + } + }; + mergeRules(rulesetMap, 'resourceTypes'); + mergeRules(rulesetMap, 'initiatorDomains'); + mergeRules(rulesetMap, 'requestDomains'); + mergeRules(rulesetMap, 'removeParams'); + mergeRules(rulesetMap, 'responseHeaders'); + + // Patch id + const rulesetFinal = []; + { + let ruleId = 1; + for ( const rule of rulesetMap.values() ) { + if ( rule._error === undefined ) { + rule.id = ruleId++; + } else { + rule.id = 0; + } + rulesetFinal.push(rule); + } + for ( const invalid of context.invalid ) { + rulesetFinal.push({ _error: [ invalid ] }); + } + } + + network.ruleset = rulesetFinal; +} + +/******************************************************************************/ + async function dnrRulesetFromRawLists(lists, options = {}) { const context = Object.assign({}, options); staticNetFilteringEngine.dnrFromCompiled('begin', context); @@ -300,8 +470,7 @@ async function dnrRulesetFromRawLists(lists, options = {}) { } } await Promise.all(toLoad); - - return { + const result = { network: staticNetFilteringEngine.dnrFromCompiled('end', context), genericCosmetic: context.genericCosmeticFilters, genericHighCosmetic: context.genericHighCosmeticFilters, @@ -309,6 +478,11 @@ async function dnrRulesetFromRawLists(lists, options = {}) { specificCosmetic: context.specificCosmeticFilters, scriptlet: context.scriptletFilters, }; + if ( context.responseHeaderRules ) { + result.network.ruleset.push(...context.responseHeaderRules); + } + finalizeRuleset(context, result.network); + return result; } /******************************************************************************/ diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index ab14f3829..e7035daea 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -1274,7 +1274,9 @@ class FilterRegex { dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`); } rule.condition.regexFilter = args[1]; - rule.condition.isUrlFilterCaseSensitive = args[2] === 1; + if ( args[2] === 1 ) { + rule.condition.isUrlFilterCaseSensitive = true; + } } static keyFromArgs(args) { @@ -4349,7 +4351,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { 'ping', 'other', ]); - let ruleset = []; + const ruleset = []; for ( const [ realmBits, realmName ] of realms ) { for ( const [ partyBits, partyName ] of partyness ) { for ( const typeName in typeNameToTypeValue ) { @@ -4521,141 +4523,8 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { } } - // Assign rule ids - const rulesetMap = new Map(); - { - let ruleId = 1; - for ( const rule of ruleset ) { - rulesetMap.set(ruleId++, rule); - } - } - - // Merge rules where possible by merging arrays of a specific property. - // - // https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579 - // Do not merge rules which have errors. - const mergeRules = (rulesetMap, mergeTarget) => { - const mergeMap = new Map(); - const sorter = (_, v) => { - if ( Array.isArray(v) ) { - return typeof v[0] === 'string' ? v.sort() : v; - } - if ( v instanceof Object ) { - const sorted = {}; - for ( const kk of Object.keys(v).sort() ) { - sorted[kk] = v[kk]; - } - return sorted; - } - return v; - }; - const ruleHasher = (rule, target) => { - return JSON.stringify(rule, (k, v) => { - if ( k.startsWith('_') ) { return; } - if ( k === target ) { return; } - return sorter(k, v); - }); - }; - const extractTargetValue = (obj, target) => { - for ( const [ k, v ] of Object.entries(obj) ) { - if ( Array.isArray(v) && k === target ) { return v; } - if ( v instanceof Object ) { - const r = extractTargetValue(v, target); - if ( r !== undefined ) { return r; } - } - } - }; - const extractTargetOwner = (obj, target) => { - for ( const [ k, v ] of Object.entries(obj) ) { - if ( Array.isArray(v) && k === target ) { return obj; } - if ( v instanceof Object ) { - const r = extractTargetOwner(v, target); - if ( r !== undefined ) { return r; } - } - } - }; - for ( const [ id, rule ] of rulesetMap ) { - if ( rule._error !== undefined ) { continue; } - const hash = ruleHasher(rule, mergeTarget); - if ( mergeMap.has(hash) === false ) { - mergeMap.set(hash, []); - } - mergeMap.get(hash).push(id); - } - for ( const ids of mergeMap.values() ) { - if ( ids.length === 1 ) { continue; } - const leftHand = rulesetMap.get(ids[0]); - const leftHandSet = new Set( - extractTargetValue(leftHand, mergeTarget) || [] - ); - for ( let i = 1; i < ids.length; i++ ) { - const rightHandId = ids[i]; - const rightHand = rulesetMap.get(rightHandId); - const rightHandArray = extractTargetValue(rightHand, mergeTarget); - if ( rightHandArray !== undefined ) { - if ( leftHandSet.size !== 0 ) { - for ( const item of rightHandArray ) { - leftHandSet.add(item); - } - } - } else { - leftHandSet.clear(); - } - rulesetMap.delete(rightHandId); - } - const leftHandOwner = extractTargetOwner(leftHand, mergeTarget); - if ( leftHandSet.size > 1 ) { - //if ( leftHandOwner === undefined ) { debugger; } - leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort(); - } else if ( leftHandSet.size === 0 ) { - if ( leftHandOwner !== undefined ) { - leftHandOwner[mergeTarget] = undefined; - } - } - } - }; - mergeRules(rulesetMap, 'resourceTypes'); - mergeRules(rulesetMap, 'initiatorDomains'); - mergeRules(rulesetMap, 'requestDomains'); - mergeRules(rulesetMap, 'removeParams'); - mergeRules(rulesetMap, 'responseHeaders'); - - // Patch case-sensitiveness - for ( const rule of rulesetMap.values() ) { - const { condition } = rule; - if ( - condition === undefined || - condition.urlFilter === undefined && - condition.regexFilter === undefined - ) { - continue; - } - if ( condition.isUrlFilterCaseSensitive === undefined ) { - condition.isUrlFilterCaseSensitive = false; - } else if ( condition.isUrlFilterCaseSensitive === true ) { - condition.isUrlFilterCaseSensitive = undefined; - } - } - - // Patch id - const rulesetFinal = []; - { - let ruleId = 1; - for ( const rule of rulesetMap.values() ) { - if ( rule._error === undefined ) { - rule.id = ruleId++; - } else { - rule.id = 0; - } - rulesetFinal.push(rule); - } - for ( const invalid of context.invalid ) { - rulesetFinal.push({ _error: [ invalid ] }); - } - } - return { - ruleset: rulesetFinal, + ruleset, filterCount: context.filterCount, acceptedFilterCount: context.acceptedFilterCount, rejectedFilterCount: context.rejectedFilterCount,