mirror of
https://github.com/gorhill/uBlock.git
synced 2024-10-02 23:57:10 +02:00
[mv3] Add support for removeparam= filter option
Consequently, AdGuard URL Tracking Protection (AUTP) has been added to the set of available filter lists. However, removeparam= equivalent DNR rules can only be enforced when granting uBOL broad permissions. If broad permissions are not granted, removeparam= equivalent DNR rules are ignored. Exception removeparam= filters are not supported, and these are present in AUTP and meant to unbreak some websites which are known to break as a result of removing query parameters. This is issue might be mitigated in the future by making the conversion from filters to DNR rules more complicated but this can never replace the accuracy of uBO's filtering engine being able to fully enforce arbitrary exception removeparam= filters. Also, it is not possible to translate regex-based removeparam= values to DNR rules, so these are dropped at conversion time. As with other filters to DNR rules conversion, the converter coallesce many distinct removeparam= filters into fewer DNR rules.
This commit is contained in:
Raymond Hill
parent
8febe2c95c
commit
28aee736a5
@ -123,12 +123,13 @@ body.needSave #revertRules {
|
||||
#rulesetStats .rulesetDetails h1 {
|
||||
font-size: 1em;
|
||||
font-weight: normal;
|
||||
margin: 0.5em 0;
|
||||
margin: 0.5em 0 0.25em 0;
|
||||
text-transform: capitalize;
|
||||
}
|
||||
#rulesetStats .rulesetDetails p {
|
||||
color: var(--ink-2);
|
||||
font-size: var(--font-size-smaller);
|
||||
margin: 0.5em 0 0.5em var(--popup-gap-thin);
|
||||
margin: 0.25em 0 0.5em 0.5em;
|
||||
}
|
||||
|
||||
.itemRibbon {
|
||||
|
||||
@ -30,19 +30,35 @@ import { simpleStorage } from './storage.js';
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const rulesetMap = new Map();
|
||||
let cachedRulesetData = {};
|
||||
let filteringSettingsHash = '';
|
||||
let hideUnusedSet = new Set([ 'regions' ]);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const renderNumber = function(value) {
|
||||
function renderNumber(value) {
|
||||
return value.toLocaleString();
|
||||
};
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
function renderFilterLists(soft) {
|
||||
function rulesetStats(rulesetId) {
|
||||
const canRemoveParams = cachedRulesetData.hasOmnipotence;
|
||||
const rulesetDetails = rulesetMap.get(rulesetId);
|
||||
if ( rulesetDetails === undefined ) { return; }
|
||||
const { rules, filters } = rulesetDetails;
|
||||
let ruleCount = rules.plain + rules.regexes;
|
||||
if ( canRemoveParams ) {
|
||||
ruleCount += rules.removeparams;
|
||||
}
|
||||
const filterCount = filters.accepted;
|
||||
return { ruleCount, filterCount };
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
function renderFilterLists(soft = false) {
|
||||
const { enabledRulesets, rulesetDetails } = cachedRulesetData;
|
||||
const listGroupTemplate = qs$('#templates .groupEntry');
|
||||
const listEntryTemplate = qs$('#templates .listEntry');
|
||||
@ -77,12 +93,19 @@ function renderFilterLists(soft) {
|
||||
dom.cl.toggle(li, 'unused', hideUnused && !on);
|
||||
}
|
||||
// https://github.com/gorhill/uBlock/issues/1429
|
||||
if ( !soft ) {
|
||||
if ( soft !== true ) {
|
||||
qs$('input[type="checkbox"]', li).checked = on;
|
||||
}
|
||||
const stats = rulesetStats(ruleset.id);
|
||||
li.title = listStatsTemplate
|
||||
.replace('{{ruleCount}}', renderNumber(ruleset.rules.accepted))
|
||||
.replace('{{filterCount}}', renderNumber(ruleset.filters.accepted));
|
||||
.replace('{{ruleCount}}', renderNumber(stats.ruleCount))
|
||||
.replace('{{filterCount}}', renderNumber(stats.filterCount));
|
||||
dom.attr(
|
||||
qs$('.input.checkbox', li),
|
||||
'disabled',
|
||||
stats.ruleCount === 0 ? '' : null
|
||||
);
|
||||
dom.cl.remove(li, 'discard');
|
||||
return li;
|
||||
};
|
||||
|
||||
@ -200,17 +223,14 @@ const renderWidgets = function() {
|
||||
);
|
||||
|
||||
// Compute total counts
|
||||
const rulesetMap = new Map(
|
||||
cachedRulesetData.rulesetDetails.map(rule => [ rule.id, rule ])
|
||||
);
|
||||
let filterCount = 0;
|
||||
let ruleCount = 0;
|
||||
for ( const liEntry of qsa$('#lists .listEntry[data-listkey]') ) {
|
||||
if ( qs$('input[type="checkbox"]:checked', liEntry) === null ) { continue; }
|
||||
const ruleset = rulesetMap.get(liEntry.dataset.listkey);
|
||||
if ( ruleset === undefined ) { continue; }
|
||||
filterCount += ruleset.filters.accepted;
|
||||
ruleCount += ruleset.rules.accepted;
|
||||
const stats = rulesetStats(liEntry.dataset.listkey);
|
||||
if ( stats === undefined ) { continue; }
|
||||
ruleCount += stats.ruleCount;
|
||||
filterCount += stats.filterCount;
|
||||
}
|
||||
qs$('#listsOfBlockedHostsPrompt').textContent = i18n$('perRulesetStats')
|
||||
.replace('{{ruleCount}}', ruleCount.toLocaleString())
|
||||
@ -239,7 +259,10 @@ async function onOmnipotenceChanged(ev) {
|
||||
}) !== true;
|
||||
}
|
||||
|
||||
cachedRulesetData.hasOmnipotence = actualState;
|
||||
qs$('#omnipotenceWidget input').checked = actualState;
|
||||
renderFilterLists(true);
|
||||
renderWidgets();
|
||||
}
|
||||
|
||||
dom.on(
|
||||
@ -385,6 +408,8 @@ sendMessage({
|
||||
}).then(data => {
|
||||
if ( !data ) { return; }
|
||||
cachedRulesetData = data;
|
||||
rulesetMap.clear();
|
||||
cachedRulesetData.rulesetDetails.forEach(rule => rulesetMap.set(rule.id, rule));
|
||||
try {
|
||||
renderFilterLists();
|
||||
} catch(ex) {
|
||||
|
||||
@ -37,8 +37,8 @@ import {
|
||||
getDynamicRules,
|
||||
defaultRulesetsFromLanguage,
|
||||
enableRulesets,
|
||||
getEnabledRulesetsStats,
|
||||
updateRegexRules,
|
||||
getEnabledRulesetsDetails,
|
||||
updateDynamicRules,
|
||||
} from './ruleset-manager.js';
|
||||
|
||||
import {
|
||||
@ -113,7 +113,8 @@ async function saveRulesetConfig() {
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
function hasGreatPowers(origin) {
|
||||
async function hasGreatPowers(origin) {
|
||||
if ( /^https?:\/\//.test(origin) === false ) { return false; }
|
||||
return browser.permissions.contains({
|
||||
origins: [ `${origin}/*` ],
|
||||
});
|
||||
@ -126,10 +127,16 @@ function hasOmnipotence() {
|
||||
}
|
||||
|
||||
function onPermissionsAdded(permissions) {
|
||||
if ( permissions.origins?.includes('<all_urls>') ) {
|
||||
updateDynamicRules();
|
||||
}
|
||||
registerInjectables(permissions.origins);
|
||||
}
|
||||
|
||||
function onPermissionsRemoved(permissions) {
|
||||
if ( permissions.origins?.includes('<all_urls>') ) {
|
||||
updateDynamicRules();
|
||||
}
|
||||
registerInjectables(permissions.origins);
|
||||
}
|
||||
|
||||
@ -170,7 +177,7 @@ function onMessage(request, sender, callback) {
|
||||
matchesTrustedSiteDirective(request),
|
||||
hasOmnipotence(),
|
||||
hasGreatPowers(request.origin),
|
||||
getEnabledRulesetsStats(),
|
||||
getEnabledRulesetsDetails(),
|
||||
getInjectableCount(request.origin),
|
||||
]).then(results => {
|
||||
callback({
|
||||
@ -208,7 +215,7 @@ async function start() {
|
||||
const currentVersion = getCurrentVersion();
|
||||
if ( currentVersion !== rulesetConfig.version ) {
|
||||
console.log(`Version change: ${rulesetConfig.version} => ${currentVersion}`);
|
||||
updateRegexRules().then(( ) => {
|
||||
updateDynamicRules().then(( ) => {
|
||||
rulesetConfig.version = currentVersion;
|
||||
saveRulesetConfig();
|
||||
});
|
||||
|
||||
@ -62,7 +62,11 @@ class dom {
|
||||
if ( value === undefined ) {
|
||||
return elem.getAttribute(attr);
|
||||
}
|
||||
elem.setAttribute(attr, value);
|
||||
if ( value === null ) {
|
||||
elem.removeAttribute(attr);
|
||||
} else {
|
||||
elem.setAttribute(attr, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -237,10 +237,14 @@ async function init() {
|
||||
const div = qs$('#templates .rulesetDetails').cloneNode(true);
|
||||
dom.text(qs$('h1', div), details.name);
|
||||
const { rules, filters, css } = details;
|
||||
let ruleCount = rules.plain + rules.regexes;
|
||||
if ( popupPanelData.hasOmnipotence ) {
|
||||
ruleCount += rules.removeparams;
|
||||
}
|
||||
dom.text(
|
||||
qs$('p', div),
|
||||
i18n$('perRulesetStats')
|
||||
.replace('{{ruleCount}}', rules.accepted.toLocaleString())
|
||||
.replace('{{ruleCount}}', ruleCount.toLocaleString())
|
||||
.replace('{{filterCount}}', filters.accepted.toLocaleString())
|
||||
.replace('{{cssSpecificCount}}', css.specific.toLocaleString())
|
||||
);
|
||||
|
||||
@ -25,7 +25,7 @@
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
import { dnr, i18n } from './ext.js';
|
||||
import { browser, dnr, i18n } from './ext.js';
|
||||
import { fetchJSON } from './fetch.js';
|
||||
|
||||
/******************************************************************************/
|
||||
@ -33,6 +33,8 @@ import { fetchJSON } from './fetch.js';
|
||||
const RULE_REALM_SIZE = 1000000;
|
||||
const REGEXES_REALM_START = 1000000;
|
||||
const REGEXES_REALM_END = REGEXES_REALM_START + RULE_REALM_SIZE;
|
||||
const REMOVEPARAMS_REALM_START = 2000000;
|
||||
const REMOVEPARAMS_REALM_END = REMOVEPARAMS_REALM_START + RULE_REALM_SIZE;
|
||||
const TRUSTED_DIRECTIVE_BASE_RULE_ID = 8000000;
|
||||
const CURRENT_CONFIG_BASE_RULE_ID = 9000000;
|
||||
|
||||
@ -79,7 +81,7 @@ async function updateRegexRules() {
|
||||
rulesetDetails,
|
||||
dynamicRules
|
||||
] = await Promise.all([
|
||||
getRulesetDetails(),
|
||||
getEnabledRulesetsDetails(),
|
||||
dnr.getDynamicRules(),
|
||||
]);
|
||||
|
||||
@ -96,8 +98,7 @@ async function updateRegexRules() {
|
||||
|
||||
// Fetch regexes for all enabled rulesets
|
||||
const toFetch = [];
|
||||
for ( const details of rulesetDetails.values() ) {
|
||||
if ( details.enabled !== true ) { continue; }
|
||||
for ( const details of rulesetDetails ) {
|
||||
if ( details.rules.regexes === 0 ) { continue; }
|
||||
toFetch.push(fetchJSON(`/rulesets/${details.id}.regexes`));
|
||||
}
|
||||
@ -131,7 +132,7 @@ async function updateRegexRules() {
|
||||
if ( result instanceof Object && result.isSupported ) {
|
||||
newRules.push(rule);
|
||||
} else {
|
||||
console.info(`${result.reason}: ${rule.condition.regexFilter}`);
|
||||
//console.info(`${result.reason}: ${rule.condition.regexFilter}`);
|
||||
}
|
||||
}
|
||||
console.info(
|
||||
@ -139,11 +140,12 @@ async function updateRegexRules() {
|
||||
);
|
||||
|
||||
// Add validated regex rules to dynamic ruleset without affecting rules
|
||||
// outside regex rule realm.
|
||||
// outside regex rules realm.
|
||||
const dynamicRuleMap = await getDynamicRules();
|
||||
const newRuleMap = new Map(newRules.map(rule => [ rule.id, rule ]));
|
||||
const addRules = [];
|
||||
const removeRuleIds = [];
|
||||
|
||||
for ( const oldRule of dynamicRuleMap.values() ) {
|
||||
if ( oldRule.id < REGEXES_REALM_START ) { continue; }
|
||||
if ( oldRule.id >= REGEXES_REALM_END ) { continue; }
|
||||
@ -157,14 +159,104 @@ async function updateRegexRules() {
|
||||
dynamicRuleMap.set(oldRule.id, newRule);
|
||||
}
|
||||
}
|
||||
|
||||
for ( const newRule of newRuleMap.values() ) {
|
||||
if ( dynamicRuleMap.has(newRule.id) ) { continue; }
|
||||
addRules.push(newRule);
|
||||
dynamicRuleMap.set(newRule.id, newRule);
|
||||
}
|
||||
if ( addRules.length !== 0 || removeRuleIds.length !== 0 ) {
|
||||
return dnr.updateDynamicRules({ addRules, removeRuleIds });
|
||||
|
||||
if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; }
|
||||
|
||||
if ( removeRuleIds.length !== 0 ) {
|
||||
console.info(`Remove ${removeRuleIds.length} DNR regex rules`);
|
||||
}
|
||||
if ( addRules.length !== 0 ) {
|
||||
console.info(`Add ${addRules.length} DNR regex rules`);
|
||||
}
|
||||
|
||||
return dnr.updateDynamicRules({ addRules, removeRuleIds });
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
async function updateRemoveparamRules() {
|
||||
const [
|
||||
hasOmnipotence,
|
||||
rulesetDetails,
|
||||
dynamicRuleMap,
|
||||
] = await Promise.all([
|
||||
browser.permissions.contains({ origins: [ '<all_urls>' ] }),
|
||||
getEnabledRulesetsDetails(),
|
||||
getDynamicRules(),
|
||||
]);
|
||||
|
||||
// Fetch removeparam rules for all enabled rulesets
|
||||
const toFetch = [];
|
||||
for ( const details of rulesetDetails ) {
|
||||
if ( details.rules.removeparams === 0 ) { continue; }
|
||||
toFetch.push(fetchJSON(`/rulesets/${details.id}.removeparams`));
|
||||
}
|
||||
const removeparamRulesets = await Promise.all(toFetch);
|
||||
|
||||
// Removeparam rules can only be enforced with omnipotence
|
||||
const newRules = [];
|
||||
if ( hasOmnipotence ) {
|
||||
let removeparamRuleId = REMOVEPARAMS_REALM_START;
|
||||
for ( const rules of removeparamRulesets ) {
|
||||
if ( Array.isArray(rules) === false ) { continue; }
|
||||
for ( const rule of rules ) {
|
||||
rule.id = removeparamRuleId++;
|
||||
newRules.push(rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add removeparam rules to dynamic ruleset without affecting rules
|
||||
// outside removeparam rules realm.
|
||||
const newRuleMap = new Map(newRules.map(rule => [ rule.id, rule ]));
|
||||
const addRules = [];
|
||||
const removeRuleIds = [];
|
||||
|
||||
for ( const oldRule of dynamicRuleMap.values() ) {
|
||||
if ( oldRule.id < REMOVEPARAMS_REALM_START ) { continue; }
|
||||
if ( oldRule.id >= REMOVEPARAMS_REALM_END ) { continue; }
|
||||
const newRule = newRuleMap.get(oldRule.id);
|
||||
if ( newRule === undefined ) {
|
||||
removeRuleIds.push(oldRule.id);
|
||||
dynamicRuleMap.delete(oldRule.id);
|
||||
} else if ( JSON.stringify(oldRule) !== JSON.stringify(newRule) ) {
|
||||
removeRuleIds.push(oldRule.id);
|
||||
addRules.push(newRule);
|
||||
dynamicRuleMap.set(oldRule.id, newRule);
|
||||
}
|
||||
}
|
||||
|
||||
for ( const newRule of newRuleMap.values() ) {
|
||||
if ( dynamicRuleMap.has(newRule.id) ) { continue; }
|
||||
addRules.push(newRule);
|
||||
dynamicRuleMap.set(newRule.id, newRule);
|
||||
}
|
||||
|
||||
if ( addRules.length === 0 && removeRuleIds.length === 0 ) { return; }
|
||||
|
||||
if ( removeRuleIds.length !== 0 ) {
|
||||
console.info(`Remove ${removeRuleIds.length} DNR removeparam rules`);
|
||||
}
|
||||
if ( addRules.length !== 0 ) {
|
||||
console.info(`Add ${addRules.length} DNR removeparam rules`);
|
||||
}
|
||||
|
||||
return dnr.updateDynamicRules({ addRules, removeRuleIds });
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
async function updateDynamicRules() {
|
||||
return Promise.all([
|
||||
updateRegexRules(),
|
||||
updateRemoveparamRules(),
|
||||
]);
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
@ -240,18 +332,23 @@ async function enableRulesets(ids) {
|
||||
if ( disableRulesetIds.length !== 0 ) {
|
||||
console.info(`Disable ruleset: ${disableRulesetIds}`);
|
||||
}
|
||||
return dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds });
|
||||
await dnr.updateEnabledRulesets({ enableRulesetIds, disableRulesetIds });
|
||||
|
||||
return Promise.all([
|
||||
updateRegexRules(),
|
||||
updateRemoveparamRules(),
|
||||
]);
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
async function getEnabledRulesetsStats() {
|
||||
async function getEnabledRulesetsDetails() {
|
||||
const [
|
||||
rulesetDetails,
|
||||
ids,
|
||||
rulesetDetails,
|
||||
] = await Promise.all([
|
||||
getRulesetDetails(),
|
||||
dnr.getEnabledRulesets(),
|
||||
getRulesetDetails(),
|
||||
]);
|
||||
const out = [];
|
||||
for ( const id of ids ) {
|
||||
@ -265,14 +362,12 @@ async function getEnabledRulesetsStats() {
|
||||
/******************************************************************************/
|
||||
|
||||
export {
|
||||
REGEXES_REALM_START,
|
||||
REGEXES_REALM_END,
|
||||
TRUSTED_DIRECTIVE_BASE_RULE_ID,
|
||||
CURRENT_CONFIG_BASE_RULE_ID,
|
||||
getRulesetDetails,
|
||||
getDynamicRules,
|
||||
enableRulesets,
|
||||
defaultRulesetsFromLanguage,
|
||||
getEnabledRulesetsStats,
|
||||
updateRegexRules,
|
||||
getEnabledRulesetsDetails,
|
||||
updateDynamicRules,
|
||||
};
|
||||
|
||||
@ -40,10 +40,9 @@ function getScriptingDetails() {
|
||||
return scriptingDetailsPromise;
|
||||
}
|
||||
scriptingDetailsPromise = fetchJSON('/rulesets/scripting-details').then(entries => {
|
||||
const out = new Map(entries);
|
||||
for ( const details of out.values() ) {
|
||||
details.matches = new Map(details.matches);
|
||||
details.excludeMatches = new Map(details.excludeMatches);
|
||||
const out = new Map();
|
||||
for ( const entry of entries ) {
|
||||
out.set(entry[0], new Map(entry[1]));
|
||||
}
|
||||
return out;
|
||||
});
|
||||
@ -52,32 +51,19 @@ function getScriptingDetails() {
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const arrayEq = (a, b) => {
|
||||
if ( a === undefined ) { return b === undefined; }
|
||||
if ( b === undefined ) { return false; }
|
||||
if ( a.length !== b.length ) { return false; }
|
||||
for ( const i of a ) {
|
||||
if ( b.includes(i) === false ) { return false; }
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const toRegisterable = (fname, entry) => {
|
||||
const toRegisterable = (fname, hostnames) => {
|
||||
const directive = {
|
||||
id: fname,
|
||||
allFrames: true,
|
||||
matchOriginAsFallback: true,
|
||||
};
|
||||
if ( entry.matches ) {
|
||||
directive.matches = ut.matchesFromHostnames(entry.matches);
|
||||
if ( hostnames ) {
|
||||
directive.matches = ut.matchesFromHostnames(hostnames);
|
||||
} else {
|
||||
directive.matches = [ '<all_urls>' ];
|
||||
}
|
||||
if ( entry.excludeMatches ) {
|
||||
directive.excludeMatches = ut.matchesFromHostnames(entry.excludeMatches);
|
||||
}
|
||||
directive.js = [ `/rulesets/js/${fname.slice(0,2)}/${fname.slice(2)}.js` ];
|
||||
if ( (ut.fidFromFileName(fname) & RUN_AT_BIT) !== 0 ) {
|
||||
if ( (ut.fidFromFileName(fname) & RUN_AT_END_BIT) !== 0 ) {
|
||||
directive.runAt = 'document_end';
|
||||
} else {
|
||||
directive.runAt = 'document_start';
|
||||
@ -88,23 +74,28 @@ const toRegisterable = (fname, entry) => {
|
||||
return directive;
|
||||
};
|
||||
|
||||
const RUN_AT_BIT = 0b10;
|
||||
const RUN_AT_END_BIT = 0b10;
|
||||
const MAIN_WORLD_BIT = 0b01;
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const shouldUpdate = (registered, candidate) => {
|
||||
const matches = candidate.matches &&
|
||||
ut.matchesFromHostnames(candidate.matches);
|
||||
if ( arrayEq(registered.matches, matches) === false ) {
|
||||
return true;
|
||||
// Important: We need to sort the arrays for fast comparison
|
||||
const arrayEq = (a, b) => {
|
||||
if ( a === undefined ) { return b === undefined; }
|
||||
if ( b === undefined ) { return false; }
|
||||
const alen = a.length;
|
||||
if ( alen !== b.length ) { return false; }
|
||||
a.sort(); b.sort();
|
||||
for ( let i = 0; i < alen; i++ ) {
|
||||
if ( a[i] !== b[i] ) { return false; }
|
||||
}
|
||||
const excludeMatches = candidate.excludeMatches &&
|
||||
ut.matchesFromHostnames(candidate.excludeMatches);
|
||||
if ( arrayEq(registered.excludeMatches, excludeMatches) === false ) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
const shouldUpdate = (registered, candidateHostnames) => {
|
||||
const registeredHostnames = registered.matches &&
|
||||
ut.hostnamesFromMatches(registered.matches);
|
||||
return arrayEq(registeredHostnames, candidateHostnames) === false;
|
||||
};
|
||||
|
||||
const isTrustedHostname = (trustedSites, hn) => {
|
||||
@ -133,11 +124,11 @@ async function getInjectableCount(origin) {
|
||||
let total = 0;
|
||||
|
||||
for ( const rulesetId of rulesetIds ) {
|
||||
if ( scriptingDetails.has(rulesetId) === false ) { continue; }
|
||||
const details = scriptingDetails.get(rulesetId);
|
||||
const hostnamesToFidsMap = scriptingDetails.get(rulesetId);
|
||||
if ( hostnamesToFidsMap === undefined ) { continue; }
|
||||
let hn = url.hostname;
|
||||
while ( hn !== '' ) {
|
||||
const fids = details.matches?.get(hn);
|
||||
const fids = hostnamesToFidsMap.get(hn);
|
||||
if ( typeof fids === 'number' ) {
|
||||
total += 1;
|
||||
} else if ( Array.isArray(fids) ) {
|
||||
@ -162,28 +153,33 @@ function registerSomeInjectables(args) {
|
||||
|
||||
const toRegisterMap = new Map();
|
||||
|
||||
const checkMatches = (details, hn) => {
|
||||
let fids = details.matches?.get(hn);
|
||||
const checkMatches = (hostnamesToFidsMap, hn) => {
|
||||
let fids = hostnamesToFidsMap.get(hn);
|
||||
if ( fids === undefined ) { return; }
|
||||
if ( typeof fids === 'number' ) { fids = [ fids ]; }
|
||||
for ( const fid of fids ) {
|
||||
const fname = ut.fnameFromFileId(fid);
|
||||
const existing = toRegisterMap.get(fname);
|
||||
let existing = toRegisterMap.get(fname);
|
||||
if ( existing ) {
|
||||
existing.matches.push(hn);
|
||||
if ( existing[0] === '*' ) { continue; }
|
||||
existing.push(hn);
|
||||
} else {
|
||||
toRegisterMap.set(fname, { matches: [ hn ] });
|
||||
toRegisterMap.set(fname, existing = [ hn ]);
|
||||
}
|
||||
if ( hn !== '*' ) { continue; }
|
||||
existing.length = 0;
|
||||
existing.push('*');
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
for ( const rulesetId of rulesetIds ) {
|
||||
const details = scriptingDetails.get(rulesetId);
|
||||
if ( details === undefined ) { continue; }
|
||||
const hostnamesToFidsMap = scriptingDetails.get(rulesetId);
|
||||
if ( hostnamesToFidsMap === undefined ) { continue; }
|
||||
for ( let hn of hostnamesSet ) {
|
||||
if ( isTrustedHostname(trustedSites, hn) ) { continue; }
|
||||
while ( hn ) {
|
||||
checkMatches(details, hn);
|
||||
checkMatches(hostnamesToFidsMap, hn);
|
||||
hn = ut.toBroaderHostname(hn);
|
||||
}
|
||||
}
|
||||
@ -202,19 +198,24 @@ function registerAllInjectables(args) {
|
||||
const toRegisterMap = new Map();
|
||||
|
||||
for ( const rulesetId of rulesetIds ) {
|
||||
const details = scriptingDetails.get(rulesetId);
|
||||
if ( details === undefined ) { continue; }
|
||||
for ( let [ hn, fids ] of details.matches ) {
|
||||
const hostnamesToFidsMap = scriptingDetails.get(rulesetId);
|
||||
if ( hostnamesToFidsMap === undefined ) { continue; }
|
||||
for ( let [ hn, fids ] of hostnamesToFidsMap ) {
|
||||
if ( isTrustedHostname(trustedSites, hn) ) { continue; }
|
||||
if ( typeof fids === 'number' ) { fids = [ fids ]; }
|
||||
for ( const fid of fids ) {
|
||||
const fname = ut.fnameFromFileId(fid);
|
||||
const existing = toRegisterMap.get(fname);
|
||||
let existing = toRegisterMap.get(fname);
|
||||
if ( existing ) {
|
||||
existing.matches.push(hn);
|
||||
if ( existing[0] === '*' ) { continue; }
|
||||
existing.push(hn);
|
||||
} else {
|
||||
toRegisterMap.set(fname, { matches: [ hn ] });
|
||||
toRegisterMap.set(fname, existing = [ hn ]);
|
||||
}
|
||||
if ( hn !== '*' ) { continue; }
|
||||
existing.length = 0;
|
||||
existing.push('*');
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -264,13 +265,13 @@ async function registerInjectables(origins) {
|
||||
|
||||
const toAdd = [];
|
||||
const toUpdate = [];
|
||||
for ( const [ fname, entry ] of toRegisterMap ) {
|
||||
for ( const [ fname, hostnames ] of toRegisterMap ) {
|
||||
if ( before.has(fname) === false ) {
|
||||
toAdd.push(toRegisterable(fname, entry));
|
||||
toAdd.push(toRegisterable(fname, hostnames));
|
||||
continue;
|
||||
}
|
||||
if ( shouldUpdate(before.get(fname), entry) ) {
|
||||
toUpdate.push(toRegisterable(fname, entry));
|
||||
if ( shouldUpdate(before.get(fname), hostnames) ) {
|
||||
toUpdate.push(toRegisterable(fname, hostnames));
|
||||
}
|
||||
}
|
||||
|
||||
@ -282,16 +283,25 @@ async function registerInjectables(origins) {
|
||||
|
||||
const todo = [];
|
||||
if ( toRemove.length !== 0 ) {
|
||||
todo.push(browser.scripting.unregisterContentScripts({ ids: toRemove }));
|
||||
console.info(`Unregistered ${toRemove} content (css/js)`);
|
||||
todo.push(
|
||||
browser.scripting.unregisterContentScripts({ ids: toRemove })
|
||||
.catch(reason => { console.info(reason); })
|
||||
);
|
||||
}
|
||||
if ( toAdd.length !== 0 ) {
|
||||
todo.push(browser.scripting.registerContentScripts(toAdd));
|
||||
console.info(`Registered ${toAdd.map(v => v.id)} content (css/js)`);
|
||||
todo.push(
|
||||
browser.scripting.registerContentScripts(toAdd)
|
||||
.catch(reason => { console.info(reason); })
|
||||
);
|
||||
}
|
||||
if ( toUpdate.length !== 0 ) {
|
||||
todo.push(browser.scripting.updateContentScripts(toUpdate));
|
||||
console.info(`Updated ${toUpdate.map(v => v.id)} content (css/js)`);
|
||||
todo.push(
|
||||
browser.scripting.updateContentScripts(toUpdate)
|
||||
.catch(reason => { console.info(reason); })
|
||||
);
|
||||
}
|
||||
if ( todo.length === 0 ) { return; }
|
||||
|
||||
|
||||
@ -46,10 +46,11 @@ const matchesFromHostnames = hostnames => {
|
||||
const out = [];
|
||||
for ( const hn of hostnames ) {
|
||||
if ( hn === '*' ) {
|
||||
out.length = 0;
|
||||
out.push('<all_urls>');
|
||||
} else {
|
||||
out.push(`*://*.${hn}/*`);
|
||||
break;
|
||||
}
|
||||
out.push(`*://*.${hn}/*`);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
@ -58,8 +59,9 @@ const hostnamesFromMatches = origins => {
|
||||
const out = [];
|
||||
for ( const origin of origins ) {
|
||||
if ( origin === '<all_urls>' ) {
|
||||
out.length = 0;
|
||||
out.push('*');
|
||||
continue;
|
||||
break;
|
||||
}
|
||||
const match = /^\*:\/\/(?:\*\.)?([^\/]+)\/\*/.exec(origin);
|
||||
if ( match === null ) { continue; }
|
||||
|
||||
@ -72,6 +72,11 @@ const uidint32 = (s) => {
|
||||
return parseInt(h,16) & 0x7FFFFFFF;
|
||||
};
|
||||
|
||||
const hnSort = (a, b) =>
|
||||
a.split('.').reverse().join('.').localeCompare(
|
||||
b.split('.').reverse().join('.')
|
||||
);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
const stdOutput = [];
|
||||
@ -239,8 +244,8 @@ async function processNetworkFilters(assetDetails, network) {
|
||||
log(`\tRejected filter count: ${network.rejectedFilterCount}`);
|
||||
log(`Output rule count: ${rules.length}`);
|
||||
|
||||
const good = rules.filter(rule => isGood(rule) && isRegex(rule) === false);
|
||||
log(`\tGood: ${good.length}`);
|
||||
const plainGood = rules.filter(rule => isGood(rule) && isRegex(rule) === false);
|
||||
log(`\tPlain good: ${plainGood.length}`);
|
||||
|
||||
const regexes = rules.filter(rule => isGood(rule) && isRegex(rule));
|
||||
log(`\tMaybe good (regexes): ${regexes.length}`);
|
||||
@ -257,24 +262,23 @@ async function processNetworkFilters(assetDetails, network) {
|
||||
);
|
||||
log(`\tcsp= (discarded): ${headers.length}`);
|
||||
|
||||
const removeparams = rules.filter(rule =>
|
||||
isUnsupported(rule) === false &&
|
||||
isRemoveparam(rule)
|
||||
const removeparamsGood = rules.filter(rule =>
|
||||
isUnsupported(rule) === false && isRemoveparam(rule)
|
||||
);
|
||||
log(`\tremoveparams= (discarded): ${removeparams.length}`);
|
||||
const removeparamsBad = rules.filter(rule =>
|
||||
isUnsupported(rule) && isRemoveparam(rule)
|
||||
);
|
||||
log(`\tremoveparams= (accepted/discarded): ${removeparamsGood.length}/${removeparamsBad.length}`);
|
||||
|
||||
const bad = rules.filter(rule =>
|
||||
isUnsupported(rule)
|
||||
);
|
||||
log(`\tUnsupported: ${bad.length}`);
|
||||
log(
|
||||
bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'),
|
||||
true
|
||||
);
|
||||
log(bad.map(rule => rule._error.map(v => `\t\t${v}`)).join('\n'), true);
|
||||
|
||||
writeFile(
|
||||
`${rulesetDir}/${assetDetails.id}.json`,
|
||||
`${JSON.stringify(good, replacer)}\n`
|
||||
`${JSON.stringify(plainGood, replacer)}\n`
|
||||
);
|
||||
|
||||
if ( regexes.length !== 0 ) {
|
||||
@ -284,12 +288,20 @@ async function processNetworkFilters(assetDetails, network) {
|
||||
);
|
||||
}
|
||||
|
||||
if ( removeparamsGood.length !== 0 ) {
|
||||
writeFile(
|
||||
`${rulesetDir}/${assetDetails.id}.removeparams.json`,
|
||||
`${JSON.stringify(removeparamsGood, replacer)}\n`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
total: rules.length,
|
||||
accepted: good.length,
|
||||
discarded: redirects.length + headers.length + removeparams.length,
|
||||
plain: plainGood.length,
|
||||
discarded: redirects.length + headers.length + removeparamsBad.length,
|
||||
rejected: bad.length,
|
||||
regexes: regexes.length,
|
||||
removeparams: removeparamsGood.length,
|
||||
};
|
||||
}
|
||||
|
||||
@ -344,25 +356,22 @@ function loadAllSourceScriptlets() {
|
||||
|
||||
const globalPatchedScriptletsSet = new Set();
|
||||
|
||||
function addScriptingAPIResources(id, entry, prop, fid) {
|
||||
if ( entry[prop] === undefined ) { return; }
|
||||
for ( const hn of entry[prop] ) {
|
||||
let details = scriptingDetails.get(id);
|
||||
if ( details === undefined ) {
|
||||
details = {};
|
||||
scriptingDetails.set(id, details);
|
||||
function addScriptingAPIResources(id, hostnames, fid) {
|
||||
if ( hostnames === undefined ) { return; }
|
||||
for ( const hn of hostnames ) {
|
||||
let hostnamesToFidMap = scriptingDetails.get(id);
|
||||
if ( hostnamesToFidMap === undefined ) {
|
||||
hostnamesToFidMap = new Map();
|
||||
scriptingDetails.set(id, hostnamesToFidMap);
|
||||
}
|
||||
if ( details[prop] === undefined ) {
|
||||
details[prop] = new Map();
|
||||
}
|
||||
let fids = details[prop].get(hn);
|
||||
let fids = hostnamesToFidMap.get(hn);
|
||||
if ( fids === undefined ) {
|
||||
details[prop].set(hn, fid);
|
||||
hostnamesToFidMap.set(hn, fid);
|
||||
} else if ( fids instanceof Set ) {
|
||||
fids.add(fid);
|
||||
} else if ( fid !== fids ) {
|
||||
fids = new Set([ fids, fid ]);
|
||||
details[prop].set(hn, fids);
|
||||
hostnamesToFidMap.set(hn, fids);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -432,10 +441,6 @@ function groupCosmeticBySelectors(arrayin) {
|
||||
}
|
||||
}
|
||||
}
|
||||
const hnSort = (a, b) =>
|
||||
a.split('.').reverse().join('.').localeCompare(
|
||||
b.split('.').reverse().join('.')
|
||||
);
|
||||
const out = Array.from(contentMap).map(a => [
|
||||
a[0], {
|
||||
a: a[1].a,
|
||||
@ -530,18 +535,7 @@ async function processCosmeticFilters(assetDetails, mapin) {
|
||||
generatedFiles.push(fname);
|
||||
}
|
||||
for ( const entry of slice ) {
|
||||
addScriptingAPIResources(
|
||||
assetDetails.id,
|
||||
{ matches: entry[1].y },
|
||||
'matches',
|
||||
fid
|
||||
);
|
||||
addScriptingAPIResources(
|
||||
assetDetails.id,
|
||||
{ excludeMatches: entry[1].n },
|
||||
'excludeMatches',
|
||||
fid
|
||||
);
|
||||
addScriptingAPIResources(assetDetails.id, entry[1].y, fid);
|
||||
}
|
||||
}
|
||||
|
||||
@ -609,18 +603,7 @@ async function processProceduralCosmeticFilters(assetDetails, mapin) {
|
||||
generatedFiles.push(fname);
|
||||
}
|
||||
for ( const entry of slice ) {
|
||||
addScriptingAPIResources(
|
||||
assetDetails.id,
|
||||
{ matches: entry[1].y },
|
||||
'matches',
|
||||
fid
|
||||
);
|
||||
addScriptingAPIResources(
|
||||
assetDetails.id,
|
||||
{ excludeMatches: entry[1].n },
|
||||
'excludeMatches',
|
||||
fid
|
||||
);
|
||||
addScriptingAPIResources(assetDetails.id, entry[1].y, fid);
|
||||
}
|
||||
}
|
||||
|
||||
@ -749,18 +732,7 @@ async function processScriptletFilters(assetDetails, mapin) {
|
||||
generatedFiles.push(fname);
|
||||
}
|
||||
for ( const details of argsDetails.values() ) {
|
||||
addScriptingAPIResources(
|
||||
assetDetails.id,
|
||||
{ matches: details.y },
|
||||
'matches',
|
||||
fid
|
||||
);
|
||||
addScriptingAPIResources(
|
||||
assetDetails.id,
|
||||
{ excludeMatches: details.n },
|
||||
'excludeMatches',
|
||||
fid
|
||||
);
|
||||
addScriptingAPIResources(assetDetails.id, details.y, fid);
|
||||
}
|
||||
}
|
||||
|
||||
@ -844,10 +816,11 @@ const rulesetFromURLS = async function(assetDetails) {
|
||||
},
|
||||
rules: {
|
||||
total: netStats.total,
|
||||
accepted: netStats.accepted,
|
||||
plain: netStats.plain,
|
||||
regexes: netStats.regexes,
|
||||
removeparams: netStats.removeparams,
|
||||
discarded: netStats.discarded,
|
||||
rejected: netStats.rejected,
|
||||
regexes: netStats.regexes,
|
||||
},
|
||||
css: {
|
||||
specific: cosmeticStats,
|
||||
@ -941,7 +914,7 @@ async function main() {
|
||||
}
|
||||
|
||||
// Handpicked rulesets from assets.json
|
||||
const handpicked = [ 'block-lan', 'dpollock-0' ];
|
||||
const handpicked = [ 'block-lan', 'dpollock-0', 'adguard-spyware-url' ];
|
||||
for ( const id of handpicked ) {
|
||||
const asset = assets[id];
|
||||
if ( asset.content !== 'filters' ) { continue; }
|
||||
@ -972,6 +945,14 @@ async function main() {
|
||||
`${JSON.stringify(rulesetDetails, null, 1)}\n`
|
||||
);
|
||||
|
||||
// We sort the hostnames for convenience/performance in the extension's
|
||||
// script manager -- the scripting API does a sort() internally.
|
||||
for ( const [ rulesetId, hostnamesToFidsMap ] of scriptingDetails ) {
|
||||
scriptingDetails.set(
|
||||
rulesetId,
|
||||
Array.from(hostnamesToFidsMap).sort()
|
||||
);
|
||||
}
|
||||
writeFile(
|
||||
`${rulesetDir}/scripting-details.json`,
|
||||
`${JSON.stringify(scriptingDetails, jsonSetMapReplacer)}\n`
|
||||
|
||||
@ -633,11 +633,6 @@ const dnrAddRuleError = (rule, msg) => {
|
||||
rule._error.push(msg);
|
||||
};
|
||||
|
||||
const dnrAddRuleWarning = (rule, msg) => {
|
||||
rule._warning = rule._warning || [];
|
||||
rule._warning.push(msg);
|
||||
};
|
||||
|
||||
/*******************************************************************************
|
||||
|
||||
Filter classes
|
||||
@ -4050,7 +4045,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
|
||||
value: rule.__modifierValue,
|
||||
}];
|
||||
if ( rule.__modifierAction === AllowAction ) {
|
||||
dnrAddRuleError(rule, 'Unhandled modifier exception');
|
||||
dnrAddRuleError(rule, 'Unsupported modifier exception');
|
||||
}
|
||||
break;
|
||||
case 'redirect-rule': {
|
||||
@ -4063,7 +4058,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
|
||||
}
|
||||
const resource = context.extensionPaths.get(token);
|
||||
if ( rule.__modifierValue !== '' && resource === undefined ) {
|
||||
dnrAddRuleWarning(rule, `Unpatchable redirect filter: ${rule.__modifierValue}`);
|
||||
dnrAddRuleError(rule, `Unpatchable redirect filter: ${rule.__modifierValue}`);
|
||||
}
|
||||
const extensionPath = resource && resource.extensionPath || token;
|
||||
if ( rule.__modifierAction !== AllowAction ) {
|
||||
@ -4078,6 +4073,9 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
|
||||
}
|
||||
case 'removeparam':
|
||||
rule.action.type = 'redirect';
|
||||
if ( rule.__modifierValue === '|' ) {
|
||||
rule.__modifierValue = '';
|
||||
}
|
||||
if ( rule.__modifierValue !== '' ) {
|
||||
rule.action.redirect = {
|
||||
transform: {
|
||||
@ -4108,7 +4106,7 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
|
||||
];
|
||||
}
|
||||
if ( rule.__modifierAction === AllowAction ) {
|
||||
dnrAddRuleError(rule, 'Unhandled modifier exception');
|
||||
dnrAddRuleError(rule, 'Unsupported modifier exception');
|
||||
}
|
||||
break;
|
||||
default:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user