1
0
mirror of https://github.com/gorhill/uBlock.git synced 2024-09-29 22:27:12 +02:00
uBlock/src/js/redirect-engine.js
Raymond Hill d49a9dce66
Fix spurious rejection of some AdGuard redirect filters
Lines in AdGuard filter lists have trailing `\r`
characters, and these caused the redirect engine
compile code to reject as invalid the redirect
token.

This is trivially fixed by trimming the raw option
strings before parsing it in the redirect engine.
2020-07-13 09:33:38 -04:00

857 lines
26 KiB
JavaScript

/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2015-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
'use strict';
/******************************************************************************/
µBlock.redirectEngine = (( ) => {
/******************************************************************************/
/******************************************************************************/
// The resources referenced below are found in ./web_accessible_resources/
//
// The content of the resources which declare a `data` property will be loaded
// in memory, and converted to a suitable internal format depending on the
// type of the loaded data. The `data` property allows for manual injection
// through `+js(...)`, or for redirection to a data: URI when a redirection
// to a web accessible resource is not desirable.
const redirectableResources = new Map([
[ '1x1.gif', {
alias: '1x1-transparent.gif',
data: 'blob',
} ],
[ '2x2.png', {
alias: '2x2-transparent.png',
data: 'blob',
} ],
[ '3x2.png', {
alias: '3x2-transparent.png',
data: 'blob',
} ],
[ '32x32.png', {
alias: '32x32-transparent.png',
data: 'blob',
} ],
[ 'addthis_widget.js', {
alias: 'addthis.com/addthis_widget.js',
} ],
[ 'amazon_ads.js', {
alias: 'amazon-adsystem.com/aax2/amzn_ads.js',
} ],
[ 'amazon_apstag.js', {
} ],
[ 'ampproject_v0.js', {
alias: 'ampproject.org/v0.js',
} ],
[ 'chartbeat.js', {
alias: 'static.chartbeat.com/chartbeat.js',
} ],
[ 'disqus_embed.js', {
alias: 'disqus.com/embed.js',
} ],
[ 'disqus_forums_embed.js', {
alias: 'disqus.com/forums/*/embed.js',
} ],
[ 'doubleclick_instream_ad_status.js', {
alias: 'doubleclick.net/instream/ad_status.js',
} ],
[ 'empty', {
data: 'text', // Important!
} ],
[ 'google-analytics_analytics.js', {
alias: 'google-analytics.com/analytics.js',
} ],
[ 'google-analytics_cx_api.js', {
alias: 'google-analytics.com/cx/api.js',
} ],
[ 'google-analytics_ga.js', {
alias: 'google-analytics.com/ga.js',
} ],
[ 'google-analytics_inpage_linkid.js', {
alias: 'google-analytics.com/inpage_linkid.js',
} ],
[ 'googlesyndication_adsbygoogle.js', {
alias: 'googlesyndication.com/adsbygoogle.js',
} ],
[ 'googletagmanager_gtm.js', {
alias: 'googletagmanager.com/gtm.js',
} ],
[ 'googletagservices_gpt.js', {
alias: 'googletagservices.com/gpt.js',
} ],
[ 'hd-main.js', {
} ],
[ 'ligatus_angular-tag.js', {
alias: 'ligatus.com/*/angular-tag.js',
} ],
[ 'monkeybroker.js', {
alias: 'd3pkae9owd2lcf.cloudfront.net/mb105.js',
} ],
[ 'noeval.js', {
data: 'text',
} ],
[ 'noeval-silent.js', {
alias: 'silent-noeval.js',
data: 'text',
} ],
[ 'nobab.js', {
alias: 'bab-defuser.js',
data: 'text',
} ],
[ 'nofab.js', {
alias: 'fuckadblock.js-3.2.0',
data: 'text',
} ],
[ 'noop-0.1s.mp3', {
alias: 'noopmp3-0.1s',
data: 'blob',
} ],
[ 'noop-1s.mp4', {
alias: 'noopmp4-1s',
data: 'blob',
} ],
[ 'noop.html', {
alias: 'noopframe',
} ],
[ 'noop.js', {
alias: 'noopjs',
data: 'text',
} ],
[ 'noop.txt', {
alias: 'nooptext',
data: 'text',
} ],
[ 'outbrain-widget.js', {
alias: 'widgets.outbrain.com/outbrain.js',
} ],
[ 'popads.js', {
alias: 'popads.net.js',
data: 'text',
} ],
[ 'popads-dummy.js', {
data: 'text',
} ],
[ 'scorecardresearch_beacon.js', {
alias: 'scorecardresearch.com/beacon.js',
} ],
[ 'window.open-defuser.js', {
alias: 'nowoif.js',
data: 'text',
} ],
]);
const extToMimeMap = new Map([
[ 'gif', 'image/gif' ],
[ 'html', 'text/html' ],
[ 'js', 'application/javascript' ],
[ 'mp3', 'audio/mp3' ],
[ 'mp4', 'video/mp4' ],
[ 'png', 'image/png' ],
[ 'txt', 'text/plain' ],
]);
const typeToMimeMap = new Map([
[ 'main_frame', 'text/html' ],
[ 'other', 'text/plain' ],
[ 'script', 'application/javascript' ],
[ 'stylesheet', 'text/css' ],
[ 'sub_frame', 'text/html' ],
[ 'xmlhttprequest', 'text/plain' ],
]);
const validMimes = new Set(extToMimeMap.values());
const mimeFromName = function(name) {
const match = /\.([^.]+)$/.exec(name);
if ( match !== null ) {
return extToMimeMap.get(match[1]);
}
};
/******************************************************************************/
/******************************************************************************/
const RedirectEntry = class {
constructor() {
this.mime = '';
this.data = '';
this.warURL = undefined;
}
// Prevent redirection to web accessible resources when the request is
// of type 'xmlhttprequest', because XMLHttpRequest.responseURL would
// cause leakage of extension id. See:
// - https://stackoverflow.com/a/8056313
// - https://bugzilla.mozilla.org/show_bug.cgi?id=998076
// https://www.reddit.com/r/uBlockOrigin/comments/cpxm1v/
// User-supplied resources may already be base64 encoded.
toURL(fctxt, asDataURI = false) {
if (
this.warURL !== undefined &&
asDataURI !== true &&
fctxt instanceof Object &&
fctxt.type !== 'xmlhttprequest'
) {
return `${this.warURL}${vAPI.warSecret()}`;
}
if ( this.data === undefined ) { return; }
// https://github.com/uBlockOrigin/uBlock-issues/issues/701
if ( this.data === '' ) {
const mime = typeToMimeMap.get(fctxt.type);
if ( mime === undefined ) { return; }
return `data:${mime},`;
}
if ( this.data.startsWith('data:') === false ) {
if ( this.mime.indexOf(';') === -1 ) {
this.data = `data:${this.mime};base64,${btoa(this.data)}`;
} else {
this.data = `data:${this.mime},${this.data}`;
}
}
return this.data;
}
toContent() {
if ( this.data.startsWith('data:') ) {
const pos = this.data.indexOf(',');
const base64 = this.data.endsWith(';base64', pos);
this.data = this.data.slice(pos + 1);
if ( base64 ) {
this.data = atob(this.data);
}
}
return this.data;
}
static fromContent(mime, content) {
const r = new RedirectEntry();
r.mime = mime;
r.data = content;
return r;
}
static fromSelfie(selfie) {
const r = new RedirectEntry();
r.mime = selfie.mime;
r.data = selfie.data;
r.warURL = selfie.warURL;
return r;
}
};
/******************************************************************************/
/******************************************************************************/
const RedirectEngine = function() {
this.aliases = new Map();
this.resources = new Map();
this.reset();
this.resourceNameRegister = '';
};
/******************************************************************************/
RedirectEngine.prototype.reset = function() {
this.rules = new Map();
this.ruleSources = new Set();
this.ruleDestinations = new Set();
this.resetCache();
this.modifyTime = Date.now();
};
RedirectEngine.prototype.resetCache = function() {
this._src = '';
this._srcAll = [ '*' ];
this._des = '';
this._desAll = [ '*' ];
};
/******************************************************************************/
RedirectEngine.prototype.freeze = function() {
};
/******************************************************************************/
RedirectEngine.prototype.toBroaderHostname = function(hostname) {
const pos = hostname.indexOf('.');
if ( pos !== -1 ) {
return hostname.slice(pos + 1);
}
return hostname !== '*' ? '*' : '';
};
/******************************************************************************/
RedirectEngine.prototype.decomposeHostname = function(hn, dict, out) {
let i = 0;
for (;;) {
if ( dict.has(hn) ) {
out[i] = hn; i += 1;
}
hn = this.toBroaderHostname(hn);
if ( hn === '' ) { break; }
}
out.length = i;
};
/******************************************************************************/
RedirectEngine.prototype.lookup = function(fctxt) {
const src = fctxt.getDocHostname();
const des = fctxt.getHostname();
const type = fctxt.type;
if ( src !== this._src ) {
this._src = src;
this.decomposeHostname(src, this.ruleSources, this._srcAll);
}
if ( this._srcAll.length === 0 ) { return; }
if ( des !== this._des ) {
this._des = des;
this.decomposeHostname(des, this.ruleDestinations, this._desAll);
}
if ( this._desAll.length === 0 ) { return; }
const reqURL = fctxt.url;
for ( const src of this._srcAll ) {
for ( const des of this._desAll ) {
let entries = this.rules.get(`${src} ${des} ${type}`);
if ( entries !== undefined ) {
const rule = this.lookupRule(entries, reqURL);
if ( rule !== undefined ) { return rule; }
}
entries = this.rules.get(`${src} ${des} *`);
if ( entries !== undefined ) {
const rule = this.lookupRule(entries, reqURL);
if ( rule !== undefined ) { return rule; }
}
}
}
};
RedirectEngine.prototype.lookupRule = function(entries, reqURL) {
for ( const entry of entries ) {
if ( entry.pat instanceof RegExp === false ) {
entry.pat = new RegExp(entry.pat, 'i');
}
if ( entry.pat.test(reqURL) ) {
return entry;
}
}
};
/******************************************************************************/
RedirectEngine.prototype.toURL = function(fctxt) {
const rule = this.lookup(fctxt);
if ( rule === undefined ) { return; }
let token = this.resourceNameRegister = rule.tok;
const asDataURI = token.charCodeAt(0) === 0x25 /* '%' */;
if ( asDataURI ) {
token = token.slice(1);
}
const entry = this.resources.get(this.aliases.get(token) || token);
if ( entry !== undefined ) {
return entry.toURL(fctxt, asDataURI);
}
};
/******************************************************************************/
RedirectEngine.prototype.addRule = function(src, des, type, pattern, redirect) {
this.ruleSources.add(src);
this.ruleDestinations.add(des);
const key = `${src} ${des} ${type}`,
entries = this.rules.get(key);
if ( entries === undefined ) {
this.rules.set(key, [ { tok: redirect, pat: pattern } ]);
this.modifyTime = Date.now();
return;
}
let entry;
for ( var i = 0, n = entries.length; i < n; i++ ) {
entry = entries[i];
if ( redirect === entry.tok ) { break; }
}
if ( i === n ) {
entries.push({ tok: redirect, pat: pattern });
return;
}
let p = entry.pat;
if ( p instanceof RegExp ) {
p = p.source;
}
// Duplicate?
let pos = p.indexOf(pattern);
if ( pos !== -1 ) {
if ( pos === 0 || p.charAt(pos - 1) === '|' ) {
pos += pattern.length;
if ( pos === p.length || p.charAt(pos) === '|' ) { return; }
}
}
entry.pat = p + '|' + pattern;
};
/******************************************************************************/
RedirectEngine.prototype.fromCompiledRule = function(line) {
const fields = line.split('\t');
if ( fields.length !== 5 ) { return; }
this.addRule(fields[0], fields[1], fields[2], fields[3], fields[4]);
};
/******************************************************************************/
RedirectEngine.prototype.compileRuleFromStaticFilter = function(line) {
const matches = this.reFilterParser.exec(line);
if ( matches === null || matches.length !== 4 ) { return; }
const des = matches[1] || '';
// https://github.com/uBlockOrigin/uBlock-issues/issues/572
// Extract best possible hostname.
let deshn = des;
let pos = deshn.lastIndexOf('*');
if ( pos !== -1 ) {
deshn = deshn.slice(pos + 1);
pos = deshn.indexOf('.');
if ( pos !== -1 ) {
deshn = deshn.slice(pos + 1);
} else {
deshn = '';
}
}
const path = matches[2] || '';
let pattern =
des
.replace(/\*/g, '[\\w.%-]*')
.replace(/\./g, '\\.') +
path
.replace(/\|$/, '$')
.replace(/[.+?{}()|[\]\/\\]/g, '\\$&')
.replace(/\^/g, '[^\\w.%-]')
.replace(/\*/g, '.*?');
if ( pattern === '' ) {
pattern = '^';
}
let type,
redirect = '',
srchns = [];
for ( const option of matches[3].trim().split(/,/) ) {
if ( option.startsWith('redirect=') ) {
redirect = option.slice(9);
continue;
}
if ( option.startsWith('redirect-rule=') ) {
redirect = option.slice(14);
continue;
}
if ( option === 'empty' ) {
redirect = 'empty';
continue;
}
if ( option === 'mp4' ) {
redirect = 'noopmp4-1s';
continue;
}
if ( option.startsWith('domain=') ) {
srchns = option.slice(7).split('|');
continue;
}
if ( (option === 'first-party' || option === '1p') && deshn !== '' ) {
srchns.push(µBlock.URI.domainFromHostname(deshn) || deshn);
continue;
}
// One and only one type must be specified.
if ( this.supportedTypes.has(option) ) {
if ( type !== undefined ) { return; }
type = this.supportedTypes.get(option);
continue;
}
}
// Need a resource token.
if ( redirect === '' ) { return; }
// Need one single type -- not negated.
if ( type === undefined ) {
if ( redirect === 'empty' ) {
type = '*';
} else if ( redirect === 'noopmp4-1s' ) {
type = 'media';
} else {
return;
}
}
if ( deshn === '' ) {
deshn = '*';
}
if ( srchns.length === 0 ) {
srchns.push('*');
}
const out = [];
for ( const srchn of srchns ) {
if ( srchn === '' ) { continue; }
if ( srchn.startsWith('~') ) { continue; }
out.push(`${srchn}\t${deshn}\t${type}\t${pattern}\t${redirect}`);
}
if ( out.length === 0 ) { return; }
return out;
};
/******************************************************************************/
RedirectEngine.prototype.reFilterParser = /^(?:\|\|([^\/:?#^]+)|\*?)([^$]+)?\$([^$]+)$/;
RedirectEngine.prototype.supportedTypes = new Map([
[ 'css', 'stylesheet' ],
[ 'font', 'font' ],
[ 'image', 'image' ],
[ 'media', 'media' ],
[ 'object', 'object' ],
[ 'script', 'script' ],
[ 'stylesheet', 'stylesheet' ],
[ 'frame', 'sub_frame' ],
[ 'subdocument', 'sub_frame' ],
[ 'xhr', 'xmlhttprequest' ],
[ 'xmlhttprequest', 'xmlhttprequest' ],
]);
/******************************************************************************/
RedirectEngine.prototype.toSelfie = function(path) {
// Because rules may contains RegExp instances, we need to manually
// convert it to a serializable format. The serialized format must be
// suitable to be used as an argument to the Map() constructor.
const rules = [];
for ( const item of this.rules ) {
const rule = [ item[0], [] ];
const entries = item[1];
let i = entries.length;
while ( i-- ) {
const entry = entries[i];
rule[1].push({
tok: entry.tok,
pat: entry.pat instanceof RegExp ? entry.pat.source : entry.pat
});
}
rules.push(rule);
}
return µBlock.assets.put(
`${path}/main`,
JSON.stringify({
rules: rules,
ruleSources: Array.from(this.ruleSources),
ruleDestinations: Array.from(this.ruleDestinations)
})
);
};
/******************************************************************************/
RedirectEngine.prototype.fromSelfie = async function(path) {
const result = await µBlock.assets.get(`${path}/main`);
let selfie;
try {
selfie = JSON.parse(result.content);
} catch (ex) {
}
if ( selfie instanceof Object === false ) { return false; }
this.rules = new Map(selfie.rules);
this.ruleSources = new Set(selfie.ruleSources);
this.ruleDestinations = new Set(selfie.ruleDestinations);
this.resetCache();
this.modifyTime = Date.now();
return true;
};
/******************************************************************************/
RedirectEngine.prototype.resourceContentFromName = function(name, mime) {
const entry = this.resources.get(this.aliases.get(name) || name);
if ( entry === undefined ) { return; }
if ( mime === undefined || entry.mime.startsWith(mime) ) {
return entry.toContent();
}
};
/******************************************************************************/
// TODO: combine same key-redirect pairs into a single regex.
// https://github.com/uBlockOrigin/uAssets/commit/deefe875551197d655f79cb540e62dfc17c95f42
// Consider 'none' a reserved keyword, to be used to disable redirection.
RedirectEngine.prototype.resourcesFromString = function(text) {
const lineIter = new µBlock.LineIterator(removeTopCommentBlock(text));
const reNonEmptyLine = /\S/;
let fields, encoded, details;
while ( lineIter.eot() === false ) {
let line = lineIter.next();
if ( line.startsWith('#') ) { continue; }
if ( line.startsWith('// ') ) { continue; }
if ( fields === undefined ) {
if ( line === '' ) { continue; }
// Modern parser
if ( line.startsWith('/// ') ) {
const name = line.slice(4).trim();
fields = [ name, mimeFromName(name) ];
continue;
}
// Legacy parser
const head = line.trim().split(/\s+/);
if ( head.length !== 2 ) { continue; }
if ( head[0] === 'none' ) { continue; }
let pos = head[1].indexOf(';');
if ( pos === -1 ) { pos = head[1].length; }
if ( validMimes.has(head[1].slice(0, pos)) === false ) {
continue;
}
encoded = head[1].indexOf(';') !== -1;
fields = head;
continue;
}
if ( line.startsWith('/// ') ) {
if ( details === undefined ) {
details = {};
}
const [ prop, value ] = line.slice(4).trim().split(/\s+/);
if ( value !== undefined ) {
details[prop] = value;
}
continue;
}
if ( reNonEmptyLine.test(line) ) {
fields.push(encoded ? line.trim() : line);
continue;
}
const name = this.aliases.get(fields[0]) || fields[0];
const mime = fields[1];
const content = µBlock.orphanizeString(
fields.slice(2).join(encoded ? '' : '\n')
);
// No more data, add the resource.
this.resources.set(
name,
RedirectEntry.fromContent(mime, content)
);
if ( details instanceof Object && details.alias ) {
this.aliases.set(details.alias, name);
}
fields = undefined;
details = undefined;
}
// Process pending resource data.
if ( fields !== undefined ) {
const name = fields[0];
const mime = fields[1];
const content = µBlock.orphanizeString(
fields.slice(2).join(encoded ? '' : '\n')
);
this.resources.set(
name,
RedirectEntry.fromContent(mime, content)
);
if ( details instanceof Object && details.alias ) {
this.aliases.set(details.alias, name);
}
}
this.modifyTime = Date.now();
};
const removeTopCommentBlock = function(text) {
return text.replace(/^\/\*[\S\s]+?\n\*\/\s*/, '');
};
/******************************************************************************/
RedirectEngine.prototype.loadBuiltinResources = function() {
// TODO: remove once usage of uBO 1.20.4 is widespread.
µBlock.assets.remove('ublock-resources');
this.resources = new Map();
this.aliases = new Map();
const fetches = [
µBlock.assets.fetchText(
'/assets/resources/scriptlets.js'
).then(result => {
const content = result.content;
if ( typeof content === 'string' && content.length !== 0 ) {
this.resourcesFromString(content);
}
}),
];
const store = (name, data = undefined) => {
const details = redirectableResources.get(name);
const entry = RedirectEntry.fromSelfie({
mime: mimeFromName(name),
data,
warURL: vAPI.getURL(`/web_accessible_resources/${name}`),
});
this.resources.set(name, entry);
if ( details.alias !== undefined ) {
this.aliases.set(details.alias, name);
}
};
const processBlob = (name, blob) => {
return new Promise(resolve => {
const reader = new FileReader();
reader.onload = ( ) => {
store(name, reader.result);
resolve();
};
reader.onabort = reader.onerror = ( ) => {
resolve();
};
reader.readAsDataURL(blob);
});
};
const processText = (name, text) => {
store(name, removeTopCommentBlock(text));
};
const process = result => {
const match = /^\/web_accessible_resources\/([^?]+)/.exec(result.url);
if ( match === null ) { return; }
const name = match[1];
return result.content instanceof Blob
? processBlob(name, result.content)
: processText(name, result.content);
};
for ( const [ name, details ] of redirectableResources ) {
if ( typeof details.data !== 'string' ) {
store(name);
continue;
}
fetches.push(
µBlock.assets.fetch(
`/web_accessible_resources/${name}${vAPI.warSecret()}`,
{ responseType: details.data }
).then(
result => process(result)
)
);
}
return Promise.all(fetches);
};
/******************************************************************************/
RedirectEngine.prototype.getResourceDetails = function() {
const out = new Map();
for ( const [ name, entry ] of this.resources ) {
out.set(name, {
canInject: typeof entry.data === 'string',
canRedirect: entry.warURL !== undefined,
aliasOf: '',
});
}
for ( const [ alias, name ] of this.aliases ) {
const original = out.get(name);
if ( original === undefined ) { continue; }
const aliased = Object.assign({}, original);
aliased.aliasOf = name;
out.set(alias, aliased);
}
return Array.from(out).sort((a, b) => {
return a[0].localeCompare(b[0]);
});
};
/******************************************************************************/
const resourcesSelfieVersion = 5;
RedirectEngine.prototype.selfieFromResources = function() {
µBlock.assets.put(
'compiled/redirectEngine/resources',
JSON.stringify({
version: resourcesSelfieVersion,
aliases: Array.from(this.aliases),
resources: Array.from(this.resources),
})
);
};
RedirectEngine.prototype.resourcesFromSelfie = async function() {
const result = await µBlock.assets.get('compiled/redirectEngine/resources');
let selfie;
try {
selfie = JSON.parse(result.content);
} catch(ex) {
}
if (
selfie instanceof Object === false ||
selfie.version !== resourcesSelfieVersion ||
Array.isArray(selfie.resources) === false
) {
return false;
}
this.aliases = new Map(selfie.aliases);
this.resources = new Map();
for ( const [ token, entry ] of selfie.resources ) {
this.resources.set(token, RedirectEntry.fromSelfie(entry));
}
return true;
};
RedirectEngine.prototype.invalidateResourcesSelfie = function() {
µBlock.assets.remove('compiled/redirectEngine/resources');
// TODO: obsolete, remove eventually
µBlock.cacheStorage.remove('resourcesSelfie');
};
/******************************************************************************/
/******************************************************************************/
return new RedirectEngine();
/******************************************************************************/
})();