From a16e4161de5b33856312226e71b05c6eef8bf83a Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sat, 23 Nov 2019 12:46:52 -0500 Subject: [PATCH] Fine tune hostname uncloaking through CNAME-lookup Related issue: - https://github.com/uBlockOrigin/uBlock-issues/issues/780 Related commit: - https://github.com/gorhill/uBlock/commit/3a564c199260 This adds two new advanced settings: - cnameIgnoreRootDocument - Default to `true` - Tells uBO to skip CNAME-lookup for root document. - cnameReplayFullURL - Default to `false` - Tells uBO whether to replay the whole URL or just the origin part of it. Replaying only the origin part is meant to lower undue breakage and improve performance by avoiding repeating the pattern-matching of the whole URL -- which pattern-matching was most likely already accomplished with the original request. This commit is meant to explore enabling CNAME-lookup by default for the next stable release while: - Eliminating a development burden by removing the need to create a new filtering syntax to deal with undesirable CNAME-cloaked hostnames - Eliminating a filter list maintainer burden by removing the need to find/deal with all base domains which engage in undesirable CNAME-cloaked hostnames The hope is that the approach implemented in this commit should require at most a few unbreak rules with no further need for special filtering syntax or filter list maintance efforts. --- platform/chromium/vapi-background.js | 27 +++++++++---- platform/firefox/vapi-webrequest.js | 36 ++++++++++++----- src/css/logger-ui.css | 2 +- src/js/background.js | 2 + src/js/filtering-context.js | 6 ++- src/js/logger-ui.js | 59 ++++++++++++++++++++-------- src/js/storage.js | 2 + src/js/traffic.js | 2 +- src/logger-ui.html | 5 ++- 9 files changed, 102 insertions(+), 39 deletions(-) diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index e294047cd..2f0669518 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -1259,18 +1259,29 @@ vAPI.Net = class { console.info('No requests found to benchmark'); return; } + const mappedTypes = new Map([ + [ 'document', 'main_frame' ], + [ 'subdocument', 'sub_frame' ], + ]); console.info('vAPI.net.onBeforeSuspendableRequest()...'); const t0 = self.performance.now(); const promises = []; + const details = { + documentUrl: '', + tabId: -1, + parentFrameId: -1, + frameId: 0, + type: '', + url: '', + }; for ( const request of requests ) { - const details = { - documentUrl: request.frameUrl, - tabId: Number.MAX_SAFE_INTEGER, - parentFrameId: -1, - frameId: 0, - type: request.cpt, - url: request.url, - }; + details.documentUrl = request.frameUrl; + details.tabId = -1; + details.parentFrameId = -1; + details.frameId = 0; + details.type = mappedTypes.get(request.cpt) || request.cpt; + details.url = request.url; + if ( details.type === 'main_frame' ) { continue; } promises.push(this.onBeforeSuspendableRequest(details)); } return Promise.all(promises).then(results => { diff --git a/platform/firefox/vapi-webrequest.js b/platform/firefox/vapi-webrequest.js index 6c82c3fa0..93055a2ab 100644 --- a/platform/firefox/vapi-webrequest.js +++ b/platform/firefox/vapi-webrequest.js @@ -63,16 +63,20 @@ this.cnames = new Map([ [ '', '' ] ]); this.cnameAliasList = null; this.cnameIgnoreList = null; - this.url = new URL(vAPI.getURL('/')); + this.cnameIgnore1stParty = true; + this.cnameIgnoreRootDocument = true; this.cnameMaxTTL = 60; + this.cnameReplayFullURL = false; this.cnameTimer = undefined; } setOptions(options) { super.setOptions(options); this.cnameAliasList = this.regexFromStrList(options.cnameAliasList); this.cnameIgnoreList = this.regexFromStrList(options.cnameIgnoreList); - this.cnameIgnore1stParty = options.cnameIgnore1stParty === true; + this.cnameIgnore1stParty = options.cnameIgnore1stParty !== false; + this.cnameIgnoreRootDocument = options.cnameIgnoreRootDocument !== false; this.cnameMaxTTL = options.cnameMaxTTL || 120; + this.cnameReplayFullURL = options.cnameReplayFullURL === true; this.cnames.clear(); this.cnames.set('', ''); } normalizeDetails(details) { @@ -123,11 +127,22 @@ } return Array.from(out); } - processCanonicalName(cname, details) { - this.url.href = details.url; - details.cnameOf = this.url.hostname; - this.url.hostname = cname; - details.url = this.url.href; + processCanonicalName(hn, cn, details) { + const hnBeg = details.url.indexOf(hn); + if ( hnBeg === -1 ) { return; } + const oldURL = details.url; + let newURL = oldURL.slice(0, hnBeg) + cn; + const hnEnd = hnBeg + hn.length; + if ( this.cnameReplayFullURL ) { + newURL += oldURL.slice(hnEnd); + } else { + const pathBeg = oldURL.indexOf('/', hnEnd); + if ( pathBeg !== -1 ) { + newURL += oldURL.slice(hnEnd, pathBeg + 1); + } + } + details.url = newURL; + details.aliasURL = oldURL; return super.onBeforeSuspendableRequest(details); } recordCanonicalName(hn, record) { @@ -187,11 +202,14 @@ let r = super.onBeforeSuspendableRequest(details); if ( r !== undefined ) { return r; } if ( this.cnameAliasList === null ) { return; } + if ( details.type === 'main_frame' && this.cnameIgnoreRootDocument ) { + return; + } const hn = vAPI.hostnameFromNetworkURL(details.url); let cname = this.cnames.get(hn); if ( cname === '' ) { return; } if ( cname !== undefined ) { - return this.processCanonicalName(cname, details); + return this.processCanonicalName(hn, cname, details); } if ( this.cnameAliasList.test(hn) === false ) { this.cnames.set(hn, ''); @@ -201,7 +219,7 @@ rec => { const cname = this.recordCanonicalName(hn, rec); if ( cname === '' ) { return; } - return this.processCanonicalName(cname, details); + return this.processCanonicalName(hn, cname, details); }, ( ) => { this.cnames.set(hn, ''); diff --git a/src/css/logger-ui.css b/src/css/logger-ui.css index 89cafffe9..7fff41f62 100644 --- a/src/css/logger-ui.css +++ b/src/css/logger-ui.css @@ -268,7 +268,7 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm, body.colorBlind #vwRenderer .logEntry > div.redirect { background-color: rgba(0, 19, 110, 0.1); } -#vwRenderer .logEntry > div[data-cnameof] { +#vwRenderer .logEntry > div[data-aliasid] { color: mediumblue; } #vwRenderer .logEntry > div[data-type="tabLoad"] { diff --git a/src/js/background.js b/src/js/background.js index 36dedd2f6..0860abf79 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -49,7 +49,9 @@ const µBlock = (( ) => { // jshint ignore:line cnameAliasList: 'unset', cnameIgnoreList: 'unset', cnameIgnore1stParty: true, + cnameIgnoreRootDocument: true, cnameMaxTTL: 120, + cnameReplayFullURL: false, consoleLogLevel: 'unset', debugScriptlets: false, debugScriptletInjector: false, diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js index 5be3afd21..5ae123ec9 100644 --- a/src/js/filtering-context.js +++ b/src/js/filtering-context.js @@ -29,9 +29,10 @@ } this.tstamp = 0; this.realm = ''; + this.id = undefined; this.type = undefined; - this.cnameOf = undefined; this.url = undefined; + this.aliasURL = undefined; this.hostname = undefined; this.domain = undefined; this.docId = undefined; @@ -64,9 +65,10 @@ } this.fromTabId(tabId); this.realm = ''; + this.id = details.requestId; this.type = details.type; this.setURL(details.url); - this.cnameOf = details.cnameOf || undefined; + this.aliasURL = details.aliasURL || undefined; this.docId = details.type !== 'sub_frame' ? details.frameId : details.parentFrameId; diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index 0acbe6221..99b46ad5d 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -157,9 +157,12 @@ const regexFromURLFilteringResult = function(result) { const nodeFromURL = function(parent, url, re) { const fragment = document.createDocumentFragment(); - if ( re instanceof RegExp === false ) { + if ( re === undefined ) { fragment.textContent = url; } else { + if ( typeof re === 'string' ) { + re = new RegExp(re.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'); + } const matches = re.exec(url); if ( matches === null || matches[0].length === 0 ) { fragment.textContent = url; @@ -211,6 +214,9 @@ const LogEntry = function(details) { this[prop] = details[prop]; } } + if ( details.aliasURL !== undefined ) { + this.aliased = true; + } if ( this.tabDomain === '' ) { this.tabDomain = this.tabHostname || ''; } @@ -222,12 +228,13 @@ const LogEntry = function(details) { } }; LogEntry.prototype = { - cnameOf: '', + aliased: false, dead: false, docDomain: '', docHostname: '', domain: '', filter: undefined, + id: '', realm: '', tabDomain: '', tabHostname: '', @@ -294,7 +301,7 @@ const processLoggerEntries = function(response) { if ( autoDeleteVoidedRows ) { continue; } parsed.voided = true; } - if ( parsed.type === 'main_frame' && parsed.cnameOf === '' ) { + if ( parsed.type === 'main_frame' && parsed.aliased === false ) { const separator = createLogSeparator(parsed, unboxed.url); loggerEntries.unshift(separator); if ( rowFilterer.filterOne(separator) ) { @@ -304,7 +311,7 @@ const processLoggerEntries = function(response) { } } } - if ( cnameOfEnabled === false && parsed.cnameOf !== '' ) { + if ( cnameOfEnabled === false && parsed.aliased ) { uDom.nodeFromId('filterExprCnameOf').style.display = ''; cnameOfEnabled = true; } @@ -405,8 +412,10 @@ const parseLogEntry = function(details) { textContent.push(normalizeToStr(details.url)); // Hidden cells -- useful for row-filtering purpose - if ( entry.cnameOf !== '' ) { - textContent.push(`cnameOf=${entry.cnameOf}`); + + // Cell 7 + if ( entry.aliased ) { + textContent.push(`aliasURL=${details.aliasURL}`); } entry.textContent = textContent.join('\t'); @@ -723,7 +732,7 @@ const viewPort = (( ) => { span.textContent = cells[5]; // URL - let re = null; + let re; if ( filteringType === 'static' ) { re = new RegExp(filter.regex, 'gi'); } else if ( filteringType === 'dynamicUrl' ) { @@ -731,9 +740,12 @@ const viewPort = (( ) => { } nodeFromURL(div.children[6], cells[6], re); - // Cname - if ( details.cnameOf !== '' ) { - div.setAttribute('data-cnameof', details.cnameOf); + // Alias URL (CNAME, etc.) + if ( cells.length > 7 ) { + const pos = details.textContent.lastIndexOf('\taliasURL='); + if ( pos !== -1 ) { + div.setAttribute('data-aliasid', details.id); + } } return div; @@ -1452,6 +1464,16 @@ const reloadTab = function(ev) { return targetRow.children[1].textContent; }; + const aliasURLFromID = function(id) { + if ( id === '' ) { return ''; } + for ( const entry of loggerEntries ) { + if ( entry.id !== id || entry.aliased ) { continue; } + const fields = entry.textContent.split('\t'); + return fields[6] || ''; + } + return ''; + }; + const toSummaryPaneFilterNode = async function(receiver, filter) { receiver.children[1].textContent = filter; if ( filterAuthorMode !== true ) { return; } @@ -1613,8 +1635,8 @@ const reloadTab = function(ev) { rows[6].style.display = 'none'; } // URL - text = trch[6].textContent; - if ( text !== '' ) { + const canonicalURL = trch[6].textContent; + if ( canonicalURL !== '' ) { const attr = tr.getAttribute('data-status') || ''; if ( attr !== '' ) { rows[7].setAttribute('data-status', attr); @@ -1623,12 +1645,17 @@ const reloadTab = function(ev) { } else { rows[7].style.display = 'none'; } - // CNAME of - text = tr.getAttribute('data-cnameof') || ''; - if ( text !== '' ) { - rows[8].children[1].textContent = text; + // Alias URL + text = tr.getAttribute('data-aliasid'); + const aliasURL = text ? aliasURLFromID(text) : ''; + if ( aliasURL !== '' ) { + rows[8].children[1].textContent = + vAPI.hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' + + vAPI.hostnameFromURI(canonicalURL); + rows[9].children[1].textContent = aliasURL; } else { rows[8].style.display = 'none'; + rows[9].style.display = 'none'; } }; diff --git a/src/js/storage.js b/src/js/storage.js index 6c518a4fa..bbced09af 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -139,7 +139,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { cnameAliasList: µBlock.hiddenSettings.cnameAliasList, cnameIgnoreList: µBlock.hiddenSettings.cnameIgnoreList, cnameIgnore1stParty: µBlock.hiddenSettings.cnameIgnore1stParty, + cnameIgnoreRootDocument: µBlock.hiddenSettings.cnameIgnoreRootDocument, cnameMaxTTL: µBlock.hiddenSettings.cnameMaxTTL, + cnameReplayFullURL: µBlock.hiddenSettings.cnameReplayFullURL, }); }); diff --git a/src/js/traffic.js b/src/js/traffic.js index 341b04acf..30ba0fd18 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -99,7 +99,7 @@ const onBeforeRequest = function(details) { if ( details.parentFrameId !== -1 && details.type === 'sub_frame' && - details.cnameOf === undefined + details.aliasURL === undefined ) { pageStore.setFrame(details.frameId, details.url); } diff --git a/src/logger-ui.html b/src/logger-ui.html index ba9caecbd..5d8f052f3 100644 --- a/src/logger-ui.html +++ b/src/logger-ui.html @@ -65,7 +65,7 @@
- + @@ -121,7 +121,8 @@
-
CNAME of
+
CNAME
+
Original URL