From 3a564c199260a857f3d78d5f12b8c3f1aa85b865 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Tue, 19 Nov 2019 12:05:33 -0500 Subject: [PATCH] Add ability to uncloak CNAME records Related issue: - https://github.com/uBlockOrigin/uBlock-issues/issues/780 New webext permission added: `dns`, which purpose is to allow an extension to fetch the DNS record of specific hostnames, reference documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/dns The webext API `dns` is available in Firefox 60+ only. The new API will enable uBO to "uncloak" the actual hostname used in network requests. The ability is currently disabled by default for now -- this is only a first commit related to the above issue to allow advanced users to immediately use the new ability. Four advanced settings have been created to control the uncloaking of actual hostnames: cnameAliasList: a space-separated list of hostnames. Default value: unset => empty list. Special value: * => all hostnames. A space-separated list of hostnames => this tells uBO to "uncloak" the hostnames in the list will. cnameIgnoreList: a space-separated list of hostnames. Default value: unset => empty list. Special value: * => all hostnames. A space-separated list of hostnames => this tells uBO to NOT re-run the network request through uBO's filtering engine with the CNAME hostname. This is useful to exclude commonly used actual hostnames from being re-run through uBO's filtering engine, so as to avoid pointless overhead. cnameIgnore1stParty: boolean. Default value: true. Whether uBO should ignore to re-run a network request through the filtering engine when the CNAME hostname is 1st-party to the alias hostname. cnameMaxTTL: number of minutes. Default value: 120. This tells uBO to clear its CNAME cache after the specified time. For efficiency purpose, uBO will cache alias=>CNAME associations for reuse so as to reduce calls to `browser.dns.resolve`. All the associations will be cleared after the specified time to ensure the map does not grow too large and too ensure uBO uses up to date CNAME information. --- platform/chromium/vapi-background.js | 15 +++-- platform/chromium/vapi-common.js | 45 +++++++++++++ platform/firefox/manifest.json | 1 + platform/firefox/vapi-webrequest.js | 99 +++++++++++++++++++++++++++- src/css/logger-ui.css | 3 + src/js/background.js | 4 ++ src/js/filtering-context.js | 2 + src/js/logger-ui.js | 66 ++++++++++++------- src/js/storage.js | 12 +++- src/js/traffic.js | 6 +- src/js/uritools.js | 63 +----------------- src/logger-ui.html | 2 + 12 files changed, 226 insertions(+), 92 deletions(-) diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index 9c119c752..450372c99 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -1164,16 +1164,17 @@ vAPI.Net = class { browser.webRequest.onBeforeRequest.addListener( details => { this.normalizeDetails(details); - if ( this.suspendDepth === 0 || details.tabId < 0 ) { - if ( this.suspendableListener === undefined ) { return; } - return this.suspendableListener(details); + if ( this.suspendDepth !== 0 && details.tabId >= 0 ) { + return this.suspendOneRequest(details); } - return this.suspendOneRequest(details); + return this.onBeforeSuspendableRequest(details); }, this.denormalizeFilters({ urls: [ 'http://*/*', 'https://*/*' ] }), [ 'blocking' ] ); } + setOptions(/* options */) { + } normalizeDetails(/* details */) { } denormalizeFilters(filters) { @@ -1208,6 +1209,10 @@ vAPI.Net = class { options ); } + onBeforeSuspendableRequest(details) { + if ( this.suspendableListener === undefined ) { return; } + return this.suspendableListener(details); + } setSuspendableListener(listener) { this.suspendableListener = listener; } @@ -1242,7 +1247,7 @@ vAPI.Net = class { this.suspendDepth -= 1; } if ( this.suspendDepth !== 0 ) { return; } - this.unsuspendAllRequests(this.suspendableListener); + this.unsuspendAllRequests(); } canSuspend() { return false; diff --git a/platform/chromium/vapi-common.js b/platform/chromium/vapi-common.js index c255b1dd4..f9772f66f 100644 --- a/platform/chromium/vapi-common.js +++ b/platform/chromium/vapi-common.js @@ -116,6 +116,51 @@ vAPI.webextFlavor = { /******************************************************************************/ +{ + const punycode = self.punycode; + const reCommonHostnameFromURL = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//; + const reAuthorityFromURI = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; + const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i; + const reHostFromAuthority = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; + const reIPv6FromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; + const reMustNormalizeHostname = /[^0-9a-z._-]/; + + vAPI.hostnameFromURI = function(uri) { + let matches = reCommonHostnameFromURL.exec(uri); + if ( matches !== null ) { return matches[1]; } + matches = reAuthorityFromURI.exec(uri); + if ( matches === null ) { return ''; } + const authority = matches[1].slice(2); + if ( reHostFromNakedAuthority.test(authority) ) { + return authority.toLowerCase(); + } + matches = reHostFromAuthority.exec(authority); + if ( matches === null ) { + matches = reIPv6FromAuthority.exec(authority); + if ( matches === null ) { return ''; } + } + let hostname = matches[1]; + while ( hostname.endsWith('.') ) { + hostname = hostname.slice(0, -1); + } + if ( reMustNormalizeHostname.test(hostname) ) { + hostname = punycode.toASCII(hostname.toLowerCase()); + } + return hostname; + }; + + const psl = self.publicSuffixList; + const reIPAddressNaive = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; + + vAPI.domainFromHostname = function(hostname) { + return reIPAddressNaive.test(hostname) + ? hostname + : psl.getDomain(hostname); + }; +} + +/******************************************************************************/ + vAPI.download = function(details) { if ( !details.url ) { return; } const a = document.createElement('a'); diff --git a/platform/firefox/manifest.json b/platform/firefox/manifest.json index 09416b5d6..1cc427445 100644 --- a/platform/firefox/manifest.json +++ b/platform/firefox/manifest.json @@ -75,6 +75,7 @@ "open_in_tab": true }, "permissions": [ + "dns", "menus", "privacy", "storage", diff --git a/platform/firefox/vapi-webrequest.js b/platform/firefox/vapi-webrequest.js index b3c379a3e..e2fcd9201 100644 --- a/platform/firefox/vapi-webrequest.js +++ b/platform/firefox/vapi-webrequest.js @@ -60,6 +60,20 @@ constructor() { super(); this.pendingRequests = []; + this.cnames = new Map(); + this.cnameAliasList = null; + this.cnameIgnoreList = null; + this.url = new URL(vAPI.getURL('/')); + this.cnameMaxTTL = 60; + this.cnameTimer = undefined; + } + setOptions(options) { + super.setOptions(options); + this.cnameAliasList = this.regexFromStrList(options.cnameAliasList); + this.cnameIgnoreList = this.regexFromStrList(options.cnameIgnoreList); + this.cnameIgnore1stParty = options.cnameIgnore1stParty === true; + this.cnameMaxTTL = options.cnameMaxTTL || 120; + this.cnames.clear(); } normalizeDetails(details) { if ( mustPunycode && !reAsciiHostname.test(details.url) ) { @@ -109,6 +123,87 @@ } return Array.from(out); } + processCanonicalName(cname, details) { + this.url.href = details.url; + details.cnameOf = this.url.hostname; + this.url.hostname = cname; + details.url = this.url.href; + return super.onBeforeSuspendableRequest(details); + } + recordCanonicalName(hn, record) { + let cname = + typeof record.canonicalName === 'string' && + record.canonicalName !== hn + ? record.canonicalName + : ''; + if ( + cname !== '' && + this.cnameIgnore1stParty && + vAPI.domainFromHostname(cname) === vAPI.domainFromHostname(hn) + ) { + cname = ''; + } + if ( + cname !== '' && + this.cnameIgnoreList !== null && + this.cnameIgnoreList.test(cname) + ) { + + cname = ''; + } + this.cnames.set(hn, cname); + if ( this.cnameTimer === undefined ) { + this.cnameTimer = self.setTimeout( + ( ) => { + this.cnameTimer = undefined; + this.cnames.clear(); + }, + this.cnameMaxTTL * 60000 + ); + } + return cname; + } + regexFromStrList(list) { + if ( + typeof list !== 'string' || + list.length === 0 || + list === 'unset' + ) { + return null; + } + if ( list === '*' ) { + return /^./; + } + return new RegExp( + '(?:^|\.)(?:' + + list.trim() + .split(/\s+/) + .map(a => a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) + .join('|') + + ')$' + ); + } + onBeforeSuspendableRequest(details) { + let r = super.onBeforeSuspendableRequest(details); + if ( r !== undefined ) { return r; } + if ( this.cnameAliasList === null ) { return; } + const hn = vAPI.hostnameFromURI(details.url); + let cname = this.cnames.get(hn); + if ( cname === '' ) { return; } + if ( cname !== undefined ) { + return this.processCanonicalName(cname, details); + } + if ( this.cnameAliasList.test(hn) === false ) { + this.cnames.set(hn, ''); + return; + } + return browser.dns.resolve(hn, [ 'canonical_name' ]).then(rec => { + const cname = this.recordCanonicalName(hn, rec); + if ( cname === '' ) { return; } + return this.processCanonicalName(cname, details); + + }); + } suspendOneRequest(details) { const pending = { details: Object.assign({}, details), @@ -121,11 +216,11 @@ this.pendingRequests.push(pending); return pending.promise; } - unsuspendAllRequests(resolver) { + unsuspendAllRequests() { const pendingRequests = this.pendingRequests; this.pendingRequests = []; for ( const entry of pendingRequests ) { - entry.resolve(resolver(entry.details)); + entry.resolve(this.onBeforeSuspendableRequest(entry.details)); } } canSuspend() { diff --git a/src/css/logger-ui.css b/src/css/logger-ui.css index 470081699..89cafffe9 100644 --- a/src/css/logger-ui.css +++ b/src/css/logger-ui.css @@ -268,6 +268,9 @@ body.colorBlind #vwRenderer .logEntry > div.cosmeticRealm, body.colorBlind #vwRenderer .logEntry > div.redirect { background-color: rgba(0, 19, 110, 0.1); } +#vwRenderer .logEntry > div[data-cnameof] { + color: mediumblue; + } #vwRenderer .logEntry > div[data-type="tabLoad"] { background-color: #666; color: white; diff --git a/src/js/background.js b/src/js/background.js index a63a598f6..36dedd2f6 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -46,6 +46,10 @@ const µBlock = (( ) => { // jshint ignore:line cacheStorageAPI: 'unset', cacheStorageCompression: true, cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', + cnameAliasList: 'unset', + cnameIgnoreList: 'unset', + cnameIgnore1stParty: true, + cnameMaxTTL: 120, consoleLogLevel: 'unset', debugScriptlets: false, debugScriptletInjector: false, diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js index 0761aa571..e9dbe8f29 100644 --- a/src/js/filtering-context.js +++ b/src/js/filtering-context.js @@ -30,6 +30,7 @@ this.tstamp = 0; this.realm = ''; this.type = undefined; + this.cnameOf = undefined; this.url = undefined; this.hostname = undefined; this.domain = undefined; @@ -65,6 +66,7 @@ this.realm = ''; this.type = details.type; this.setURL(details.url); + this.cnameOf = details.cnameOf !== undefined ? details.cnameOf : ''; this.docId = details.type !== 'sub_frame' ? details.frameId : details.parentFrameId; diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index 11e8e9e68..0acbe6221 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -44,6 +44,7 @@ let activeTabId = 0; let filterAuthorMode = false; let selectedTabId = 0; let netInspectorPaused = false; +let cnameOfEnabled = false; /******************************************************************************/ @@ -221,6 +222,7 @@ const LogEntry = function(details) { } }; LogEntry.prototype = { + cnameOf: '', dead: false, docDomain: '', docHostname: '', @@ -292,7 +294,7 @@ const processLoggerEntries = function(response) { if ( autoDeleteVoidedRows ) { continue; } parsed.voided = true; } - if ( parsed.type === 'main_frame' ) { + if ( parsed.type === 'main_frame' && parsed.cnameOf === '' ) { const separator = createLogSeparator(parsed, unboxed.url); loggerEntries.unshift(separator); if ( rowFilterer.filterOne(separator) ) { @@ -302,6 +304,10 @@ const processLoggerEntries = function(response) { } } } + if ( cnameOfEnabled === false && parsed.cnameOf !== '' ) { + uDom.nodeFromId('filterExprCnameOf').style.display = ''; + cnameOfEnabled = true; + } loggerEntries.unshift(parsed); if ( rowFilterer.filterOne(parsed) ) { filteredLoggerEntries.unshift(parsed); @@ -364,29 +370,28 @@ const parseLogEntry = function(details) { textContent.push(normalizeToStr(entry.docHostname)); // Cell 4 - if ( - entry.realm === 'network' && - typeof entry.domain === 'string' && - entry.domain !== '' - ) { - let partyness = ''; - if ( entry.tabDomain !== undefined ) { - if ( entry.tabId < 0 ) { - partyness += '0,'; - } - partyness += entry.domain === entry.tabDomain ? '1' : '3'; - } else { - partyness += '?'; - } - if ( entry.docDomain !== entry.tabDomain ) { - partyness += ','; - if ( entry.docDomain !== undefined ) { - partyness += entry.domain === entry.docDomain ? '1' : '3'; + if ( entry.realm === 'network' ) { + // partyness + if ( typeof entry.domain === 'string' && entry.domain !== '' ) { + let partyness = ''; + if ( entry.tabDomain !== undefined ) { + if ( entry.tabId < 0 ) { + partyness += '0,'; + } + partyness += entry.domain === entry.tabDomain ? '1' : '3'; } else { partyness += '?'; } + if ( entry.docDomain !== entry.tabDomain ) { + partyness += ','; + if ( entry.docDomain !== undefined ) { + partyness += entry.domain === entry.docDomain ? '1' : '3'; + } else { + partyness += '?'; + } + } + textContent.push(partyness); } - textContent.push(partyness); } else { textContent.push(''); } @@ -399,6 +404,11 @@ const parseLogEntry = function(details) { // Cell 6 textContent.push(normalizeToStr(details.url)); + // Hidden cells -- useful for row-filtering purpose + if ( entry.cnameOf !== '' ) { + textContent.push(`cnameOf=${entry.cnameOf}`); + } + entry.textContent = textContent.join('\t'); return entry; }; @@ -721,6 +731,11 @@ const viewPort = (( ) => { } nodeFromURL(div.children[6], cells[6], re); + // Cname + if ( details.cnameOf !== '' ) { + div.setAttribute('data-cnameof', details.cnameOf); + } + return div; }; @@ -1608,6 +1623,13 @@ const reloadTab = function(ev) { } else { rows[7].style.display = 'none'; } + // CNAME of + text = tr.getAttribute('data-cnameof') || ''; + if ( text !== '' ) { + rows[8].children[1].textContent = text; + } else { + rows[8].style.display = 'none'; + } }; // Fill dynamic URL filtering pane @@ -1951,14 +1973,14 @@ const rowFilterer = (( ) => { ); }; - const onFilterChangedAsync = (function() { + const onFilterChangedAsync = (( ) => { let timer; const commit = ( ) => { timer = undefined; parseInput(); filterAll(); }; - return function() { + return ( ) => { if ( timer !== undefined ) { clearTimeout(timer); } diff --git a/src/js/storage.js b/src/js/storage.js index 6718573a0..3f228c8b1 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -112,7 +112,6 @@ : 'unset'; } } - self.log.verbosity = this.hiddenSettings.consoleLogLevel; this.fireDOMEvent('hiddenSettingsChanged'); }; @@ -132,9 +131,18 @@ } vAPI.storage.set(bin); this.saveImmediateHiddenSettings(); - self.log.verbosity = this.hiddenSettings.consoleLogLevel; }; +self.addEventListener('hiddenSettingsChanged', ( ) => { + self.log.verbosity = µBlock.hiddenSettings.consoleLogLevel; + vAPI.net.setOptions({ + cnameAliasList: µBlock.hiddenSettings.cnameAliasList, + cnameIgnoreList: µBlock.hiddenSettings.cnameIgnoreList, + cnameIgnore1stParty: µBlock.hiddenSettings.cnameIgnore1stParty, + cnameMaxTTL: µBlock.hiddenSettings.cnameMaxTTL, + }); +}); + /******************************************************************************/ µBlock.hiddenSettingsFromString = function(raw) { diff --git a/src/js/traffic.js b/src/js/traffic.js index 9ce242120..341b04acf 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -96,7 +96,11 @@ const onBeforeRequest = function(details) { // Not blocked if ( result !== 1 ) { - if ( details.parentFrameId !== -1 && details.type === 'sub_frame' ) { + if ( + details.parentFrameId !== -1 && + details.type === 'sub_frame' && + details.cnameOf === undefined + ) { pageStore.setFrame(details.frameId, details.url); } return; diff --git a/src/js/uritools.js b/src/js/uritools.js index 24f28b395..cb0a17883 100644 --- a/src/js/uritools.js +++ b/src/js/uritools.js @@ -19,8 +19,6 @@ Home: https://github.com/gorhill/uBlock */ -/* global publicSuffixList */ - 'use strict'; /******************************************************************************* @@ -33,12 +31,10 @@ Naming convention from https://en.wikipedia.org/wiki/URI_scheme#Examples /******************************************************************************/ -µBlock.URI = (function() { +µBlock.URI = (( ) => { /******************************************************************************/ -const punycode = self.punycode; - // Favorite regex tool: http://regex101.com/ // Ref: @@ -50,11 +46,8 @@ const reRFC3986 = /^([^:\/?#]+:)?(\/\/[^\/?#]*)?([^?#]*)(\?[^#]*)?(#.*)?/; // Derived const reSchemeFromURI = /^[^:\/?#]+:/; -const reAuthorityFromURI = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; const reOriginFromURI = /^(?:[^:\/?#]+:)\/\/[^\/?#]+/; -const reCommonHostnameFromURL = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//; const rePathFromURI = /^(?:[^:\/?#]+:)?(?:\/\/[^\/?#]*)?([^?#]*)/; -const reMustNormalizeHostname = /[^0-9a-z._-]/; // These are to parse authority field, not parsed by above official regex // IPv6 is seen as an exception: a non-compatible IPv6 is first tried, and @@ -68,12 +61,9 @@ const reHostPortFromAuthority = /^(?:[^@]*@)?([^:]*)(:\d*)?$/; const reIPv6PortFromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]*\])(:\d*)?$/i; const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i; -const reHostFromAuthority = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; -const reIPv6FromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; // Coarse (but fast) tests const reValidHostname = /^([a-z\d]+(-*[a-z\d]+)*)(\.[a-z\d]+(-*[a-z\d])*)*$/; -const reIPAddressNaive = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; /******************************************************************************/ @@ -239,60 +229,13 @@ URI.schemeFromURI = function(uri) { /******************************************************************************/ -URI.authorityFromURI = function(uri) { - const matches = reAuthorityFromURI.exec(uri); - if ( !matches ) { return ''; } - return matches[1].slice(2).toLowerCase(); -}; - -/******************************************************************************/ - -// The most used function, so it better be fast. - -// https://github.com/gorhill/uBlock/issues/1559 -// See http://en.wikipedia.org/wiki/FQDN -// https://bugzilla.mozilla.org/show_bug.cgi?id=1360285 -// Revisit punycode dependency when above issue is fixed in Firefox. - -URI.hostnameFromURI = function(uri) { - let matches = reCommonHostnameFromURL.exec(uri); - if ( matches !== null ) { return matches[1]; } - matches = reAuthorityFromURI.exec(uri); - if ( matches === null ) { return ''; } - const authority = matches[1].slice(2); - // Assume very simple authority (most common case for µBlock) - if ( reHostFromNakedAuthority.test(authority) ) { - return authority.toLowerCase(); - } - matches = reHostFromAuthority.exec(authority); - if ( matches === null ) { - matches = reIPv6FromAuthority.exec(authority); - if ( matches === null ) { return ''; } - } - let hostname = matches[1]; - while ( hostname.endsWith('.') ) { - hostname = hostname.slice(0, -1); - } - if ( reMustNormalizeHostname.test(hostname) ) { - hostname = punycode.toASCII(hostname.toLowerCase()); - } - return hostname; -}; - -/******************************************************************************/ - -URI.domainFromHostname = function(hostname) { - return reIPAddressNaive.test(hostname) ? hostname : psl.getDomain(hostname); -}; +URI.hostnameFromURI = vAPI.hostnameFromURI; +URI.domainFromHostname = vAPI.domainFromHostname; URI.domain = function() { return this.domainFromHostname(this.hostname); }; -// It is expected that there is higher-scoped `publicSuffixList` lingering -// somewhere. Cache it. See . -const psl = publicSuffixList; - /******************************************************************************/ URI.entityFromDomain = function(domain) { diff --git a/src/logger-ui.html b/src/logger-ui.html index 36fabf235..ba9caecbd 100644 --- a/src/logger-ui.html +++ b/src/logger-ui.html @@ -65,6 +65,7 @@
+ @@ -120,6 +121,7 @@
+
CNAME of