From bde3164eb445a4e74acca303ec9fa07f82ba1b1c Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Mon, 23 Nov 2020 08:22:43 -0500 Subject: [PATCH] Add support for `1P`, `3P`, `header=` filter options and other changes New filter options ================== Strict partyness: `1P`, `3P` ---------------------------- The current options 1p/3p are meant to "weakly" match partyness, i.e. a network request is considered 1st-party to its context as long as both the context and the request share the same base domain. The new partyness options are meant to check for strict partyness, i.e. a network request will be considered 1st-party if and only if both the context and the request share the same hostname. For examples: - context: `www.example.org` - request: `www.example.org` - `1p`: yes, `1P`: yes - `3p`: no, `3P`: no - context: `www.example.org` - request: `subdomain.example.org` - `1p`: yes, `1P`: no - `3p`: no, `3P`: yes - context: `www.example.org` - request: `www.example.com` - `1p`: no, `1P`: no - `3p`: yes, `3P`: yes The strict partyness options will be visually emphasized in the editor so as to prevent mistakenly using `1P` or `3P` where weak partyness is meant to be used. Filter on response headers: `header=` ------------------------------------- Currently experimental and under evaluation. Disabled by default, enable by toggling `filterOnHeaders` to `true` in advanced settings. Ability to filter network requests according to whether a specific response header is present and whether it matches or does not match a specific value. For example: *$1p,3P,script,header=via:1\.1\s+google The above filter is meant to block network requests which fullfill all the following conditions: - is weakly 1st-party to the context - is not strictly 1st-party to the context - is of type `script` - has a response HTTP header named `via`, which value matches the regular expression `1\.1\s+google`. The matches are always performed in a case-insensitive manner. The header value is assumed to be a literal regular expression, except for the following special characters: - to anchor to start of string, use leading `|`, not `^` - to anchor to end of string, use trailing `|`, not `$` - to invert the test, use a leading `!` To block a network request if it merely contains a specific HTTP header is just a matter of specifying the header name without a header value: *$1p,3P,script,header=via Generic exception filters can be used to disable specific block `header=` filters, i.e. `@@*$1p,3P,script,header` will override the block `header=` filters given as example above. Dynamic filtering's `allow` rules override block `headers=` filters. Important: It is key that filter authors use as many narrowing filter options as possible when using the `header=` option, and the `header=` option should be used ONLY when other filter options are not sufficient. More documentation justifying the purpose of `header=` option will be provided eventually if ever it is decided to move it from experimental to stable status. To be decided: to restrict usage of this filter option to only uBO's own filter lists or "My filters". Changes ======= Fine tuning `queryprune=` ------------------------- The following changes have been implemented: The special value `*` (i.e. `queryprune=*`) means "remove all query parameters". If the `queryprune=` value is made only of alphanumeric characters (including `_`), the value will be internally converted to regex equivalent `^value=`. This ensures a better future compatibility with AdGuard's `removeparam=`. If the `queryprune=` value starts with `!`, the test will be inverted. This can be used to remove all query parameters EXCEPT those who match the specified value. Other ----- The legacy code to test for spurious CSP reports has been removed. This is no longer an issue ever since uBO redirects to local resources through web accessible resources. Notes ===== The following new and recently added filter options are not compatible with Chromium's manifest v3 changes: - `queryprune=` - `1P` - `3P` - `header=` --- src/css/codemirror.css | 4 +- src/css/themes/default.css | 2 +- src/js/background.js | 3 +- src/js/codemirror/ubo-static-filtering.js | 46 +++- src/js/pagestore.js | 49 +++- src/js/static-filtering-parser.js | 98 +++++--- src/js/static-net-filtering.js | 283 +++++++++++++++++++--- src/js/traffic.js | 134 ++-------- 8 files changed, 410 insertions(+), 209 deletions(-) diff --git a/src/css/codemirror.css b/src/css/codemirror.css index 4bd080e99..29f450368 100644 --- a/src/css/codemirror.css +++ b/src/css/codemirror.css @@ -74,9 +74,9 @@ .cm-s-default .cm-keyword { color: var(--sf-keyword-ink); } -.cm-s-default .cm-regex { +.cm-s-default .cm-notice { text-underline-position: under; - text-decoration-color: var(--sf-regex-ink); + text-decoration-color: var(--sf-notice-ink); text-decoration-style: solid; text-decoration-line: underline; } diff --git a/src/css/themes/default.css b/src/css/themes/default.css index 2b4c08cf7..b9dbb7965 100644 --- a/src/css/themes/default.css +++ b/src/css/themes/default.css @@ -191,7 +191,7 @@ --sf-error-ink: #ff0000; --sf-error-surface: #ff000016; --sf-keyword-ink: var(--purple-60); - --sf-regex-ink: var(--light-gray-60); + --sf-notice-ink: var(--light-gray-60); --sf-tag-ink: #117700; --sf-value-ink: var(--orange-80); --sf-variable-ink: var(--default-ink); diff --git a/src/js/background.js b/src/js/background.js index db905a34b..f4c6f475f 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -61,9 +61,10 @@ const µBlock = (( ) => { // jshint ignore:line debugScriptletInjector: false, disableWebAssembly: false, extensionUpdateForceReload: false, + filterAuthorMode: false, + filterOnHeaders: false, ignoreRedirectFilters: false, ignoreScriptInjectFilters: false, - filterAuthorMode: false, loggerPopupType: 'popup', manualUpdateAssetFetchPeriod: 500, popupFontSize: 'unset', diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js index c4b5d68e6..aec5bb92f 100644 --- a/src/js/codemirror/ubo-static-filtering.js +++ b/src/js/codemirror/ubo-static-filtering.js @@ -212,20 +212,42 @@ CodeMirror.defineMode('ubo-static-filtering', function() { const colorNetOptionSpan = function(stream) { const bits = parser.slices[parserSlot]; - let style; if ( (bits & parser.BITComma) !== 0 ) { - style = 'def strong'; netOptionValueMode = false; - } else if ( netOptionValueMode ) { - return colorNetOptionValueSpan(stream, bits); - } else if ( (bits & parser.BITTilde) !== 0 ) { - style = 'keyword strong'; - } else if ( (bits & parser.BITEqual) !== 0 ) { - netOptionValueMode = true; + stream.pos += parser.slices[parserSlot+2]; + parserSlot += 3; + return 'def strong'; } - stream.pos += parser.slices[parserSlot+2]; - parserSlot += 3; - return style || 'def'; + if ( netOptionValueMode ) { + return colorNetOptionValueSpan(stream, bits); + } + if ( (bits & parser.BITTilde) !== 0 ) { + stream.pos += parser.slices[parserSlot+2]; + parserSlot += 3; + return 'keyword strong'; + } + if ( (bits & parser.BITEqual) !== 0 ) { + netOptionValueMode = true; + stream.pos += parser.slices[parserSlot+2]; + parserSlot += 3; + return 'def'; + } + const to = parser.skipUntil( + parserSlot, + parser.commentSpan.i, + parser.BITComma | parser.BITEqual + ); + if ( + to > parserSlot && + /^[13]P/.test(parser.strFromSlices(parserSlot, to - 3)) + ) { + parserSlot = to; + stream.pos = parser.slices[to+1]; + return 'def notice'; + } + parserSlot = to; + stream.pos = parser.slices[to+1]; + return 'def'; }; const colorNetSpan = function(stream) { @@ -259,7 +281,7 @@ CodeMirror.defineMode('ubo-static-filtering', function() { if ( parser.patternIsRegex() ) { stream.pos = parser.slices[parser.optionsAnchorSpan.i+1]; parserSlot = parser.optionsAnchorSpan.i; - return 'variable regex'; + return 'variable notice'; } if ( (parser.slices[parserSlot] & (parser.BITAsterisk | parser.BITCaret)) !== 0 ) { stream.pos += parser.slices[parserSlot+2]; diff --git a/src/js/pagestore.js b/src/js/pagestore.js index 20198234e..505fe2a2a 100644 --- a/src/js/pagestore.js +++ b/src/js/pagestore.js @@ -272,7 +272,6 @@ const PageStore = class { this.popupBlockedCount = 0; this.largeMediaCount = 0; this.largeMediaTimer = null; - this.internalRedirectionCount = 0; this.allowLargeMediaElementsRegex = undefined; this.extraData.clear(); @@ -668,11 +667,57 @@ const PageStore = class { return result; } + filterOnHeaders(fctxt, headers) { + fctxt.filter = undefined; + + if ( this.getNetFilteringSwitch(fctxt) === false ) { return 0; } + + let result = µb.staticNetFilteringEngine.matchHeaders(fctxt, headers); + if ( result === 0 ) { return 0; } + + const loggerEnabled = µb.logger.enabled; + if ( loggerEnabled ) { + fctxt.filter = µb.staticNetFilteringEngine.toLogData(); + } + + // Dynamic filtering allow rules + // URL filtering + if ( + result === 1 && + µb.sessionURLFiltering.evaluateZ( + fctxt.getTabHostname(), + fctxt.url, + fctxt.type + ) === 2 + ) { + result = 2; + if ( loggerEnabled ) { + fctxt.filter = µb.sessionURLFiltering.toLogData(); + } + } + // Hostname filtering + if ( + result === 1 && + µb.userSettings.advancedUserEnabled && + µb.sessionFirewall.evaluateCellZY( + fctxt.getTabHostname(), + fctxt.getHostname(), + fctxt.type + ) === 2 + ) { + result = 2; + if ( loggerEnabled ) { + fctxt.filter = µb.sessionFirewall.toLogData(); + } + } + + return result; + } + redirectBlockedRequest(fctxt) { if ( µb.hiddenSettings.ignoreRedirectFilters === true ) { return; } const directive = µb.staticNetFilteringEngine.redirectRequest(fctxt); if ( directive === undefined ) { return; } - this.internalRedirectionCount += 1; if ( µb.logger.enabled !== true ) { return; } fctxt.pushFilter(directive.logData()); if ( fctxt.redirectURL === undefined ) { return; } diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index 002178bdf..866fc0ba9 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -346,6 +346,7 @@ const Parser = class { // patternRightAnchorSpan: first slice to right-hand pattern anchor // optionsAnchorSpan: first slice to options anchor // optionsSpan: first slice to options + // commentSpan: first slice to trailing comment analyzeNet() { let islice = this.leftSpaceSpan.len; @@ -369,7 +370,7 @@ const Parser = class { } // Assume no options - this.optionsAnchorSpan.i = this.optionsSpan.i = this.commentSpan.i; + this.optionsAnchorSpan.i = this.optionsSpan.i = this.commentSpan.i; // Assume all is part of pattern this.patternSpan.i = islice; @@ -1900,41 +1901,44 @@ const BITFlavorNetAnchor = BITFlavorNetLeftAnchor | BITFlavorNetRightAnc const OPTTokenMask = 0x000000ff; const OPTTokenInvalid = 0; const OPTToken1p = 1; -const OPTToken3p = 2; -const OPTTokenAll = 3; -const OPTTokenBadfilter = 4; -const OPTTokenCname = 5; -const OPTTokenCsp = 6; -const OPTTokenCss = 7; -const OPTTokenDenyAllow = 8; -const OPTTokenDoc = 9; -const OPTTokenDomain = 10; -const OPTTokenEhide = 11; -const OPTTokenEmpty = 12; -const OPTTokenFont = 13; -const OPTTokenFrame = 14; -const OPTTokenGenericblock = 15; -const OPTTokenGhide = 16; -const OPTTokenImage = 17; -const OPTTokenImportant = 18; -const OPTTokenInlineFont = 19; -const OPTTokenInlineScript = 20; -const OPTTokenMedia = 21; -const OPTTokenMp4 = 22; -const OPTTokenObject = 23; -const OPTTokenOther = 24; -const OPTTokenPing = 25; -const OPTTokenPopunder = 26; -const OPTTokenPopup = 27; -const OPTTokenRedirect = 28; -const OPTTokenRedirectRule = 29; -const OPTTokenQueryprune = 30; -const OPTTokenScript = 31; -const OPTTokenShide = 32; -const OPTTokenXhr = 33; -const OPTTokenWebrtc = 34; -const OPTTokenWebsocket = 35; -const OPTTokenCount = 36; +const OPTToken1pStrict = 2; +const OPTToken3p = 3; +const OPTToken3pStrict = 4; +const OPTTokenAll = 5; +const OPTTokenBadfilter = 6; +const OPTTokenCname = 7; +const OPTTokenCsp = 8; +const OPTTokenCss = 9; +const OPTTokenDenyAllow = 10; +const OPTTokenDoc = 11; +const OPTTokenDomain = 12; +const OPTTokenEhide = 13; +const OPTTokenEmpty = 14; +const OPTTokenFont = 15; +const OPTTokenFrame = 16; +const OPTTokenGenericblock = 17; +const OPTTokenGhide = 18; +const OPTTokenHeader = 19; +const OPTTokenImage = 20; +const OPTTokenImportant = 21; +const OPTTokenInlineFont = 22; +const OPTTokenInlineScript = 23; +const OPTTokenMedia = 24; +const OPTTokenMp4 = 25; +const OPTTokenObject = 26; +const OPTTokenOther = 27; +const OPTTokenPing = 28; +const OPTTokenPopunder = 29; +const OPTTokenPopup = 30; +const OPTTokenRedirect = 31; +const OPTTokenRedirectRule = 32; +const OPTTokenQueryprune = 33; +const OPTTokenScript = 34; +const OPTTokenShide = 35; +const OPTTokenXhr = 36; +const OPTTokenWebrtc = 37; +const OPTTokenWebsocket = 38; +const OPTTokenCount = 39; //const OPTPerOptionMask = 0x0000ff00; const OPTCanNegate = 1 << 8; @@ -1974,9 +1978,11 @@ Parser.prototype.BITHostname = BITHostname; Parser.prototype.BITPeriod = BITPeriod; Parser.prototype.BITDash = BITDash; Parser.prototype.BITHash = BITHash; +Parser.prototype.BITNum = BITNum; Parser.prototype.BITEqual = BITEqual; Parser.prototype.BITQuestion = BITQuestion; Parser.prototype.BITPercent = BITPercent; +Parser.prototype.BITAlpha = BITAlpha; Parser.prototype.BITTilde = BITTilde; Parser.prototype.BITUnicode = BITUnicode; Parser.prototype.BITIgnore = BITIgnore; @@ -1993,7 +1999,10 @@ Parser.prototype.BITFlavorIgnore = BITFlavorIgnore; Parser.prototype.BITFlavorUnsupported = BITFlavorUnsupported; Parser.prototype.BITFlavorError = BITFlavorError; -Parser.prototype.OPTTokenInvalid = OPTTokenInvalid; +Parser.prototype.OPTToken1p = OPTToken1p; +Parser.prototype.OPTToken1pStrict = OPTToken1pStrict; +Parser.prototype.OPTToken3p = OPTToken3p; +Parser.prototype.OPTToken3pStrict = OPTToken3pStrict; Parser.prototype.OPTTokenAll = OPTTokenAll; Parser.prototype.OPTTokenBadfilter = OPTTokenBadfilter; Parser.prototype.OPTTokenCname = OPTTokenCname; @@ -2003,14 +2012,15 @@ Parser.prototype.OPTTokenDoc = OPTTokenDoc; Parser.prototype.OPTTokenDomain = OPTTokenDomain; Parser.prototype.OPTTokenEhide = OPTTokenEhide; Parser.prototype.OPTTokenEmpty = OPTTokenEmpty; -Parser.prototype.OPTToken1p = OPTToken1p; Parser.prototype.OPTTokenFont = OPTTokenFont; Parser.prototype.OPTTokenGenericblock = OPTTokenGenericblock; Parser.prototype.OPTTokenGhide = OPTTokenGhide; +Parser.prototype.OPTTokenHeader = OPTTokenHeader; Parser.prototype.OPTTokenImage = OPTTokenImage; Parser.prototype.OPTTokenImportant = OPTTokenImportant; Parser.prototype.OPTTokenInlineFont = OPTTokenInlineFont; Parser.prototype.OPTTokenInlineScript = OPTTokenInlineScript; +Parser.prototype.OPTTokenInvalid = OPTTokenInvalid; Parser.prototype.OPTTokenMedia = OPTTokenMedia; Parser.prototype.OPTTokenMp4 = OPTTokenMp4; Parser.prototype.OPTTokenObject = OPTTokenObject; @@ -2025,7 +2035,6 @@ Parser.prototype.OPTTokenScript = OPTTokenScript; Parser.prototype.OPTTokenShide = OPTTokenShide; Parser.prototype.OPTTokenCss = OPTTokenCss; Parser.prototype.OPTTokenFrame = OPTTokenFrame; -Parser.prototype.OPTToken3p = OPTToken3p; Parser.prototype.OPTTokenXhr = OPTTokenXhr; Parser.prototype.OPTTokenWebrtc = OPTTokenWebrtc; Parser.prototype.OPTTokenWebsocket = OPTTokenWebsocket; @@ -2045,8 +2054,10 @@ Parser.prototype.OPTNotSupported = OPTNotSupported; const netOptionTokenDescriptors = new Map([ [ '1p', OPTToken1p | OPTCanNegate ], [ 'first-party', OPTToken1p | OPTCanNegate ], + [ '1P', OPTToken1pStrict ], [ '3p', OPTToken3p | OPTCanNegate ], [ 'third-party', OPTToken3p | OPTCanNegate ], + [ '3P', OPTToken3pStrict ], [ 'all', OPTTokenAll | OPTNetworkType | OPTNonCspableType ], [ 'badfilter', OPTTokenBadfilter ], [ 'cname', OPTTokenCname | OPTAllowOnly | OPTModifierType ], @@ -2066,6 +2077,7 @@ const netOptionTokenDescriptors = new Map([ [ 'genericblock', OPTTokenGenericblock | OPTNotSupported ], [ 'ghide', OPTTokenGhide | OPTNonNetworkType | OPTNonCspableType | OPTNonRedirectableType ], [ 'generichide', OPTTokenGhide | OPTNonNetworkType | OPTNonCspableType | OPTNonRedirectableType ], + [ 'header', OPTTokenHeader | OPTMustAssign | OPTAllowMayAssign | OPTNonCspableType | OPTNonRedirectableType ], [ 'image', OPTTokenImage | OPTCanNegate | OPTNetworkType | OPTModifiableType | OPTRedirectableType | OPTNonCspableType ], [ 'important', OPTTokenImportant | OPTBlockOnly ], [ 'inline-font', OPTTokenInlineFont | OPTNonNetworkType | OPTCanNegate | OPTNonCspableType | OPTNonRedirectableType ], @@ -2097,8 +2109,10 @@ Parser.prototype.netOptionTokenDescriptors = Parser.netOptionTokenIds = new Map([ [ '1p', OPTToken1p ], [ 'first-party', OPTToken1p ], + [ '1P', OPTToken1pStrict ], [ '3p', OPTToken3p ], [ 'third-party', OPTToken3p ], + [ '3P', OPTToken3pStrict ], [ 'all', OPTTokenAll ], [ 'badfilter', OPTTokenBadfilter ], [ 'cname', OPTTokenCname ], @@ -2118,6 +2132,7 @@ Parser.netOptionTokenIds = new Map([ [ 'genericblock', OPTTokenGenericblock ], [ 'ghide', OPTTokenGhide ], [ 'generichide', OPTTokenGhide ], + [ 'header', OPTTokenHeader ], [ 'image', OPTTokenImage ], [ 'important', OPTTokenImportant ], [ 'inline-font', OPTTokenInlineFont ], @@ -2145,7 +2160,9 @@ Parser.netOptionTokenIds = new Map([ Parser.netOptionTokenNames = new Map([ [ OPTToken1p, '1p' ], + [ OPTToken1pStrict, '1P' ], [ OPTToken3p, '3p' ], + [ OPTToken3pStrict, '3P' ], [ OPTTokenAll, 'all' ], [ OPTTokenBadfilter, 'badfilter' ], [ OPTTokenCname, 'cname' ], @@ -2160,6 +2177,7 @@ Parser.netOptionTokenNames = new Map([ [ OPTTokenFont, 'font' ], [ OPTTokenGenericblock, 'genericblock' ], [ OPTTokenGhide, 'generichide' ], + [ OPTTokenHeader, 'header' ], [ OPTTokenImage, 'image' ], [ OPTTokenImportant, 'important' ], [ OPTTokenInlineFont, 'inline-font' ], @@ -2300,6 +2318,8 @@ const NetOptionsIterator = class { } // Keep track of which options are present: any given option can // appear only once. + // TODO: might need to make an exception for `header=` option so as + // to allow filters which need to match more than one header. const tokenId = descriptor & OPTTokenMask; if ( tokenId !== OPTTokenInvalid ) { if ( this.tokenPos[tokenId] !== -1 ) { diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index c5d480c3b..3077d346e 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -32,31 +32,34 @@ const µb = µBlock; // fedcba9876543210 -// | | || | -// | | || | -// | | || | -// | | || | -// | | || +---- bit 0- 1: block=0, allow=1, block important=2 -// | | |+------ bit 2: modifier -// | | +------- bit 3- 4: party [0-3] -// | +--------- bit 5- 9: type [0-31] -// +-------------- bit 10-15: unused -const CategoryCount = 1 << 0xa; // shift left to first unused bit +// || | || | +// || | || | +// || | || | +// || | || | +// || | || +---- bit 0- 1: block=0, allow=1, block important=2 +// || | |+------ bit 2: modifier +// || | +------- bit 3- 4: party [0-3] +// || +--------- bit 5- 9: type [0-31] +// |+-------------- bit 10-15: unused +// +--------------- bit 16: headers-based filters -const RealmBitsMask = 0b0000000111; -const ActionBitsMask = 0b0000000011; -const TypeBitsMask = 0b1111100000; +const CategoryCount = 1 << 0xb; // shift left to first unused bit + +const RealmBitsMask = 0b00000000111; +const ActionBitsMask = 0b00000000011; +const TypeBitsMask = 0b01111100000; const TypeBitsOffset = 5; -const BlockAction = 0b0000000000; -const AllowAction = 0b0000000001; -const Important = 0b0000000010; +const BlockAction = 0b00000000000; +const AllowAction = 0b00000000001; +const Important = 0b00000000010; const BlockImportant = BlockAction | Important; -const ModifyAction = 0b0000000100; -const AnyParty = 0b0000000000; -const FirstParty = 0b0000001000; -const ThirdParty = 0b0000010000; -const AllParties = 0b0000011000; +const ModifyAction = 0b00000000100; +const AnyParty = 0b00000000000; +const FirstParty = 0b00000001000; +const ThirdParty = 0b00000010000; +const AllParties = 0b00000011000; +const Headers = 0b10000000000; const typeNameToTypeValue = { 'no_type': 0 << TypeBitsOffset, @@ -166,6 +169,28 @@ const $docEntity = { }, }; +const $httpHeaders = { + init(headers) { + this.headers = headers; + this.parsed.clear(); + }, + reset() { + this.headers = []; + this.parsed.clear(); + }, + lookup(name) { + if ( this.parsed.size === 0 ) { + for ( let i = 0, n = this.headers.length; i < n; i++ ) { + const { name, value } = this.headers[i]; + this.parsed.set(name, value); + } + } + return this.parsed.get(name); + }, + headers: [], + parsed: new Map(), +}; + /******************************************************************************/ // Local helpers @@ -2310,6 +2335,105 @@ const FilterBucketOfOriginHits = class extends FilterBucket { registerFilterClass(FilterBucketOfOriginHits); +/******************************************************************************/ + +const FilterStrictParty = class { + constructor(not) { + this.not = not; + } + + // TODO: diregard `www.`? + match() { + return ($requestHostname === $docHostname) !== this.not; + } + + logData(details) { + details.options.push(this.not ? '3P' : '1P'); + } + + toSelfie() { + return [ this.fid, this.not ]; + } + + static compile(details) { + return [ FilterStrictParty.fid, details.strictParty < 0 ]; + } + + static fromCompiled(args) { + return new FilterStrictParty(args[1]); + } + + static fromSelfie(args) { + return new FilterStrictParty(args[1]); + } + + static keyFromArgs(args) { + return `${args[1]}`; + } +}; + +registerFilterClass(FilterStrictParty); + +/******************************************************************************/ + +const FilterOnHeaders = class { + constructor(headerOpt) { + this.headerOpt = headerOpt; + if ( headerOpt !== '' ) { + let pos = headerOpt.indexOf(':'); + if ( pos === -1 ) { pos = headerOpt.length; } + this.name = headerOpt.slice(0, pos); + this.value = headerOpt.slice(pos + 1); + this.not = this.value.charCodeAt(0) === 0x21 /* '!' */; + if ( this.not ) { this.value.slice(1); } + } else { + this.name = this.value = ''; + this.not = false; + } + this.reValue = null; + } + + match() { + if ( this.name === '' ) { return true; } + const value = $httpHeaders.lookup(this.name); + if ( value === undefined ) { return false; } + if ( this.value === '' ) { return true; } + if ( this.reValue === null ) { + let reText = this.value; + if ( reText.startsWith('|') ) { reText = '^' + reText.slice(1); } + if ( reText.endsWith('|') ) { reText = reText.slice(0, -1) + '$'; } + this.reValue = new RegExp(reText, 'i'); + } + return this.reValue.test(value) !== this.not; + } + + logData(details) { + let opt = 'header'; + if ( this.headerOpt !== '' ) { + opt += `=${this.headerOpt}`; + } + details.options.push(opt); + } + + toSelfie() { + return [ this.fid, this.headerOpt ]; + } + + static compile(details) { + return [ FilterOnHeaders.fid, details.headerOpt ]; + } + + static fromCompiled(args) { + return new FilterOnHeaders(args[1]); + } + + static fromSelfie(args) { + return new FilterOnHeaders(args[1]); + } +}; + +registerFilterClass(FilterOnHeaders); + /******************************************************************************/ /******************************************************************************/ @@ -2656,10 +2780,13 @@ const FilterParser = class { this.invalid = false; this.pattern = ''; this.party = AnyParty; + this.hasOptionUnits = false; this.domainOpt = ''; this.denyallowOpt = ''; + this.headerOpt = undefined; this.isPureHostname = false; this.isRegex = false; + this.strictParty = 0; this.token = '*'; this.tokenHash = this.noTokenHash; this.tokenBeg = 0; @@ -2731,6 +2858,7 @@ const FilterParser = class { } else if ( this.action === AllowAction ) { this.modifyValue = ''; } + this.hasOptionUnits = true; return true; } @@ -2740,9 +2868,17 @@ const FilterParser = class { case parser.OPTToken1p: this.parsePartyOption(true, not); break; + case parser.OPTToken1pStrict: + this.strictParty = this.strictParty === -1 ? 0 : 1; + this.hasOptionUnits = true; + break; case parser.OPTToken3p: this.parsePartyOption(false, not); break; + case parser.OPTToken3pStrict: + this.strictParty = this.strictParty === 1 ? 0 : -1; + this.hasOptionUnits = true; + break; case parser.OPTTokenAll: this.parseTypeOption(-1); break; @@ -2769,10 +2905,12 @@ const FilterParser = class { this.domainOptList ); if ( this.domainOpt === '' ) { return false; } + this.hasOptionUnits = true; break; case parser.OPTTokenDenyAllow: this.denyallowOpt = this.parseHostnameList(parser, val, 0b0000); if ( this.denyallowOpt === '' ) { return false; } + this.hasOptionUnits = true; break; // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ // Add support for `elemhide`. Rarely used but it happens. @@ -2780,6 +2918,10 @@ const FilterParser = class { this.parseTypeOption(parser.OPTTokenShide, not); this.parseTypeOption(parser.OPTTokenGhide, not); break; + case parser.OPTTokenHeader: + this.headerOpt = val !== undefined ? val : ''; + this.hasOptionUnits = true; + break; case parser.OPTTokenImportant: if ( this.action === AllowAction ) { return false; } this.action = BlockImportant; @@ -2790,11 +2932,13 @@ const FilterParser = class { if ( this.modifyType !== undefined ) { return false; } this.modifyType = parser.OPTTokenRedirect; this.modifyValue = 'empty'; + this.hasOptionUnits = true; break; case parser.OPTTokenMp4: if ( this.modifyType !== undefined ) { return false; } this.modifyType = parser.OPTTokenRedirect; this.modifyValue = 'noopmp4-1s'; + this.hasOptionUnits = true; break; case parser.OPTTokenQueryprune: case parser.OPTTokenRedirect: @@ -3018,9 +3162,16 @@ const FilterParser = class { } } + isJustPattern() { + return this.hasOptionUnits === false; + } + isJustOrigin() { - return this.isRegex === false && + return this.hasOptionUnits && + this.isRegex === false && this.modifyType === undefined && + this.strictParty === 0 && + this.headerOpt === undefined && this.denyallowOpt === '' && this.domainOpt !== '' && ( this.pattern === '*' || ( @@ -3410,12 +3561,7 @@ FilterContainer.prototype.compile = function(parser, writer) { // Pure hostnames, use more efficient dictionary lookup // https://github.com/chrisaljoudi/uBlock/issues/665 // Create a dict keyed on request type etc. - if ( - parsed.isPureHostname && - parsed.domainOpt === '' && - parsed.denyallowOpt === '' && - parsed.modifyType === undefined - ) { + if ( parsed.isPureHostname && parsed.isJustPattern() ) { parsed.tokenHash = this.dotTokenHash; this.compileToAtomicFilter(parsed, parsed.pattern, writer); return true; @@ -3483,6 +3629,11 @@ FilterContainer.prototype.compile = function(parser, writer) { units.push(FilterAnchorRight.compile()); } + // Strict partiness + if ( parsed.strictParty !== 0 ) { + units.push(FilterStrictParty.compile(parsed)); + } + // Origin if ( parsed.domainOpt !== '' ) { filterOrigin.compile( @@ -3497,6 +3648,12 @@ FilterContainer.prototype.compile = function(parser, writer) { units.push(FilterDenyAllow.compile(parsed)); } + // Header + if ( parsed.headerOpt !== undefined ) { + units.push(FilterOnHeaders.compile(parsed)); + parsed.action |= Headers; + } + // Modifier // // IMPORTANT: the modifier unit MUST always appear first in a sequence. @@ -3958,6 +4115,39 @@ FilterContainer.prototype.matchString = function(fctxt, modifiers = 0) { /******************************************************************************/ +FilterContainer.prototype.matchHeaders = function(fctxt, headers) { + const typeValue = typeNameToTypeValue[fctxt.type] || otherTypeBitValue; + const partyBits = fctxt.is3rdPartyToDoc() ? ThirdParty : FirstParty; + + // Prime tokenizer: we get a normalized URL in return. + $requestURL = urlTokenizer.setURL(fctxt.url); + this.$filterUnit = 0; + + // These registers will be used by various filters + $docHostname = fctxt.getDocHostname(); + $docDomain = fctxt.getDocDomain(); + $docEntity.reset(); + $requestHostname = fctxt.getHostname(); + $httpHeaders.init(headers); + + let r = 0; + if ( this.realmMatchString(Headers | BlockImportant, typeValue, partyBits) ) { + r = 1; + } + if ( r !== 1 && this.realmMatchString(Headers | BlockAction, typeValue, partyBits) ) { + r = 1; + if ( r === 1 && this.realmMatchString(Headers | AllowAction, typeValue, partyBits) ) { + r = 2; + } + } + + $httpHeaders.reset(); + + return r; +}; + +/******************************************************************************/ + FilterContainer.prototype.redirectRequest = function(fctxt) { const directives = this.matchAndFetchModifiers(fctxt, 'redirect-rule'); // No directive is the most common occurrence. @@ -4036,12 +4226,14 @@ FilterContainer.prototype.filterQuery = function(fctxt) { break; } if ( modifier.cache === undefined ) { - modifier.cache = this.parseFilterPruneValue(modifier.value); + this.parseFilterPruneValue(modifier); } - const re = modifier.cache; + const { all, not, re } = modifier.cache; let filtered = false; for ( const [ key, value ] of params ) { - if ( re.test(`${key}=${value}`) === false ) { continue; } + if ( all !== true && re.test(`${key}=${value}`) === not ) { + continue; + } if ( isException === false ) { params.delete(key); } @@ -4061,15 +4253,27 @@ FilterContainer.prototype.filterQuery = function(fctxt) { return out; }; -FilterContainer.prototype.parseFilterPruneValue = function(rawValue) { - let retext = rawValue; - if ( retext.startsWith('|') ) { retext = `^${retext.slice(1)}`; } - if ( retext.endsWith('|') ) { retext = `${retext.slice(0,-1)}$`; } - try { - return new RegExp(retext); - } catch(ex) { +FilterContainer.prototype.parseFilterPruneValue = function(modifier) { + const cache = {}; + let retext = modifier.value; + if ( retext === '*' ) { + cache.all = true; + } else { + cache.not = retext.charCodeAt(0) === 0x21 /* '!' */; + if ( cache.not ) { retext = retext.slice(1); } + if ( /^\w+$/.test(retext) ) { + retext = `^${retext}=`; + } else { + if ( retext.startsWith('|') ) { retext = `^${retext.slice(1)}`; } + if ( retext.endsWith('|') ) { retext = `${retext.slice(0,-1)}$`; } + } + try { + cache.re = new RegExp(retext, 'i'); + } catch(ex) { + cache.re = /.^/; + } } - return /.^/; + modifier.cache = cache; }; /******************************************************************************/ @@ -4190,6 +4394,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) { this.matchAndFetchModifiers(fctxt, 'csp'); } + this.matchHeaders(fctxt, []); } else { this.redirectRequest(fctxt); } diff --git a/src/js/traffic.js b/src/js/traffic.js index 224b956cd..17cdb615a 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -371,92 +371,6 @@ const onBeforeBehindTheSceneRequest = function(fctxt) { /******************************************************************************/ -// https://github.com/gorhill/uBlock/issues/3140 - -const onBeforeMaybeSpuriousCSPReport = (function() { - let textDecoder; - - return function(details) { - const fctxt = µBlock.filteringContext.fromWebrequestDetails(details); - - // Ignore behind-the-scene requests. - if ( fctxt.tabId < 0 ) { return; } - - // Lookup the page store associated with this tab id. - const pageStore = µBlock.pageStoreFromTabId(fctxt.tabId); - if ( pageStore === null ) { return; } - - // If uBO is disabled for the page, it can't possibly causes CSP - // reports to be triggered. - if ( pageStore.getNetFilteringSwitch() === false ) { return; } - - // A resource was redirected to a neutered one? - // TODO: mind injected scripts/styles as well. - if ( pageStore.internalRedirectionCount === 0 ) { return; } - - if ( - textDecoder === undefined && - typeof self.TextDecoder === 'function' - ) { - textDecoder = new TextDecoder(); - } - - // Find out whether the CSP report is a potentially spurious CSP report. - // If from this point on we are unable to parse the CSP report data, - // the safest assumption to protect users is to assume the CSP report - // is spurious. - if ( - textDecoder !== undefined && - details.method === 'POST' - ) { - const raw = details.requestBody && details.requestBody.raw; - if ( - Array.isArray(raw) && - raw.length !== 0 && - raw[0] instanceof Object && - raw[0].bytes instanceof ArrayBuffer - ) { - let data; - try { - data = JSON.parse(textDecoder.decode(raw[0].bytes)); - } catch (ex) { - } - if ( data instanceof Object ) { - const report = data['csp-report']; - if ( report instanceof Object ) { - const blocked = - report['blocked-uri'] || report['blockedURI']; - const validBlocked = typeof blocked === 'string'; - const source = - report['source-file'] || report['sourceFile']; - const validSource = typeof source === 'string'; - if ( - (validBlocked || validSource) && - (!validBlocked || !blocked.startsWith('data')) && - (!validSource || !source.startsWith('data')) - ) { - return; - } - } - } - } - } - - // At this point, we have a potentially spurious CSP report. - - if ( µBlock.logger.enabled ) { - fctxt.setRealm('network') - .setType('csp_report') - .setFilter({ result: 1, source: 'global', raw: 'no-spurious-csp-report' }) - .toLogger(); - } - - return { cancel: true }; - }; -})(); - -/******************************************************************************/ - // To handle: // - Media elements larger than n kB // - Scriptlet injection (requires ability to modify response body) @@ -485,16 +399,30 @@ const onHeadersReceived = function(details) { if ( pageStore.getNetFilteringSwitch(fctxt) === false ) { return; } if ( fctxt.itype === fctxt.IMAGE || fctxt.itype === fctxt.MEDIA ) { - return foilLargeMediaElement(details, fctxt, pageStore); + const result = foilLargeMediaElement(details, fctxt, pageStore); + if ( result !== undefined ) { return result; } } - if ( isRootDoc === false && fctxt.itype !== fctxt.SUB_FRAME ) { return; } - // Keep in mind response headers will be modified in-place if needed, so // `details.responseHeaders` will always point to the modified response // headers. const responseHeaders = details.responseHeaders; + if ( isRootDoc === false && µb.hiddenSettings.filterOnHeaders === true ) { + const result = pageStore.filterOnHeaders(fctxt, responseHeaders); + if ( result !== 0 ) { + if ( µb.logger.enabled ) { + fctxt.setRealm('network').toLogger(); + } + if ( result === 1 ) { + pageStore.journalAddRequest(fctxt.getHostname(), 1); + return { cancel: true }; + } + } + } + + if ( isRootDoc === false && fctxt.itype !== fctxt.SUB_FRAME ) { return; } + // https://github.com/gorhill/uBlock/issues/2813 // Disable the blocking of large media elements if the document is itself // a media element: the resource was not prevented from loading so no @@ -1083,41 +1011,21 @@ return { vAPI.net = new vAPI.Net(); vAPI.net.suspend(); - return function() { + return ( ) => { vAPI.net.setSuspendableListener(onBeforeRequest); vAPI.net.addListener( 'onHeadersReceived', onHeadersReceived, - { - types: [ - 'main_frame', - 'sub_frame', - 'image', - 'media', - 'xmlhttprequest', - ], - urls: [ 'http://*/*', 'https://*/*' ], - }, + { urls: [ 'http://*/*', 'https://*/*' ] }, [ 'blocking', 'responseHeaders' ] ); - if ( vAPI.net.validTypes.has('csp_report') ) { - vAPI.net.addListener( - 'onBeforeRequest', - onBeforeMaybeSpuriousCSPReport, - { - types: [ 'csp_report' ], - urls: [ 'http://*/*', 'https://*/*' ] - }, - [ 'blocking', 'requestBody' ] - ); - } vAPI.net.unsuspend(true); }; })(), - strictBlockBypass: function(hostname) { + strictBlockBypass: hostname => { strictBlockBypasser.bypass(hostname); - } + }, }; /******************************************************************************/