diff --git a/dist/firefox/updates.json b/dist/firefox/updates.json index d23a8da59..63a7f9422 100644 --- a/dist/firefox/updates.json +++ b/dist/firefox/updates.json @@ -3,9 +3,9 @@ "uBlock0@raymondhill.net": { "updates": [ { - "version": "1.37.1.1", + "version": "1.37.1.2", "browser_specific_settings": { "gecko": { "strict_min_version": "57" } }, - "update_link": "https://github.com/gorhill/uBlock/releases/download/1.37.1b1/uBlock0_1.37.1b1.firefox.signed.xpi" + "update_link": "https://github.com/gorhill/uBlock/releases/download/1.37.1b2/uBlock0_1.37.1b2.firefox.signed.xpi" } ] } diff --git a/dist/version b/dist/version index 29ae45efb..e56539b5d 100644 --- a/dist/version +++ b/dist/version @@ -1 +1 @@ -1.37.1.1 +1.37.1.2 diff --git a/platform/browser/main.js b/platform/browser/main.js new file mode 100644 index 000000000..aaacfe9a2 --- /dev/null +++ b/platform/browser/main.js @@ -0,0 +1,125 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import './lib/publicsuffixlist/publicsuffixlist.js'; +import './lib/punycode.js'; + +import globals from './js/globals.js'; +import { FilteringContext } from './js/filtering-context.js'; +import { LineIterator } from './js/text-iterators.js'; +import { StaticFilteringParser } from './js/static-filtering-parser.js'; +import { staticNetFilteringEngine } from './js/static-net-filtering.js'; + +import { + CompiledListReader, + CompiledListWriter +} from './js/static-filtering-io.js'; + +/******************************************************************************/ + +function compileList(rawText, writer) { + const lineIter = new LineIterator(rawText); + const parser = new StaticFilteringParser(true); + + parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); + + while ( lineIter.eot() === false ) { + let line = lineIter.next(); + + while ( line.endsWith(' \\') ) { + if ( lineIter.peek(4) !== ' ' ) { break; } + line = line.slice(0, -2).trim() + lineIter.next().trim(); + } + parser.analyze(line); + + if ( parser.shouldIgnore() ) { continue; } + if ( parser.category !== parser.CATStaticNetFilter ) { continue; } + if ( parser.patternHasUnicode() && parser.toASCII() === false ) { + continue; + } + if ( staticNetFilteringEngine.compile(parser, writer) ) { continue; } + if ( staticNetFilteringEngine.error !== undefined ) { + console.info(JSON.stringify({ + realm: 'message', + type: 'error', + text: staticNetFilteringEngine.error + })); + } + } + + return writer.toString(); +} + +function applyList(name, raw) { + const writer = new CompiledListWriter(); + writer.properties.set('name', name); + const compiled = compileList(raw, writer); + const reader = new CompiledListReader(compiled); + staticNetFilteringEngine.fromCompiled(reader); +} + +function enableWASM(path) { + return Promise.all([ + globals.publicSuffixList.enableWASM(`${path}/lib/publicsuffixlist`), + staticNetFilteringEngine.enableWASM(`${path}/js`), + ]); +} + +function pslInit(raw) { + if ( typeof raw !== 'string' || raw.trim() === '' ) { + console.info('Unable to populate public suffix list'); + return; + } + globals.publicSuffixList.parse(raw, globals.punycode.toASCII); + console.info('Public suffix list populated'); +} + +function restart(lists) { + // Remove all filters + reset(); + + if ( Array.isArray(lists) && lists.length !== 0 ) { + // Populate filtering engine with filter lists + for ( const { name, raw } of lists ) { + applyList(name, raw); + } + // Commit changes + staticNetFilteringEngine.freeze(); + staticNetFilteringEngine.optimize(); + } + + return staticNetFilteringEngine; +} + +function reset() { + staticNetFilteringEngine.reset(); +} + +export { + FilteringContext, + enableWASM, + pslInit, + restart, +}; diff --git a/platform/browser/test.html b/platform/browser/test.html new file mode 100644 index 000000000..32b1aba8e --- /dev/null +++ b/platform/browser/test.html @@ -0,0 +1,71 @@ + + + + +uBO Static Network Filtering Engine + + + + + diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 9a515985a..f9ab901be 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -70,7 +70,7 @@ }, "incognito": "split", "manifest_version": 2, - "minimum_chrome_version": "55.0", + "minimum_chrome_version": "61.0", "name": "uBlock Origin", "options_ui": { "page": "dashboard.html", diff --git a/platform/common/vapi-background.js b/platform/common/vapi-background.js index 489f231fe..b9f7e2721 100644 --- a/platform/common/vapi-background.js +++ b/platform/common/vapi-background.js @@ -26,12 +26,6 @@ /******************************************************************************/ -{ -// >>>>> start of local scope - -/******************************************************************************/ -/******************************************************************************/ - const browser = self.browser; const manifest = browser.runtime.getManifest(); @@ -1719,9 +1713,3 @@ vAPI.cloud = (( ) => { })(); /******************************************************************************/ -/******************************************************************************/ - -// <<<<< end of local scope -} - -/******************************************************************************/ diff --git a/platform/common/vapi-common.js b/platform/common/vapi-common.js index e8de131d2..184260681 100644 --- a/platform/common/vapi-common.js +++ b/platform/common/vapi-common.js @@ -100,59 +100,6 @@ vAPI.webextFlavor = { /******************************************************************************/ -{ - const punycode = self.punycode; - const reCommonHostnameFromURL = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//; - const reAuthorityFromURI = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; - const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i; - const reHostFromAuthority = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; - const reIPv6FromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; - const reMustNormalizeHostname = /[^0-9a-z._-]/; - - vAPI.hostnameFromURI = function(uri) { - let matches = reCommonHostnameFromURL.exec(uri); - if ( matches !== null ) { return matches[1]; } - matches = reAuthorityFromURI.exec(uri); - if ( matches === null ) { return ''; } - const authority = matches[1].slice(2); - if ( reHostFromNakedAuthority.test(authority) ) { - return authority.toLowerCase(); - } - matches = reHostFromAuthority.exec(authority); - if ( matches === null ) { - matches = reIPv6FromAuthority.exec(authority); - if ( matches === null ) { return ''; } - } - let hostname = matches[1]; - while ( hostname.endsWith('.') ) { - hostname = hostname.slice(0, -1); - } - if ( reMustNormalizeHostname.test(hostname) ) { - hostname = punycode.toASCII(hostname.toLowerCase()); - } - return hostname; - }; - - const reHostnameFromNetworkURL = - /^(?:http|ws|ftp)s?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])(?::\d+)?\//; - - vAPI.hostnameFromNetworkURL = function(url) { - const matches = reHostnameFromNetworkURL.exec(url); - return matches !== null ? matches[1] : ''; - }; - - const psl = self.publicSuffixList; - const reIPAddressNaive = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; - - vAPI.domainFromHostname = function(hostname) { - return reIPAddressNaive.test(hostname) - ? hostname - : psl.getDomain(hostname); - }; -} - -/******************************************************************************/ - vAPI.download = function(details) { if ( !details.url ) { return; } const a = document.createElement('a'); diff --git a/platform/firefox/vapi-background-ext.js b/platform/firefox/vapi-background-ext.js index c083cabfe..69609f216 100644 --- a/platform/firefox/vapi-background-ext.js +++ b/platform/firefox/vapi-background-ext.js @@ -25,30 +25,17 @@ /******************************************************************************/ +import { + domainFromHostname, + hostnameFromNetworkURL, +} from './uri-utils.js'; + +/******************************************************************************/ + (( ) => { // https://github.com/uBlockOrigin/uBlock-issues/issues/407 if ( vAPI.webextFlavor.soup.has('firefox') === false ) { return; } - // https://github.com/gorhill/uBlock/issues/2950 - // Firefox 56 does not normalize URLs to ASCII, uBO must do this itself. - // https://bugzilla.mozilla.org/show_bug.cgi?id=945240 - const evalMustPunycode = ( ) => { - return vAPI.webextFlavor.soup.has('firefox') && - vAPI.webextFlavor.major < 57; - }; - - let mustPunycode = evalMustPunycode(); - - // The real actual webextFlavor value may not be set in stone, so listen - // for possible future changes. - window.addEventListener('webextFlavor', ( ) => { - mustPunycode = evalMustPunycode(); - }, { once: true }); - - const punycode = self.punycode; - const reAsciiHostname = /^https?:\/\/[0-9a-z_.:@-]+[/?#]/; - const parsedURL = new URL('about:blank'); - // Canonical name-uncloaking feature. let cnameUncloakEnabled = browser.dns instanceof Object; let cnameUncloakProxied = false; @@ -144,14 +131,6 @@ } } normalizeDetails(details) { - if ( mustPunycode && !reAsciiHostname.test(details.url) ) { - parsedURL.href = details.url; - details.url = details.url.replace( - parsedURL.hostname, - punycode.toASCII(parsedURL.hostname) - ); - } - const type = details.type; if ( type === 'imageset' ) { @@ -231,7 +210,7 @@ if ( cname !== '' && this.cnameIgnore1stParty && - vAPI.domainFromHostname(cname) === vAPI.domainFromHostname(hn) + domainFromHostname(cname) === domainFromHostname(hn) ) { cname = ''; } @@ -284,7 +263,7 @@ ) { return; } - const hn = vAPI.hostnameFromNetworkURL(details.url); + const hn = hostnameFromNetworkURL(details.url); const cname = this.cnames.get(hn); if ( cname === '' ) { return; } if ( cname !== undefined ) { diff --git a/platform/nodejs/main.js b/platform/nodejs/main.js new file mode 100644 index 000000000..a104fae84 --- /dev/null +++ b/platform/nodejs/main.js @@ -0,0 +1,127 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import './lib/punycode.js'; +import './lib/publicsuffixlist/publicsuffixlist.js'; + +import globals from './js/globals.js'; +import { FilteringContext } from './js/filtering-context.js'; +import { LineIterator } from './js/text-iterators.js'; +import { StaticFilteringParser } from './js/static-filtering-parser.js'; +import { staticNetFilteringEngine } from './js/static-net-filtering.js'; + +import { + CompiledListReader, + CompiledListWriter, +} from './js/static-filtering-io.js'; + +/******************************************************************************/ + +function compileList(rawText, writer) { + const lineIter = new LineIterator(rawText); + const parser = new StaticFilteringParser(true); + + parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); + + while ( lineIter.eot() === false ) { + let line = lineIter.next(); + + while ( line.endsWith(' \\') ) { + if ( lineIter.peek(4) !== ' ' ) { break; } + line = line.slice(0, -2).trim() + lineIter.next().trim(); + } + parser.analyze(line); + + if ( parser.shouldIgnore() ) { continue; } + if ( parser.category !== parser.CATStaticNetFilter ) { continue; } + if ( parser.patternHasUnicode() && parser.toASCII() === false ) { + continue; + } + if ( staticNetFilteringEngine.compile(parser, writer) ) { continue; } + if ( staticNetFilteringEngine.error !== undefined ) { + console.info(JSON.stringify({ + realm: 'message', + type: 'error', + text: staticNetFilteringEngine.error + })); + } + } + + return writer.toString(); +} + +function applyList(name, raw) { + const writer = new CompiledListWriter(); + writer.properties.set('name', name); + const compiled = compileList(raw, writer); + const reader = new CompiledListReader(compiled); + staticNetFilteringEngine.fromCompiled(reader); +} + +function enableWASM(path) { + return Promise.all([ + globals.publicSuffixList.enableWASM(`${path}/lib/publicsuffixlist`), + staticNetFilteringEngine.enableWASM(`${path}/js`), + ]); +} + +function pslInit(raw) { + if ( typeof raw !== 'string' || raw.trim() === '' ) { + console.info('Unable to populate public suffix list'); + return; + } + globals.publicSuffixList.parse(raw, globals.punycode.toASCII); + console.info('Public suffix list populated'); +} + +function restart(lists) { + // Remove all filters + reset(); + + if ( Array.isArray(lists) && lists.length !== 0 ) { + // Populate filtering engine with filter lists + for ( const { name, raw } of lists ) { + applyList(name, raw); + } + // Commit changes + staticNetFilteringEngine.freeze(); + staticNetFilteringEngine.optimize(); + } + + console.info('Static network filtering engine populated'); + + return staticNetFilteringEngine; +} + +function reset() { + staticNetFilteringEngine.reset(); +} + +export { + FilteringContext, + enableWASM, + pslInit, + restart, +}; diff --git a/platform/nodejs/package.json b/platform/nodejs/package.json new file mode 100644 index 000000000..37dbc7755 --- /dev/null +++ b/platform/nodejs/package.json @@ -0,0 +1,25 @@ +{ + "name": "uBO-snfe", + "version": "0.1.0", + "description": "To create a working instance of uBlock's static network filtering engine", + "type": "module", + "main": "main.js", + "scripts": { + "test": "node test.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/gorhill/uBlock.git" + }, + "keywords": [ + "uBlock", + "uBO", + "adblock" + ], + "author": "Raymond Hill", + "license": "GPL-3.0-or-later", + "bugs": { + "url": "https://github.com/gorhill/uBlock/issues" + }, + "homepage": "https://github.com/gorhill/uBlock#readme" +} diff --git a/platform/nodejs/test.js b/platform/nodejs/test.js new file mode 100644 index 000000000..0d544e9d1 --- /dev/null +++ b/platform/nodejs/test.js @@ -0,0 +1,107 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals process */ + +'use strict'; + +/******************************************************************************/ + +import { readFile } from 'fs'; + +import { + FilteringContext, + pslInit, + restart, +} from './main.js'; + +/******************************************************************************/ + +function fetch(path) { + return new Promise((resolve, reject) => { + readFile(path, 'utf8', (err, data) => { + if ( err ) { + reject(err); + } else { + resolve(data); + } + }); + }); +} + +(async ( ) => { + /* + * WASM require fetch(), not present in Node + try { + await enableWASM('//ublock/dist/build/uBlock0.nodejs'); + } catch(ex) { + } + */ + + await fetch('./data/effective_tld_names.dat').then(pslRaw => { + pslInit(pslRaw); + }); + + const snfe = await Promise.all([ + fetch('./data/easylist.txt'), + fetch('./data/easyprivacy.txt'), + ]).then(rawLists => { + return restart([ + { name: 'easylist', raw: rawLists[0] }, + { name: 'easyprivacy', raw: rawLists[1] }, + ]); + }); + + // Reuse filtering context: it's what uBO does + const fctxt = new FilteringContext(); + + // Tests + // Not blocked + fctxt.setDocOriginFromURL('https://www.bloomberg.com/'); + fctxt.setURL('https://www.bloomberg.com/tophat/assets/v2.6.1/that.css'); + fctxt.setType('stylesheet'); + if ( snfe.matchRequest(fctxt) !== 0 ) { + console.log(snfe.toLogData()); + } + + // Blocked + fctxt.setDocOriginFromURL('https://www.bloomberg.com/'); + fctxt.setURL('https://securepubads.g.doubleclick.net/tag/js/gpt.js'); + fctxt.setType('script'); + if ( snfe.matchRequest(fctxt) !== 0 ) { + console.log(snfe.toLogData()); + } + + // Unblocked + fctxt.setDocOriginFromURL('https://www.bloomberg.com/'); + fctxt.setURL('https://sourcepointcmp.bloomberg.com/ccpa.js'); + fctxt.setType('script'); + if ( snfe.matchRequest(fctxt) !== 0 ) { + console.log(snfe.toLogData()); + } + + // Remove all filters + restart(); + + process.exit(); +})(); + +/******************************************************************************/ diff --git a/platform/opera/manifest.json b/platform/opera/manifest.json index 69af37e96..2545566ee 100644 --- a/platform/opera/manifest.json +++ b/platform/opera/manifest.json @@ -69,7 +69,7 @@ }, "incognito": "split", "manifest_version": 2, - "minimum_opera_version": "42.0", + "minimum_opera_version": "48.0", "name": "uBlock Origin", "options_page": "dashboard.html", "permissions": [ diff --git a/platform/thunderbird/manifest.json b/platform/thunderbird/manifest.json index a0762a129..3f909edbe 100644 --- a/platform/thunderbird/manifest.json +++ b/platform/thunderbird/manifest.json @@ -2,7 +2,7 @@ "applications": { "gecko": { "id": "uBlock0@raymondhill.net", - "strict_min_version": "65.0" + "strict_min_version": "78.0" } }, "author": "Raymond Hill & contributors", diff --git a/src/1p-filters.html b/src/1p-filters.html index 26fce7f6e..29d615501 100644 --- a/src/1p-filters.html +++ b/src/1p-filters.html @@ -50,11 +50,9 @@ - - @@ -65,8 +63,7 @@ - - + diff --git a/src/asset-viewer.html b/src/asset-viewer.html index d94fc0bdc..126cbc6e4 100644 --- a/src/asset-viewer.html +++ b/src/asset-viewer.html @@ -33,11 +33,9 @@ - - @@ -46,8 +44,7 @@ - - + diff --git a/src/background.html b/src/background.html index 078824442..21429406f 100644 --- a/src/background.html +++ b/src/background.html @@ -7,45 +7,36 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/dyna-rules.html b/src/dyna-rules.html index bb9b28260..752f997f4 100644 --- a/src/dyna-rules.html +++ b/src/dyna-rules.html @@ -51,10 +51,6 @@ - - - - @@ -64,7 +60,7 @@ - + diff --git a/src/js/1p-filters.js b/src/js/1p-filters.js index 86a5b707d..e52beaedf 100644 --- a/src/js/1p-filters.js +++ b/src/js/1p-filters.js @@ -25,7 +25,7 @@ /******************************************************************************/ -(( ) => { +import './codemirror/ubo-static-filtering.js'; /******************************************************************************/ @@ -352,5 +352,3 @@ cmEditor.on('changes', userFiltersChanged); CodeMirror.commands.save = applyChanges; /******************************************************************************/ - -})(); diff --git a/src/js/asset-viewer.js b/src/js/asset-viewer.js index ecdef01c2..ef0824d6d 100644 --- a/src/js/asset-viewer.js +++ b/src/js/asset-viewer.js @@ -25,6 +25,10 @@ /******************************************************************************/ +import './codemirror/ubo-static-filtering.js'; + +/******************************************************************************/ + (async ( ) => { const subscribeURL = new URL(document.location); const subscribeParams = subscribeURL.searchParams; diff --git a/src/js/assets.js b/src/js/assets.js index f63596efb..d7e144e55 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -23,7 +23,7 @@ /******************************************************************************/ -µBlock.assets = (( ) => { +import µBlock from './background.js'; /******************************************************************************/ @@ -1048,10 +1048,8 @@ api.isUpdating = function() { /******************************************************************************/ -return api; - -/******************************************************************************/ - -})(); +// Export + +µBlock.assets = api; /******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js index 0139b2286..b6794f5ac 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -19,212 +19,340 @@ Home: https://github.com/gorhill/uBlock */ - 'use strict'; /******************************************************************************/ +import globals from './globals.js'; + +import { + domainFromHostname, + hostnameFromURI, + originFromURI, +} from './uri-utils.js'; + +import { FilteringContext } from './filtering-context.js'; +import { CompiledListWriter } from './static-filtering-io.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; +import { staticNetFilteringEngine } from './static-net-filtering.js'; + +/******************************************************************************/ + // Not all platforms may have properly declared vAPI.webextFlavor. if ( vAPI.webextFlavor === undefined ) { vAPI.webextFlavor = { major: 0, soup: new Set([ 'ublock' ]) }; } +/******************************************************************************/ + +const hiddenSettingsDefault = { + allowGenericProceduralFilters: false, + assetFetchTimeout: 30, + autoCommentFilterTemplate: '{{date}} {{origin}}', + autoUpdateAssetFetchPeriod: 120, + autoUpdateDelayAfterLaunch: 180, + autoUpdatePeriod: 4, + benchmarkDatasetURL: 'unset', + blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', + cacheStorageAPI: 'unset', + cacheStorageCompression: true, + cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', + cloudStorageCompression: true, + cnameIgnoreList: 'unset', + cnameIgnore1stParty: true, + cnameIgnoreExceptions: true, + cnameIgnoreRootDocument: true, + cnameMaxTTL: 120, + cnameReplayFullURL: false, + cnameUncloak: true, + cnameUncloakProxied: false, + consoleLogLevel: 'unset', + debugScriptlets: false, + debugScriptletInjector: false, + disableWebAssembly: false, + extensionUpdateForceReload: false, + filterAuthorMode: false, + filterOnHeaders: false, + loggerPopupType: 'popup', + manualUpdateAssetFetchPeriod: 500, + popupFontSize: 'unset', + popupPanelDisabledSections: 0, + popupPanelLockedSections: 0, + popupPanelHeightMode: 0, + requestJournalProcessPeriod: 1000, + selfieAfter: 3, + strictBlockingBypassDuration: 120, + suspendTabsUntilReady: 'unset', + uiPopupConfig: 'undocumented', + uiFlavor: 'unset', + uiStyles: 'unset', + uiTheme: 'unset', + updateAssetBypassBrowserCache: false, + userResourcesLocation: 'unset', +}; + +const userSettingsDefault = { + advancedUserEnabled: false, + alwaysDetachLogger: true, + autoUpdate: true, + cloudStorageEnabled: false, + cnameUncloakEnabled: true, + collapseBlocked: true, + colorBlindFriendly: false, + contextMenuEnabled: true, + dynamicFilteringEnabled: false, + externalLists: '', + firewallPaneMinimized: true, + hyperlinkAuditingDisabled: true, + ignoreGenericCosmeticFilters: vAPI.webextFlavor.soup.has('mobile'), + importedLists: [], + largeMediaSize: 50, + parseAllABPHideFilters: true, + popupPanelSections: 0b111, + prefetchingDisabled: true, + requestLogMaxEntries: 1000, + showIconBadge: true, + tooltipsDisabled: false, + webrtcIPAddressHidden: false, +}; + +const µBlock = { // jshint ignore:line + userSettingsDefault: userSettingsDefault, + userSettings: Object.assign({}, userSettingsDefault), + + hiddenSettingsDefault: hiddenSettingsDefault, + hiddenSettingsAdmin: {}, + hiddenSettings: Object.assign({}, hiddenSettingsDefault), + + noDashboard: false, + + // Features detection. + privacySettingsSupported: vAPI.browserSettings instanceof Object, + cloudStorageSupported: vAPI.cloud instanceof Object, + canFilterResponseData: typeof browser.webRequest.filterResponseData === 'function', + canInjectScriptletsNow: vAPI.webextFlavor.soup.has('chromium'), + + // https://github.com/chrisaljoudi/uBlock/issues/180 + // Whitelist directives need to be loaded once the PSL is available + netWhitelist: new Map(), + netWhitelistModifyTime: 0, + netWhitelistDefault: [ + 'about-scheme', + 'chrome-extension-scheme', + 'chrome-scheme', + 'edge-scheme', + 'moz-extension-scheme', + 'opera-scheme', + 'vivaldi-scheme', + 'wyciwyg-scheme', // Firefox's "What-You-Cache-Is-What-You-Get" + ], + + localSettings: { + blockedRequestCount: 0, + allowedRequestCount: 0, + }, + localSettingsLastModified: 0, + localSettingsLastSaved: 0, + + // Read-only + systemSettings: { + compiledMagic: 37, // Increase when compiled format changes + selfieMagic: 37, // Increase when selfie format changes + }, + + // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501 + // The assumption is that cache storage state reflects whether + // compiled or selfie assets are available or not. The properties + // below is to no longer rely on this assumption -- though it's still + // not clear how the assumption could be wrong, and it's still not + // clear whether relying on those properties will really solve the + // issue. It's just an attempt at hardening. + compiledFormatChanged: false, + selfieIsInvalid: false, + + compiledCosmeticSection: 200, + compiledScriptletSection: 300, + compiledHTMLSection: 400, + compiledHTTPHeaderSection: 500, + compiledSentinelSection: 1000, + compiledBadSubsection: 1, + + restoreBackupSettings: { + lastRestoreFile: '', + lastRestoreTime: 0, + lastBackupFile: '', + lastBackupTime: 0, + }, + + commandShortcuts: new Map(), + + // Allows to fully customize uBO's assets, typically set through admin + // settings. The content of 'assets.json' will also tell which filter + // lists to enable by default when uBO is first installed. + assetsBootstrapLocation: undefined, + + userFiltersPath: 'user-filters', + pslAssetKey: 'public_suffix_list.dat', + + selectedFilterLists: [], + availableFilterLists: {}, + badLists: new Map(), + + // https://github.com/uBlockOrigin/uBlock-issues/issues/974 + // This can be used to defer filtering decision-making. + readyToFilter: false, + + pageStores: new Map(), + pageStoresToken: 0, + + storageQuota: vAPI.storage.QUOTA_BYTES, + storageUsed: 0, + + noopFunc: function(){}, + + apiErrorCount: 0, + + maybeGoodPopup: { + tabId: 0, + url: '', + }, + + epickerArgs: { + eprom: null, + mouse: false, + target: '', + zap: false, + }, + + scriptlets: {}, + + cspNoInlineScript: "script-src 'unsafe-eval' * blob: data:", + cspNoScripting: 'script-src http: https:', + cspNoInlineFont: 'font-src *', + + liveBlockingProfiles: [], + blockingProfileColorCache: new Map(), +}; + +µBlock.domainFromHostname = domainFromHostname; +µBlock.hostnameFromURI = hostnameFromURI; + +µBlock.FilteringContext = class extends FilteringContext { + duplicate() { + return (new µBlock.FilteringContext(this)); + } + + fromTabId(tabId) { + const tabContext = µBlock.tabContextManager.mustLookup(tabId); + this.tabOrigin = tabContext.origin; + this.tabHostname = tabContext.rootHostname; + this.tabDomain = tabContext.rootDomain; + this.tabId = tabContext.tabId; + return this; + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/459 + // In case of a request for frame and if ever no context is specified, + // assume the origin of the context is the same as the request itself. + fromWebrequestDetails(details) { + const tabId = details.tabId; + this.type = details.type; + if ( this.itype === this.MAIN_FRAME && tabId > 0 ) { + µBlock.tabContextManager.push(tabId, details.url); + } + this.fromTabId(tabId); // Must be called AFTER tab context management + this.realm = ''; + this.id = details.requestId; + this.setURL(details.url); + this.aliasURL = details.aliasURL || undefined; + if ( this.itype !== this.SUB_FRAME ) { + this.docId = details.frameId; + this.frameId = -1; + } else { + this.docId = details.parentFrameId; + this.frameId = details.frameId; + } + if ( this.tabId > 0 ) { + if ( this.docId === 0 ) { + this.docOrigin = this.tabOrigin; + this.docHostname = this.tabHostname; + this.docDomain = this.tabDomain; + } else if ( details.documentUrl !== undefined ) { + this.setDocOriginFromURL(details.documentUrl); + } else { + const pageStore = µBlock.pageStoreFromTabId(this.tabId); + const docStore = pageStore && pageStore.getFrameStore(this.docId); + if ( docStore ) { + this.setDocOriginFromURL(docStore.rawURL); + } else { + this.setDocOrigin(this.tabOrigin); + } + } + } else if ( details.documentUrl !== undefined ) { + const origin = originFromURI( + µBlock.normalizeTabURL(0, details.documentUrl) + ); + this.setDocOrigin(origin).setTabOrigin(origin); + } else if ( this.docId === -1 || (this.itype & this.FRAME_ANY) !== 0 ) { + const origin = originFromURI(this.url); + this.setDocOrigin(origin).setTabOrigin(origin); + } else { + this.setDocOrigin(this.tabOrigin); + } + this.redirectURL = undefined; + this.filter = undefined; + return this; + } + + getTabOrigin() { + if ( this.tabOrigin === undefined ) { + const tabContext = µBlock.tabContextManager.mustLookup(this.tabId); + this.tabOrigin = tabContext.origin; + this.tabHostname = tabContext.rootHostname; + this.tabDomain = tabContext.rootDomain; + } + return super.getTabOrigin(); + } + + getTabHostname() { + if ( this.tabHostname === undefined ) { + this.tabHostname = hostnameFromURI(this.getTabOrigin()); + } + return super.getTabHostname(); + } + + toLogger() { + this.tstamp = Date.now(); + if ( this.domain === undefined ) { + void this.getDomain(); + } + if ( this.docDomain === undefined ) { + void this.getDocDomain(); + } + if ( this.tabDomain === undefined ) { + void this.getTabDomain(); + } + const logger = µBlock.logger; + const filters = this.filter; + // Many filters may have been applied to the current context + if ( Array.isArray(filters) === false ) { + return logger.writeOne(this); + } + for ( const filter of filters ) { + this.filter = filter; + logger.writeOne(this); + } + } +}; + +µBlock.filteringContext = new µBlock.FilteringContext(); +µBlock.CompiledListWriter = CompiledListWriter; +µBlock.StaticFilteringParser = StaticFilteringParser; +µBlock.staticNetFilteringEngine = staticNetFilteringEngine; + +globals.µBlock = µBlock; /******************************************************************************/ -const µBlock = (( ) => { // jshint ignore:line - - const hiddenSettingsDefault = { - allowGenericProceduralFilters: false, - assetFetchTimeout: 30, - autoCommentFilterTemplate: '{{date}} {{origin}}', - autoUpdateAssetFetchPeriod: 120, - autoUpdateDelayAfterLaunch: 180, - autoUpdatePeriod: 4, - benchmarkDatasetURL: 'unset', - blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', - cacheStorageAPI: 'unset', - cacheStorageCompression: true, - cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', - cloudStorageCompression: true, - cnameIgnoreList: 'unset', - cnameIgnore1stParty: true, - cnameIgnoreExceptions: true, - cnameIgnoreRootDocument: true, - cnameMaxTTL: 120, - cnameReplayFullURL: false, - cnameUncloak: true, - cnameUncloakProxied: false, - consoleLogLevel: 'unset', - debugScriptlets: false, - debugScriptletInjector: false, - disableWebAssembly: false, - extensionUpdateForceReload: false, - filterAuthorMode: false, - filterOnHeaders: false, - loggerPopupType: 'popup', - manualUpdateAssetFetchPeriod: 500, - popupFontSize: 'unset', - popupPanelDisabledSections: 0, - popupPanelLockedSections: 0, - popupPanelHeightMode: 0, - requestJournalProcessPeriod: 1000, - selfieAfter: 3, - strictBlockingBypassDuration: 120, - suspendTabsUntilReady: 'unset', - uiPopupConfig: 'undocumented', - uiFlavor: 'unset', - uiStyles: 'unset', - uiTheme: 'unset', - updateAssetBypassBrowserCache: false, - userResourcesLocation: 'unset', - }; - - const userSettingsDefault = { - advancedUserEnabled: false, - alwaysDetachLogger: true, - autoUpdate: true, - cloudStorageEnabled: false, - cnameUncloakEnabled: true, - collapseBlocked: true, - colorBlindFriendly: false, - contextMenuEnabled: true, - dynamicFilteringEnabled: false, - externalLists: '', - firewallPaneMinimized: true, - hyperlinkAuditingDisabled: true, - ignoreGenericCosmeticFilters: vAPI.webextFlavor.soup.has('mobile'), - importedLists: [], - largeMediaSize: 50, - parseAllABPHideFilters: true, - popupPanelSections: 0b111, - prefetchingDisabled: true, - requestLogMaxEntries: 1000, - showIconBadge: true, - tooltipsDisabled: false, - webrtcIPAddressHidden: false, - }; - - return { - userSettingsDefault: userSettingsDefault, - userSettings: Object.assign({}, userSettingsDefault), - - hiddenSettingsDefault: hiddenSettingsDefault, - hiddenSettingsAdmin: {}, - hiddenSettings: Object.assign({}, hiddenSettingsDefault), - - noDashboard: false, - - // Features detection. - privacySettingsSupported: vAPI.browserSettings instanceof Object, - cloudStorageSupported: vAPI.cloud instanceof Object, - canFilterResponseData: typeof browser.webRequest.filterResponseData === 'function', - canInjectScriptletsNow: vAPI.webextFlavor.soup.has('chromium'), - - // https://github.com/chrisaljoudi/uBlock/issues/180 - // Whitelist directives need to be loaded once the PSL is available - netWhitelist: new Map(), - netWhitelistModifyTime: 0, - netWhitelistDefault: [ - 'about-scheme', - 'chrome-extension-scheme', - 'chrome-scheme', - 'edge-scheme', - 'moz-extension-scheme', - 'opera-scheme', - 'vivaldi-scheme', - 'wyciwyg-scheme', // Firefox's "What-You-Cache-Is-What-You-Get" - ], - - localSettings: { - blockedRequestCount: 0, - allowedRequestCount: 0, - }, - localSettingsLastModified: 0, - localSettingsLastSaved: 0, - - // Read-only - systemSettings: { - compiledMagic: 37, // Increase when compiled format changes - selfieMagic: 37, // Increase when selfie format changes - }, - - // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501 - // The assumption is that cache storage state reflects whether - // compiled or selfie assets are available or not. The properties - // below is to no longer rely on this assumption -- though it's still - // not clear how the assumption could be wrong, and it's still not - // clear whether relying on those properties will really solve the - // issue. It's just an attempt at hardening. - compiledFormatChanged: false, - selfieIsInvalid: false, - - compiledNetworkSection: 100, - compiledCosmeticSection: 200, - compiledScriptletSection: 300, - compiledHTMLSection: 400, - compiledHTTPHeaderSection: 500, - compiledSentinelSection: 1000, - compiledBadSubsection: 1, - - restoreBackupSettings: { - lastRestoreFile: '', - lastRestoreTime: 0, - lastBackupFile: '', - lastBackupTime: 0, - }, - - commandShortcuts: new Map(), - - // Allows to fully customize uBO's assets, typically set through admin - // settings. The content of 'assets.json' will also tell which filter - // lists to enable by default when uBO is first installed. - assetsBootstrapLocation: undefined, - - userFiltersPath: 'user-filters', - pslAssetKey: 'public_suffix_list.dat', - - selectedFilterLists: [], - availableFilterLists: {}, - badLists: new Map(), - - // https://github.com/uBlockOrigin/uBlock-issues/issues/974 - // This can be used to defer filtering decision-making. - readyToFilter: false, - - pageStores: new Map(), - pageStoresToken: 0, - - storageQuota: vAPI.storage.QUOTA_BYTES, - storageUsed: 0, - - noopFunc: function(){}, - - apiErrorCount: 0, - - maybeGoodPopup: { - tabId: 0, - url: '', - }, - - epickerArgs: { - eprom: null, - mouse: false, - target: '', - zap: false, - }, - - scriptlets: {}, - - cspNoInlineScript: "script-src 'unsafe-eval' * blob: data:", - cspNoScripting: 'script-src http: https:', - cspNoInlineFont: 'font-src *', - - liveBlockingProfiles: [], - blockingProfileColorCache: new Map(), - }; - -})(); - -/******************************************************************************/ +export default µBlock; diff --git a/src/js/base64-custom.js b/src/js/base64-custom.js new file mode 100644 index 000000000..fab6c08c6 --- /dev/null +++ b/src/js/base64-custom.js @@ -0,0 +1,246 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// Custom base64 codecs. These codecs are meant to encode/decode typed arrays +// to/from strings. + +// https://github.com/uBlockOrigin/uBlock-issues/issues/461 +// Provide a fallback encoding for Chromium 59 and less by issuing a plain +// JSON string. The fallback can be removed once min supported version is +// above 59. + +// TODO: rename µBlock.base64 to µBlock.SparseBase64, now that +// µBlock.DenseBase64 has been introduced. +// TODO: Should no longer need to test presence of TextEncoder/TextDecoder. + +const valToDigit = new Uint8Array(64); +const digitToVal = new Uint8Array(128); +{ + const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@%'; + for ( let i = 0, n = chars.length; i < n; i++ ) { + const c = chars.charCodeAt(i); + valToDigit[i] = c; + digitToVal[c] = i; + } +} + +// The sparse base64 codec is best for buffers which contains a lot of +// small u32 integer values. Those small u32 integer values are better +// represented with stringified integers, because small values can be +// represented with fewer bits than the usual base64 codec. For example, +// 0 become '0 ', i.e. 16 bits instead of 48 bits with official base64 +// codec. + +const sparseBase64 = { + magic: 'Base64_1', + + encode: function(arrbuf, arrlen) { + const inputLength = (arrlen + 3) >>> 2; + const inbuf = new Uint32Array(arrbuf, 0, inputLength); + const outputLength = this.magic.length + 7 + inputLength * 7; + const outbuf = new Uint8Array(outputLength); + // magic bytes + let j = 0; + for ( let i = 0; i < this.magic.length; i++ ) { + outbuf[j++] = this.magic.charCodeAt(i); + } + // array size + let v = inputLength; + do { + outbuf[j++] = valToDigit[v & 0b111111]; + v >>>= 6; + } while ( v !== 0 ); + outbuf[j++] = 0x20 /* ' ' */; + // array content + for ( let i = 0; i < inputLength; i++ ) { + v = inbuf[i]; + do { + outbuf[j++] = valToDigit[v & 0b111111]; + v >>>= 6; + } while ( v !== 0 ); + outbuf[j++] = 0x20 /* ' ' */; + } + if ( typeof TextDecoder === 'undefined' ) { + return JSON.stringify( + Array.from(new Uint32Array(outbuf.buffer, 0, j >>> 2)) + ); + } + const textDecoder = new TextDecoder(); + return textDecoder.decode(new Uint8Array(outbuf.buffer, 0, j)); + }, + + decode: function(instr, arrbuf) { + if ( instr.charCodeAt(0) === 0x5B /* '[' */ ) { + const inbuf = JSON.parse(instr); + if ( arrbuf instanceof ArrayBuffer === false ) { + return new Uint32Array(inbuf); + } + const outbuf = new Uint32Array(arrbuf); + outbuf.set(inbuf); + return outbuf; + } + if ( instr.startsWith(this.magic) === false ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + const inputLength = instr.length; + const outputLength = this.decodeSize(instr) >> 2; + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint32Array(outputLength) + : new Uint32Array(arrbuf); + let i = instr.indexOf(' ', this.magic.length) + 1; + if ( i === -1 ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + // array content + let j = 0; + for (;;) { + if ( j === outputLength || i >= inputLength ) { break; } + let v = 0, l = 0; + for (;;) { + const c = instr.charCodeAt(i++); + if ( c === 0x20 /* ' ' */ ) { break; } + v += digitToVal[c] << l; + l += 6; + } + outbuf[j++] = v; + } + if ( i < inputLength || j < outputLength ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + return outbuf; + }, + + decodeSize: function(instr) { + if ( instr.startsWith(this.magic) === false ) { return 0; } + let v = 0, l = 0, i = this.magic.length; + for (;;) { + const c = instr.charCodeAt(i++); + if ( c === 0x20 /* ' ' */ ) { break; } + v += digitToVal[c] << l; + l += 6; + } + return v << 2; + }, +}; + +// The dense base64 codec is best for typed buffers which values are +// more random. For example, buffer contents as a result of compression +// contain less repetitive values and thus the content is more +// random-looking. + +// TODO: Investigate that in Firefox, creating a new Uint8Array from the +// ArrayBuffer fails, the content of the resulting Uint8Array is +// non-sensical. WASM-related? + +const denseBase64 = { + magic: 'DenseBase64_1', + + encode: function(input) { + const m = input.length % 3; + const n = input.length - m; + let outputLength = n / 3 * 4; + if ( m !== 0 ) { + outputLength += m + 1; + } + const output = new Uint8Array(outputLength); + let j = 0; + for ( let i = 0; i < n; i += 3) { + const i1 = input[i+0]; + const i2 = input[i+1]; + const i3 = input[i+2]; + output[j+0] = valToDigit[ i1 >>> 2]; + output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; + output[j+2] = valToDigit[i2 << 2 & 0b111100 | i3 >>> 6]; + output[j+3] = valToDigit[i3 & 0b111111 ]; + j += 4; + } + if ( m !== 0 ) { + const i1 = input[n]; + output[j+0] = valToDigit[i1 >>> 2]; + if ( m === 1 ) { // 1 value + output[j+1] = valToDigit[i1 << 4 & 0b110000]; + } else { // 2 values + const i2 = input[n+1]; + output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; + output[j+2] = valToDigit[i2 << 2 & 0b111100 ]; + } + } + const textDecoder = new TextDecoder(); + const b64str = textDecoder.decode(output); + return this.magic + b64str; + }, + + decode: function(instr, arrbuf) { + if ( instr.startsWith(this.magic) === false ) { + throw new Error('Invalid µBlock.denseBase64 encoding'); + } + const outputLength = this.decodeSize(instr); + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint8Array(outputLength) + : new Uint8Array(arrbuf); + const inputLength = instr.length - this.magic.length; + let i = this.magic.length; + let j = 0; + const m = inputLength & 3; + const n = i + inputLength - m; + while ( i < n ) { + const i1 = digitToVal[instr.charCodeAt(i+0)]; + const i2 = digitToVal[instr.charCodeAt(i+1)]; + const i3 = digitToVal[instr.charCodeAt(i+2)]; + const i4 = digitToVal[instr.charCodeAt(i+3)]; + i += 4; + outbuf[j+0] = i1 << 2 | i2 >>> 4; + outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; + outbuf[j+2] = i3 << 6 & 0b11000000 | i4; + j += 3; + } + if ( m !== 0 ) { + const i1 = digitToVal[instr.charCodeAt(i+0)]; + const i2 = digitToVal[instr.charCodeAt(i+1)]; + outbuf[j+0] = i1 << 2 | i2 >>> 4; + if ( m === 3 ) { + const i3 = digitToVal[instr.charCodeAt(i+2)]; + outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; + } + } + return outbuf; + }, + + decodeSize: function(instr) { + if ( instr.startsWith(this.magic) === false ) { return 0; } + const inputLength = instr.length - this.magic.length; + const m = inputLength & 3; + const n = inputLength - m; + let outputLength = (n >>> 2) * 3; + if ( m !== 0 ) { + outputLength += m - 1; + } + return outputLength; + }, +}; + +/******************************************************************************/ + +export { denseBase64, sparseBase64 }; diff --git a/src/js/strie.js b/src/js/biditrie.js similarity index 97% rename from src/js/strie.js rename to src/js/biditrie.js index 9c26af224..78dc7754c 100644 --- a/src/js/strie.js +++ b/src/js/biditrie.js @@ -23,11 +23,6 @@ 'use strict'; -// ***************************************************************************** -// start of local namespace - -{ - /******************************************************************************* A BidiTrieContainer is mostly a large buffer in which distinct but related @@ -130,7 +125,7 @@ const toSegmentInfo = (aL, l, r) => ((r - l) << 24) | (aL + l); const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); -µBlock.BidiTrieContainer = class { +const BidiTrieContainer = class { constructor(extraHandler) { const len = PAGE_SIZE * 4; @@ -697,10 +692,10 @@ const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); return -1; } - async enableWASM() { + async enableWASM(modulePath) { if ( typeof WebAssembly !== 'object' ) { return false; } if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; } - const module = await getWasmModule(); + const module = await getWasmModule(modulePath); if ( module instanceof WebAssembly.Module === false ) { return false; } const memory = new WebAssembly.Memory({ initial: roundToPageSize(this.buf8.length) >>> 16 @@ -828,7 +823,7 @@ const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); */ -µBlock.BidiTrieContainer.prototype.STrieRef = class { +BidiTrieContainer.prototype.STrieRef = class { constructor(container, iroot, size) { this.container = container; this.iroot = iroot; @@ -930,20 +925,7 @@ const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); const getWasmModule = (( ) => { let wasmModulePromise; - // The directory from which the current script was fetched should also - // contain the related WASM file. The script is fetched from a trusted - // location, and consequently so will be the related WASM file. - let workingDir; - { - const url = new URL(document.currentScript.src); - const match = /[^\/]+$/.exec(url.pathname); - if ( match !== null ) { - url.pathname = url.pathname.slice(0, match.index); - } - workingDir = url.href; - } - - return async function() { + return async function(modulePath) { if ( wasmModulePromise instanceof Promise ) { return wasmModulePromise; } @@ -967,19 +949,18 @@ const getWasmModule = (( ) => { if ( uint8s[0] !== 1 ) { return; } wasmModulePromise = fetch( - workingDir + 'wasm/biditrie.wasm', + `${modulePath}/wasm/biditrie.wasm`, { mode: 'same-origin' } ).then( WebAssembly.compileStreaming ).catch(reason => { - log.info(reason); + console.info(reason); }); return wasmModulePromise; }; })(); -// end of local namespace -// ***************************************************************************** +/******************************************************************************/ -} +export { BidiTrieContainer }; diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js index 08e948e32..85cd9c0ac 100644 --- a/src/js/cachestorage.js +++ b/src/js/cachestorage.js @@ -25,6 +25,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + // The code below has been originally manually imported from: // Commit: https://github.com/nikrolls/uBlock-Edge/commit/d1538ea9bea89d507219d3219592382eee306134 // Commit date: 29 October 2016 diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js index eebc81125..70be0dc7f 100644 --- a/src/js/codemirror/ubo-static-filtering.js +++ b/src/js/codemirror/ubo-static-filtering.js @@ -25,8 +25,7 @@ /******************************************************************************/ -{ -// >>>>> start of local scope +import { StaticFilteringParser } from '../static-filtering-parser.js'; /******************************************************************************/ @@ -40,9 +39,6 @@ let hintHelperRegistered = false; /******************************************************************************/ CodeMirror.defineMode('ubo-static-filtering', function() { - const StaticFilteringParser = typeof vAPI === 'object' - ? vAPI.StaticFilteringParser - : self.StaticFilteringParser; if ( StaticFilteringParser instanceof Object === false ) { return; } const parser = new StaticFilteringParser({ interactive: true }); @@ -417,9 +413,6 @@ CodeMirror.defineMode('ubo-static-filtering', function() { // https://codemirror.net/demo/complete.html const initHints = function() { - const StaticFilteringParser = typeof vAPI === 'object' - ? vAPI.StaticFilteringParser - : self.StaticFilteringParser; if ( StaticFilteringParser instanceof Object === false ) { return; } const parser = new StaticFilteringParser(); @@ -802,8 +795,3 @@ CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => { } /******************************************************************************/ - -// <<<<< end of local scope -} - -/******************************************************************************/ diff --git a/src/js/commands.js b/src/js/commands.js index a695427fb..463e63c08 100644 --- a/src/js/commands.js +++ b/src/js/commands.js @@ -19,9 +19,12 @@ Home: https://github.com/gorhill/uBlock */ +'use strict'; + /******************************************************************************/ -'use strict'; +import { hostnameFromURI } from './uri-utils.js'; +import µBlock from './background.js'; /******************************************************************************/ @@ -71,11 +74,11 @@ const relaxBlockingMode = (( ) => { if ( tab instanceof Object === false || tab.id <= 0 ) { return; } const µb = µBlock; - const normalURL = µb.normalizePageURL(tab.id, tab.url); + const normalURL = µb.normalizeTabURL(tab.id, tab.url); if ( µb.getNetFilteringSwitch(normalURL) === false ) { return; } - const hn = µb.URI.hostnameFromURI(normalURL); + const hn = hostnameFromURI(normalURL); const curProfileBits = µb.blockingModeFromHostname(hn); let newProfileBits; for ( const profile of µb.liveBlockingProfiles ) { diff --git a/src/js/contextmenu.js b/src/js/contextmenu.js index 12190a430..e664c3364 100644 --- a/src/js/contextmenu.js +++ b/src/js/contextmenu.js @@ -23,6 +23,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + µBlock.contextMenu = (( ) => { /******************************************************************************/ diff --git a/src/js/cosmetic-filtering.js b/src/js/cosmetic-filtering.js index f0d045475..3601a7229 100644 --- a/src/js/cosmetic-filtering.js +++ b/src/js/cosmetic-filtering.js @@ -23,7 +23,13 @@ /******************************************************************************/ -µBlock.cosmeticFilteringEngine = (( ) => { +import { + domainFromHostname, + entityFromDomain, + hostnameFromURI, +} from './uri-utils.js'; + +import µBlock from './background.js'; /******************************************************************************/ @@ -253,7 +259,6 @@ const FilterContainer = function() { // Reset all, thus reducing to a minimum memory footprint of the context. FilterContainer.prototype.reset = function() { - this.µburi = µb.URI; this.frozen = false; this.acceptedCount = 0; this.discardedCount = 0; @@ -369,7 +374,6 @@ FilterContainer.prototype.compile = function(parser, writer) { /******************************************************************************/ FilterContainer.prototype.compileGenericSelector = function(parser, writer) { - writer.select(µb.compiledCosmeticSection + COMPILED_GENERIC_SECTION); if ( parser.isException() ) { this.compileGenericUnhideSelector(parser, writer); } else { @@ -385,14 +389,17 @@ FilterContainer.prototype.compileGenericHideSelector = function( ) { const { raw, compiled, pseudoclass } = parser.result; if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', text: `Invalid generic cosmetic filter in ${who}: ${raw}` }); + return; } + writer.select(µb.compiledCosmeticSection + COMPILED_GENERIC_SECTION); + const type = compiled.charCodeAt(0); let key; @@ -429,7 +436,7 @@ FilterContainer.prototype.compileGenericHideSelector = function( if ( µb.hiddenSettings.allowGenericProceduralFilters === true ) { return this.compileSpecificSelector(parser, '', false, writer); } - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', @@ -485,7 +492,7 @@ FilterContainer.prototype.compileGenericUnhideSelector = function( // Procedural cosmetic filters are acceptable as generic exception filters. const { raw, compiled } = parser.result; if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', @@ -494,6 +501,8 @@ FilterContainer.prototype.compileGenericUnhideSelector = function( return; } + writer.select(µb.compiledCosmeticSection + COMPILED_SPECIFIC_SECTION); + // https://github.com/chrisaljoudi/uBlock/issues/497 // All generic exception filters are stored as hostname-based filter // whereas the hostname is the empty string (which matches all @@ -511,10 +520,9 @@ FilterContainer.prototype.compileSpecificSelector = function( not, writer ) { - writer.select(µb.compiledCosmeticSection + COMPILED_SPECIFIC_SECTION); const { raw, compiled, exception } = parser.result; if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', @@ -523,6 +531,8 @@ FilterContainer.prototype.compileSpecificSelector = function( return; } + writer.select(µb.compiledCosmeticSection + COMPILED_SPECIFIC_SECTION); + // https://github.com/chrisaljoudi/uBlock/issues/145 let unhide = exception ? 1 : 0; if ( not ) { unhide ^= 1; } @@ -1146,9 +1156,9 @@ FilterContainer.prototype.benchmark = async function() { const request = requests[i]; if ( request.cpt !== 'main_frame' ) { continue; } count += 1; - details.hostname = µb.URI.hostnameFromURI(request.url); - details.domain = µb.URI.domainFromHostname(details.hostname); - details.entity = µb.URI.entityFromDomain(details.domain); + details.hostname = hostnameFromURI(request.url); + details.domain = domainFromHostname(details.hostname); + details.entity = entityFromDomain(details.domain); void this.retrieveSpecificSelectors(details, options); } const t1 = self.performance.now(); @@ -1159,10 +1169,8 @@ FilterContainer.prototype.benchmark = async function() { /******************************************************************************/ -return new FilterContainer(); - -/******************************************************************************/ - -})(); +// Export + +µBlock.cosmeticFilteringEngine = new FilterContainer(); /******************************************************************************/ diff --git a/src/js/dyna-rules.js b/src/js/dyna-rules.js index 0e489bfab..1abd8e841 100644 --- a/src/js/dyna-rules.js +++ b/src/js/dyna-rules.js @@ -19,17 +19,23 @@ Home: https://github.com/gorhill/uMatrix */ -/* global diff_match_patch, CodeMirror, uDom, uBlockDashboard */ +/* global CodeMirror, diff_match_patch, uDom, uBlockDashboard */ 'use strict'; /******************************************************************************/ -(( ) => { +import '../lib/publicsuffixlist/publicsuffixlist.js'; + +import globals from './globals.js'; +import { hostnameFromURI } from './uri-utils.js'; + +import './codemirror/ubo-dynamic-filtering.js'; /******************************************************************************/ -const psl = self.publicSuffixList; +const publicSuffixList = globals.publicSuffixList; + const hostnameToDomainMap = new Map(); const mergeView = new CodeMirror.MergeView( @@ -376,8 +382,8 @@ const onFilterChanged = (( ) => { }; return function() { - if ( timer !== undefined ) { self.cancelIdleCallback(timer); } - timer = self.requestIdleCallback(process, { timeout: 773 }); + if ( timer !== undefined ) { globals.cancelIdleCallback(timer); } + timer = globals.requestIdleCallback(process, { timeout: 773 }); }; })(); @@ -393,7 +399,9 @@ const onPresentationChanged = (( ) => { const sortNormalizeHn = function(hn) { let domain = hostnameToDomainMap.get(hn); if ( domain === undefined ) { - domain = /(\d|\])$/.test(hn) ? hn : psl.getDomain(hn); + domain = /(\d|\])$/.test(hn) + ? hn + : publicSuffixList.getDomain(hn); hostnameToDomainMap.set(hn, domain); } let normalized = domain || hn; @@ -424,7 +432,7 @@ const onPresentationChanged = (( ) => { } else if ( (match = reUrlRule.exec(rule)) !== null ) { type = '\x10FFFF'; srcHn = sortNormalizeHn(match[1]); - desHn = sortNormalizeHn(vAPI.hostnameFromURI(match[2])); + desHn = sortNormalizeHn(hostnameFromURI(match[2])); extra = match[3]; } if ( sortType === 0 ) { @@ -499,13 +507,13 @@ const onPresentationChanged = (( ) => { const mode = origPane.doc.getMode(); mode.sortType = sortType; mode.setHostnameToDomainMap(hostnameToDomainMap); - mode.setPSL(psl); + mode.setPSL(publicSuffixList); } { const mode = editPane.doc.getMode(); mode.sortType = sortType; mode.setHostnameToDomainMap(hostnameToDomainMap); - mode.setPSL(psl); + mode.setPSL(publicSuffixList); } sort(origPane.modified); sort(editPane.modified); @@ -547,8 +555,8 @@ const onTextChanged = (( ) => { }; return function(now) { - if ( timer !== undefined ) { self.cancelIdleCallback(timer); } - timer = now ? process() : self.requestIdleCallback(process, { timeout: 57 }); + if ( timer !== undefined ) { globals.cancelIdleCallback(timer); } + timer = now ? process() : globals.requestIdleCallback(process, { timeout: 57 }); }; })(); @@ -617,11 +625,11 @@ const editSaveHandler = function() { /******************************************************************************/ -self.cloud.onPush = function() { +globals.cloud.onPush = function() { return thePanes.orig.original.join('\n'); }; -self.cloud.onPull = function(data, append) { +globals.cloud.onPull = function(data, append) { if ( typeof data !== 'string' ) { return; } applyDiff( false, @@ -632,7 +640,7 @@ self.cloud.onPull = function(data, append) { /******************************************************************************/ -self.hasUnsavedData = function() { +globals.hasUnsavedData = function() { return mergeView.editor().isClean(cleanEditToken) === false; }; @@ -643,7 +651,7 @@ vAPI.messaging.send('dashboard', { }).then(details => { thePanes.orig.original = details.permanentRules; thePanes.edit.original = details.sessionRules; - psl.fromSelfie(details.pslSelfie); + publicSuffixList.fromSelfie(details.pslSelfie); onPresentationChanged(true); }); @@ -668,5 +676,3 @@ mergeView.editor().on('updateDiff', ( ) => { onTextChanged(); }); /******************************************************************************/ -})(); - diff --git a/src/js/dynamic-net-filtering.js b/src/js/dynamic-net-filtering.js index c0f322e03..6f2833027 100644 --- a/src/js/dynamic-net-filtering.js +++ b/src/js/dynamic-net-filtering.js @@ -19,18 +19,23 @@ Home: https://github.com/gorhill/uBlock */ -/* global punycode */ /* jshint bitwise: false */ 'use strict'; /******************************************************************************/ -{ -// >>>>> start of local scope +import '../lib/punycode.js'; + +import globals from './globals.js'; +import { domainFromHostname } from './uri-utils.js'; +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; /******************************************************************************/ +const punycode = globals.punycode; + const supportedDynamicTypes = { '3p': true, 'image': true, @@ -89,8 +94,6 @@ const is3rdParty = function(srcHostname, desHostname) { desHostname.charAt(desHostname.length - srcDomain.length - 1) !== '.'; }; -const domainFromHostname = µBlock.URI.domainFromHostname; - /******************************************************************************/ const Matrix = class { @@ -429,7 +432,7 @@ const Matrix = class { fromString(text, append) { - const lineIter = new µBlock.LineIterator(text); + const lineIter = new LineIterator(text); if ( append !== true ) { this.reset(); } while ( lineIter.eot() === false ) { this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); @@ -541,13 +544,10 @@ Matrix.prototype.magicId = 1; /******************************************************************************/ +// Export + µBlock.Firewall = Matrix; -// <<<<< end of local scope -} - -/******************************************************************************/ - µBlock.sessionFirewall = new µBlock.Firewall(); µBlock.permanentFirewall = new µBlock.Firewall(); diff --git a/src/js/epicker-ui.js b/src/js/epicker-ui.js index e45c0035f..5cbbc1d1f 100644 --- a/src/js/epicker-ui.js +++ b/src/js/epicker-ui.js @@ -23,6 +23,13 @@ 'use strict'; +/******************************************************************************/ + +import './codemirror/ubo-static-filtering.js'; + +import { hostnameFromURI } from './uri-utils.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; + /******************************************************************************/ /******************************************************************************/ @@ -144,7 +151,7 @@ const renderRange = function(id, value, invert = false) { const userFilterFromCandidate = function(filter) { if ( filter === '' || filter === '!' ) { return; } - const hn = vAPI.hostnameFromURI(docURL.href); + const hn = hostnameFromURI(docURL.href); // Cosmetic filter? if ( reCosmeticAnchor.test(filter) ) { @@ -828,7 +835,7 @@ const startPicker = function() { $id('candidateFilters').addEventListener('click', onCandidateClicked); $stor('#resultsetDepth input').addEventListener('input', onDepthChanged); $stor('#resultsetSpecificity input').addEventListener('input', onSpecificityChanged); - staticFilteringParser = new vAPI.StaticFilteringParser({ interactive: true }); + staticFilteringParser = new StaticFilteringParser({ interactive: true }); }; /******************************************************************************/ diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js index af68ca19e..53d99e80d 100644 --- a/src/js/filtering-context.js +++ b/src/js/filtering-context.js @@ -23,14 +23,11 @@ /******************************************************************************/ -{ -// >>>>> start of local scope - -/******************************************************************************/ - -const originFromURI = µBlock.URI.originFromURI; -const hostnameFromURI = vAPI.hostnameFromURI; -const domainFromHostname = vAPI.domainFromHostname; +import { + hostnameFromURI, + domainFromHostname, + originFromURI, +} from './uri-utils.js'; /******************************************************************************/ @@ -133,68 +130,6 @@ const FilteringContext = class { return (this.itype & FONT_ANY) !== 0; } - fromTabId(tabId) { - const tabContext = µBlock.tabContextManager.mustLookup(tabId); - this.tabOrigin = tabContext.origin; - this.tabHostname = tabContext.rootHostname; - this.tabDomain = tabContext.rootDomain; - this.tabId = tabContext.tabId; - return this; - } - - // https://github.com/uBlockOrigin/uBlock-issues/issues/459 - // In case of a request for frame and if ever no context is specified, - // assume the origin of the context is the same as the request itself. - fromWebrequestDetails(details) { - const tabId = details.tabId; - this.type = details.type; - if ( this.itype === MAIN_FRAME && tabId > 0 ) { - µBlock.tabContextManager.push(tabId, details.url); - } - this.fromTabId(tabId); // Must be called AFTER tab context management - this.realm = ''; - this.id = details.requestId; - this.setURL(details.url); - this.aliasURL = details.aliasURL || undefined; - if ( this.itype !== SUB_FRAME ) { - this.docId = details.frameId; - this.frameId = -1; - } else { - this.docId = details.parentFrameId; - this.frameId = details.frameId; - } - if ( this.tabId > 0 ) { - if ( this.docId === 0 ) { - this.docOrigin = this.tabOrigin; - this.docHostname = this.tabHostname; - this.docDomain = this.tabDomain; - } else if ( details.documentUrl !== undefined ) { - this.setDocOriginFromURL(details.documentUrl); - } else { - const pageStore = µBlock.pageStoreFromTabId(this.tabId); - const docStore = pageStore && pageStore.getFrameStore(this.docId); - if ( docStore ) { - this.setDocOriginFromURL(docStore.rawURL); - } else { - this.setDocOrigin(this.tabOrigin); - } - } - } else if ( details.documentUrl !== undefined ) { - const origin = originFromURI( - µBlock.normalizePageURL(0, details.documentUrl) - ); - this.setDocOrigin(origin).setTabOrigin(origin); - } else if ( this.docId === -1 || (this.itype & FRAME_ANY) !== 0 ) { - const origin = originFromURI(this.url); - this.setDocOrigin(origin).setTabOrigin(origin); - } else { - this.setDocOrigin(this.tabOrigin); - } - this.redirectURL = undefined; - this.filter = undefined; - return this; - } - fromFilteringContext(other) { this.realm = other.realm; this.type = other.type; @@ -331,12 +266,6 @@ const FilteringContext = class { } getTabOrigin() { - if ( this.tabOrigin === undefined ) { - const tabContext = µBlock.tabContextManager.mustLookup(this.tabId); - this.tabOrigin = tabContext.origin; - this.tabHostname = tabContext.rootHostname; - this.tabDomain = tabContext.rootDomain; - } return this.tabOrigin; } @@ -353,9 +282,6 @@ const FilteringContext = class { } getTabHostname() { - if ( this.tabHostname === undefined ) { - this.tabHostname = hostnameFromURI(this.getTabOrigin()); - } return this.tabHostname; } @@ -422,29 +348,6 @@ const FilteringContext = class { } return this; } - - toLogger() { - this.tstamp = Date.now(); - if ( this.domain === undefined ) { - void this.getDomain(); - } - if ( this.docDomain === undefined ) { - void this.getDocDomain(); - } - if ( this.tabDomain === undefined ) { - void this.getTabDomain(); - } - const logger = µBlock.logger; - const filters = this.filter; - // Many filters may have been applied to the current context - if ( Array.isArray(filters) === false ) { - return logger.writeOne(this); - } - for ( const filter of filters ) { - this.filter = filter; - logger.writeOne(this); - } - } }; /******************************************************************************/ @@ -475,8 +378,4 @@ FilteringContext.prototype.SCRIPT_ANY = FilteringContext.SCRIPT_ANY = SCRIPT_ANY /******************************************************************************/ -µBlock.FilteringContext = FilteringContext; -µBlock.filteringContext = new FilteringContext(); - -// <<<<< end of local scope -} +export { FilteringContext }; diff --git a/src/js/globals.js b/src/js/globals.js new file mode 100644 index 000000000..3401bd8c2 --- /dev/null +++ b/src/js/globals.js @@ -0,0 +1,53 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/globalThis + +const globals = (( ) => { + // jshint ignore:start + if ( typeof globalThis !== 'undefined' ) { return globalThis; } + if ( typeof self !== 'undefined' ) { return self; } + if ( typeof global !== 'undefined' ) { return global; } + // jshint ignore:end +})(); + +// https://en.wikipedia.org/wiki/.invalid +if ( globals.location === undefined ) { + globals.location = new URL('https://ublock0.invalid/'); +} + +// https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback +if ( globals.requestIdleCallback === undefined ) { + globals.requestIdleCallback = function(callback) { + return globals.setTimeout(callback, 1); + }; + globals.cancelIdleCallback = function(handle) { + return globals.clearTimeout(handle); + }; +} + +/******************************************************************************/ + +export default globals; diff --git a/src/js/hnswitches.js b/src/js/hnswitches.js index 4f00517de..9df8a7494 100644 --- a/src/js/hnswitches.js +++ b/src/js/hnswitches.js @@ -19,24 +19,31 @@ Home: https://github.com/gorhill/uBlock */ -/* global punycode */ /* jshint bitwise: false */ 'use strict'; /******************************************************************************/ -µBlock.HnSwitches = (( ) => { +import '../lib/punycode.js'; + +import globals from './globals.js'; +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; /******************************************************************************/ -var HnSwitches = function() { +const punycode = globals.punycode; + +/******************************************************************************/ + +const HnSwitches = function() { this.reset(); }; /******************************************************************************/ -var switchBitOffsets = { +const switchBitOffsets = { 'no-strict-blocking': 0, 'no-popups': 2, 'no-cosmetic-filtering': 4, @@ -46,12 +53,12 @@ var switchBitOffsets = { 'no-scripting': 12, }; -var switchStateToNameMap = { +const switchStateToNameMap = { '1': 'true', '2': 'false' }; -var nameToSwitchStateMap = { +const nameToSwitchStateMap = { 'true': 1, 'false': 2, 'on': 1, @@ -61,7 +68,7 @@ var nameToSwitchStateMap = { /******************************************************************************/ // For performance purpose, as simple tests as possible -var reNotASCII = /[^\x20-\x7F]/; +const reNotASCII = /[^\x20-\x7F]/; // http://tools.ietf.org/html/rfc5952 // 4.3: "MUST be represented in lowercase" @@ -82,14 +89,14 @@ HnSwitches.prototype.reset = function() { HnSwitches.prototype.assign = function(from) { // Remove rules not in other - for ( let hn of this.switches.keys() ) { + for ( const hn of this.switches.keys() ) { if ( from.switches.has(hn) === false ) { this.switches.delete(hn); this.changed = true; } } // Add/change rules in other - for ( let [hn, bits] of from.switches ) { + for ( const [hn, bits] of from.switches ) { if ( this.switches.get(hn) !== bits ) { this.switches.set(hn, bits); this.changed = true; @@ -100,8 +107,8 @@ HnSwitches.prototype.assign = function(from) { /******************************************************************************/ HnSwitches.prototype.copyRules = function(from, srcHostname) { - let thisBits = this.switches.get(srcHostname); - let fromBits = from.switches.get(srcHostname); + const thisBits = this.switches.get(srcHostname); + const fromBits = from.switches.get(srcHostname); if ( fromBits !== thisBits ) { if ( fromBits !== undefined ) { this.switches.set(srcHostname, fromBits); @@ -124,7 +131,7 @@ HnSwitches.prototype.hasSameRules = function(other, srcHostname) { // If value is undefined, the switch is removed HnSwitches.prototype.toggle = function(switchName, hostname, newVal) { - let bitOffset = switchBitOffsets[switchName]; + const bitOffset = switchBitOffsets[switchName]; if ( bitOffset === undefined ) { return false; } if ( newVal === this.evaluate(switchName, hostname) ) { return false; } let bits = this.switches.get(hostname) || 0; @@ -142,7 +149,7 @@ HnSwitches.prototype.toggle = function(switchName, hostname, newVal) { /******************************************************************************/ HnSwitches.prototype.toggleOneZ = function(switchName, hostname, newState) { - let bitOffset = switchBitOffsets[switchName]; + const bitOffset = switchBitOffsets[switchName]; if ( bitOffset === undefined ) { return false; } let state = this.evaluateZ(switchName, hostname); if ( newState === state ) { return false; } @@ -171,8 +178,8 @@ HnSwitches.prototype.toggleBranchZ = function(switchName, targetHostname, newSta // Turn off all descendant switches, they will inherit the state of the // branch's origin. - let targetLen = targetHostname.length; - for ( let hostname of this.switches.keys() ) { + const targetLen = targetHostname.length; + for ( const hostname of this.switches.keys() ) { if ( hostname === targetHostname ) { continue; } if ( hostname.length <= targetLen ) { continue; } if ( hostname.endsWith(targetHostname) === false ) { continue; } @@ -201,7 +208,7 @@ HnSwitches.prototype.toggleZ = function(switchName, hostname, deep, newState) { // 2 = forced default state (to override a broader non-default state) HnSwitches.prototype.evaluate = function(switchName, hostname) { - let bits = this.switches.get(hostname); + const bits = this.switches.get(hostname); if ( bits === undefined ) { return 0; } @@ -215,14 +222,14 @@ HnSwitches.prototype.evaluate = function(switchName, hostname) { /******************************************************************************/ HnSwitches.prototype.evaluateZ = function(switchName, hostname) { - let bitOffset = switchBitOffsets[switchName]; + const bitOffset = switchBitOffsets[switchName]; if ( bitOffset === undefined ) { this.r = 0; return false; } this.n = switchName; µBlock.decomposeHostname(hostname, this.decomposedSource); - for ( let shn of this.decomposedSource ) { + for ( const shn of this.decomposedSource ) { let bits = this.switches.get(shn); if ( bits !== undefined ) { bits = bits >>> bitOffset & 3; @@ -250,16 +257,16 @@ HnSwitches.prototype.toLogData = function() { /******************************************************************************/ HnSwitches.prototype.toArray = function() { - let out = [], - toUnicode = punycode.toUnicode; + const out = []; + const toUnicode = punycode.toUnicode; for ( let hostname of this.switches.keys() ) { - for ( var switchName in switchBitOffsets ) { + for ( const switchName in switchBitOffsets ) { if ( switchBitOffsets.hasOwnProperty(switchName) === false ) { continue; } - let val = this.evaluate(switchName, hostname); + const val = this.evaluate(switchName, hostname); if ( val === 0 ) { continue; } - if ( hostname.indexOf('xn--') !== -1 ) { + if ( hostname.includes('xn--') ) { hostname = toUnicode(hostname); } out.push(switchName + ': ' + hostname + ' ' + switchStateToNameMap[val]); @@ -275,7 +282,7 @@ HnSwitches.prototype.toString = function() { /******************************************************************************/ HnSwitches.prototype.fromString = function(text, append) { - let lineIter = new µBlock.LineIterator(text); + const lineIter = new LineIterator(text); if ( append !== true ) { this.reset(); } while ( lineIter.eot() === false ) { this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); @@ -297,7 +304,7 @@ HnSwitches.prototype.validateRuleParts = function(parts) { HnSwitches.prototype.addFromRuleParts = function(parts) { if ( this.validateRuleParts(parts) !== undefined ) { - let switchName = parts[0].slice(0, -1); + const switchName = parts[0].slice(0, -1); if ( switchBitOffsets.hasOwnProperty(switchName) ) { this.toggle(switchName, parts[1], nameToSwitchStateMap[parts[2]]); return true; @@ -316,15 +323,11 @@ HnSwitches.prototype.removeFromRuleParts = function(parts) { /******************************************************************************/ -return HnSwitches; - -/******************************************************************************/ - -})(); - -/******************************************************************************/ - -µBlock.sessionSwitches = new µBlock.HnSwitches(); -µBlock.permanentSwitches = new µBlock.HnSwitches(); +// Export + +µBlock.HnSwitches = HnSwitches; + +µBlock.sessionSwitches = new HnSwitches(); +µBlock.permanentSwitches = new HnSwitches(); /******************************************************************************/ diff --git a/src/js/hntrie.js b/src/js/hntrie.js index dd9c82e67..f18c13a90 100644 --- a/src/js/hntrie.js +++ b/src/js/hntrie.js @@ -23,11 +23,6 @@ 'use strict'; -// ***************************************************************************** -// start of local namespace - -{ - /******************************************************************************* The original prototype was to develop an idea I had about using jump indices @@ -461,10 +456,10 @@ const HNTrieContainer = class { return n === hr || hn.charCodeAt(hl-1) === 0x2E /* '.' */; } - async enableWASM() { + async enableWASM(modulePath) { if ( typeof WebAssembly !== 'object' ) { return false; } if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; } - const module = await getWasmModule(); + const module = await getWasmModule(modulePath); if ( module instanceof WebAssembly.Module === false ) { return false; } const memory = new WebAssembly.Memory({ initial: 2 }); const instance = await WebAssembly.instantiate(module, { @@ -772,20 +767,7 @@ HNTrieContainer.prototype.HNTrieRef.prototype.needle = ''; const getWasmModule = (( ) => { let wasmModulePromise; - // The directory from which the current script was fetched should also - // contain the related WASM file. The script is fetched from a trusted - // location, and consequently so will be the related WASM file. - let workingDir; - { - const url = new URL(document.currentScript.src); - const match = /[^\/]+$/.exec(url.pathname); - if ( match !== null ) { - url.pathname = url.pathname.slice(0, match.index); - } - workingDir = url.href; - } - - return async function() { + return async function(modulePath) { if ( wasmModulePromise instanceof Promise ) { return wasmModulePromise; } @@ -809,7 +791,7 @@ const getWasmModule = (( ) => { if ( uint8s[0] !== 1 ) { return; } wasmModulePromise = fetch( - workingDir + 'wasm/hntrie.wasm', + `${modulePath}/wasm/hntrie.wasm`, { mode: 'same-origin' } ).then( WebAssembly.compileStreaming @@ -823,9 +805,4 @@ const getWasmModule = (( ) => { /******************************************************************************/ -µBlock.HNTrieContainer = HNTrieContainer; - -// end of local namespace -// ***************************************************************************** - -} +export { HNTrieContainer }; diff --git a/src/js/html-filtering.js b/src/js/html-filtering.js index 8431d670c..52e9bb962 100644 --- a/src/js/html-filtering.js +++ b/src/js/html-filtering.js @@ -23,421 +23,426 @@ /******************************************************************************/ -µBlock.htmlFilteringEngine = (function() { - const µb = µBlock; - const pselectors = new Map(); - const duplicates = new Set(); - - const filterDB = new µb.staticExtFilteringEngine.HostnameBasedDB(2); - const sessionFilterDB = new µb.staticExtFilteringEngine.SessionDB(); - - let acceptedCount = 0; - let discardedCount = 0; - let docRegister; - - const api = { - get acceptedCount() { - return acceptedCount; - }, - get discardedCount() { - return discardedCount; - } - }; - - const PSelectorHasTextTask = class { - constructor(task) { - let arg0 = task[1], arg1; - if ( Array.isArray(task[1]) ) { - arg1 = arg0[1]; arg0 = arg0[0]; - } - this.needle = new RegExp(arg0, arg1); - } - transpose(node, output) { - if ( this.needle.test(node.textContent) ) { - output.push(node); - } - } - }; - - const PSelectorIfTask = class { - constructor(task) { - this.pselector = new PSelector(task[1]); - } - transpose(node, output) { - if ( this.pselector.test(node) === this.target ) { - output.push(node); - } - } - get invalid() { - return this.pselector.invalid; - } - }; - PSelectorIfTask.prototype.target = true; - - const PSelectorIfNotTask = class extends PSelectorIfTask { - }; - PSelectorIfNotTask.prototype.target = false; - - const PSelectorMinTextLengthTask = class { - constructor(task) { - this.min = task[1]; - } - transpose(node, output) { - if ( node.textContent.length >= this.min ) { - output.push(node); - } - } - }; - - const PSelectorSpathTask = class { - constructor(task) { - this.spath = task[1]; - } - transpose(node, output) { - const parent = node.parentElement; - if ( parent === null ) { return; } - let pos = 1; - for (;;) { - node = node.previousElementSibling; - if ( node === null ) { break; } - pos += 1; - } - const nodes = parent.querySelectorAll( - `:scope > :nth-child(${pos})${this.spath}` - ); - for ( const node of nodes ) { - output.push(node); - } - } - }; - - const PSelectorUpwardTask = class { - constructor(task) { - const arg = task[1]; - if ( typeof arg === 'number' ) { - this.i = arg; - } else { - this.s = arg; - } - } - transpose(node, output) { - if ( this.s !== '' ) { - const parent = node.parentElement; - if ( parent === null ) { return; } - node = parent.closest(this.s); - if ( node === null ) { return; } - } else { - let nth = this.i; - for (;;) { - node = node.parentElement; - if ( node === null ) { return; } - nth -= 1; - if ( nth === 0 ) { break; } - } - } - output.push(node); - } - }; - PSelectorUpwardTask.prototype.i = 0; - PSelectorUpwardTask.prototype.s = ''; - - const PSelectorXpathTask = class { - constructor(task) { - this.xpe = task[1]; - } - transpose(node, output) { - const xpr = docRegister.evaluate( - this.xpe, - node, - null, - XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, - null - ); - let j = xpr.snapshotLength; - while ( j-- ) { - const node = xpr.snapshotItem(j); - if ( node.nodeType === 1 ) { - output.push(node); - } - } - } - }; - - const PSelector = class { - constructor(o) { - this.raw = o.raw; - this.selector = o.selector; - this.tasks = []; - if ( !o.tasks ) { return; } - for ( const task of o.tasks ) { - const ctor = this.operatorToTaskMap.get(task[0]); - if ( ctor === undefined ) { - this.invalid = true; - break; - } - const pselector = new ctor(task); - if ( pselector instanceof PSelectorIfTask && pselector.invalid ) { - this.invalid = true; - break; - } - this.tasks.push(pselector); - } - } - prime(input) { - const root = input || docRegister; - if ( this.selector === '' ) { return [ root ]; } - return Array.from(root.querySelectorAll(this.selector)); - } - exec(input) { - if ( this.invalid ) { return []; } - let nodes = this.prime(input); - for ( const task of this.tasks ) { - if ( nodes.length === 0 ) { break; } - const transposed = []; - for ( const node of nodes ) { - task.transpose(node, transposed); - } - nodes = transposed; - } - return nodes; - } - test(input) { - if ( this.invalid ) { return false; } - const nodes = this.prime(input); - for ( const node of nodes ) { - let output = [ node ]; - for ( const task of this.tasks ) { - const transposed = []; - for ( const node of output ) { - task.transpose(node, transposed); - } - output = transposed; - if ( output.length === 0 ) { break; } - } - if ( output.length !== 0 ) { return true; } - } - return false; - } - }; - PSelector.prototype.operatorToTaskMap = new Map([ - [ ':has', PSelectorIfTask ], - [ ':has-text', PSelectorHasTextTask ], - [ ':if', PSelectorIfTask ], - [ ':if-not', PSelectorIfNotTask ], - [ ':min-text-length', PSelectorMinTextLengthTask ], - [ ':not', PSelectorIfNotTask ], - [ ':nth-ancestor', PSelectorUpwardTask ], - [ ':spath', PSelectorSpathTask ], - [ ':upward', PSelectorUpwardTask ], - [ ':xpath', PSelectorXpathTask ], - ]); - PSelector.prototype.invalid = false; - - const logOne = function(details, exception, selector) { - µBlock.filteringContext - .duplicate() - .fromTabId(details.tabId) - .setRealm('extended') - .setType('dom') - .setURL(details.url) - .setDocOriginFromURL(details.url) - .setFilter({ - source: 'extended', - raw: `${exception === 0 ? '##' : '#@#'}^${selector}` - }) - .toLogger(); - }; - - const applyProceduralSelector = function(details, selector) { - let pselector = pselectors.get(selector); - if ( pselector === undefined ) { - pselector = new PSelector(JSON.parse(selector)); - pselectors.set(selector, pselector); - } - const nodes = pselector.exec(); - let modified = false; - for ( const node of nodes ) { - node.remove(); - modified = true; - } - if ( modified && µb.logger.enabled ) { - logOne(details, 0, pselector.raw); - } - return modified; - }; - - const applyCSSSelector = function(details, selector) { - const nodes = docRegister.querySelectorAll(selector); - let modified = false; - for ( const node of nodes ) { - node.remove(); - modified = true; - } - if ( modified && µb.logger.enabled ) { - logOne(details, 0, selector); - } - return modified; - }; - - api.reset = function() { - filterDB.clear(); - pselectors.clear(); - duplicates.clear(); - acceptedCount = 0; - discardedCount = 0; - }; - - api.freeze = function() { - duplicates.clear(); - filterDB.collectGarbage(); - }; - - api.compile = function(parser, writer) { - const { raw, compiled, exception } = parser.result; - if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; - µb.logger.writeOne({ - realm: 'message', - type: 'error', - text: `Invalid HTML filter in ${who}: ##${raw}` - }); - return; - } - - writer.select(µb.compiledHTMLSection); - - // TODO: Mind negated hostnames, they are currently discarded. - - for ( const { hn, not, bad } of parser.extOptions() ) { - if ( bad ) { continue; } - let kind = 0; - if ( exception ) { - if ( not ) { continue; } - kind |= 0b01; - } - if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { - kind |= 0b10; - } - writer.push([ 64, hn, kind, compiled ]); - } - }; - - api.compileTemporary = function(parser) { - return { - session: sessionFilterDB, - selector: parser.result.compiled, - }; - }; - - api.fromCompiledContent = function(reader) { - // Don't bother loading filters if stream filtering is not supported. - if ( µb.canFilterResponseData === false ) { return; } - - reader.select(µb.compiledHTMLSection); - - while ( reader.next() ) { - acceptedCount += 1; - const fingerprint = reader.fingerprint(); - if ( duplicates.has(fingerprint) ) { - discardedCount += 1; - continue; - } - duplicates.add(fingerprint); - const args = reader.args(); - filterDB.store(args[1], args[2], args[3]); - } - }; - - api.getSession = function() { - return sessionFilterDB; - }; - - api.retrieve = function(details) { - const hostname = details.hostname; - - const plains = new Set(); - const procedurals = new Set(); - const exceptions = new Set(); - - if ( sessionFilterDB.isNotEmpty ) { - sessionFilterDB.retrieve([ null, exceptions ]); - } - filterDB.retrieve( - hostname, - [ plains, exceptions, procedurals, exceptions ] - ); - const entity = details.entity !== '' - ? `${hostname.slice(0, -details.domain.length)}${details.entity}` - : '*'; - filterDB.retrieve( - entity, - [ plains, exceptions, procedurals, exceptions ], - 1 - ); - - if ( plains.size === 0 && procedurals.size === 0 ) { return; } - - // https://github.com/gorhill/uBlock/issues/2835 - // Do not filter if the site is under an `allow` rule. - if ( - µb.userSettings.advancedUserEnabled && - µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 - ) { - return; - } - - const out = { plains, procedurals }; - - if ( exceptions.size === 0 ) { - return out; - } - - for ( const selector of exceptions ) { - if ( plains.has(selector) ) { - plains.delete(selector); - logOne(details, 1, selector); - continue; - } - if ( procedurals.has(selector) ) { - procedurals.delete(selector); - logOne(details, 1, JSON.parse(selector).raw); - continue; - } - } - - if ( plains.size !== 0 || procedurals.size !== 0 ) { - return out; - } - }; - - api.apply = function(doc, details) { - docRegister = doc; - let modified = false; - for ( const selector of details.selectors.plains ) { - if ( applyCSSSelector(details, selector) ) { - modified = true; - } - } - for ( const selector of details.selectors.procedurals ) { - if ( applyProceduralSelector(details, selector) ) { - modified = true; - } - } - - docRegister = undefined; - return modified; - }; - - api.toSelfie = function() { - return filterDB.toSelfie(); - }; - - api.fromSelfie = function(selfie) { - filterDB.fromSelfie(selfie); - pselectors.clear(); - }; - - return api; -})(); +import µBlock from './background.js'; + +/******************************************************************************/ + +const µb = µBlock; +const pselectors = new Map(); +const duplicates = new Set(); + +const filterDB = new µb.staticExtFilteringEngine.HostnameBasedDB(2); +const sessionFilterDB = new µb.staticExtFilteringEngine.SessionDB(); + +let acceptedCount = 0; +let discardedCount = 0; +let docRegister; + +const api = { + get acceptedCount() { + return acceptedCount; + }, + get discardedCount() { + return discardedCount; + } +}; + +const PSelectorHasTextTask = class { + constructor(task) { + let arg0 = task[1], arg1; + if ( Array.isArray(task[1]) ) { + arg1 = arg0[1]; arg0 = arg0[0]; + } + this.needle = new RegExp(arg0, arg1); + } + transpose(node, output) { + if ( this.needle.test(node.textContent) ) { + output.push(node); + } + } +}; + +const PSelectorIfTask = class { + constructor(task) { + this.pselector = new PSelector(task[1]); + } + transpose(node, output) { + if ( this.pselector.test(node) === this.target ) { + output.push(node); + } + } + get invalid() { + return this.pselector.invalid; + } +}; +PSelectorIfTask.prototype.target = true; + +const PSelectorIfNotTask = class extends PSelectorIfTask { +}; +PSelectorIfNotTask.prototype.target = false; + +const PSelectorMinTextLengthTask = class { + constructor(task) { + this.min = task[1]; + } + transpose(node, output) { + if ( node.textContent.length >= this.min ) { + output.push(node); + } + } +}; + +const PSelectorSpathTask = class { + constructor(task) { + this.spath = task[1]; + } + transpose(node, output) { + const parent = node.parentElement; + if ( parent === null ) { return; } + let pos = 1; + for (;;) { + node = node.previousElementSibling; + if ( node === null ) { break; } + pos += 1; + } + const nodes = parent.querySelectorAll( + `:scope > :nth-child(${pos})${this.spath}` + ); + for ( const node of nodes ) { + output.push(node); + } + } +}; + +const PSelectorUpwardTask = class { + constructor(task) { + const arg = task[1]; + if ( typeof arg === 'number' ) { + this.i = arg; + } else { + this.s = arg; + } + } + transpose(node, output) { + if ( this.s !== '' ) { + const parent = node.parentElement; + if ( parent === null ) { return; } + node = parent.closest(this.s); + if ( node === null ) { return; } + } else { + let nth = this.i; + for (;;) { + node = node.parentElement; + if ( node === null ) { return; } + nth -= 1; + if ( nth === 0 ) { break; } + } + } + output.push(node); + } +}; +PSelectorUpwardTask.prototype.i = 0; +PSelectorUpwardTask.prototype.s = ''; + +const PSelectorXpathTask = class { + constructor(task) { + this.xpe = task[1]; + } + transpose(node, output) { + const xpr = docRegister.evaluate( + this.xpe, + node, + null, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null + ); + let j = xpr.snapshotLength; + while ( j-- ) { + const node = xpr.snapshotItem(j); + if ( node.nodeType === 1 ) { + output.push(node); + } + } + } +}; + +const PSelector = class { + constructor(o) { + this.raw = o.raw; + this.selector = o.selector; + this.tasks = []; + if ( !o.tasks ) { return; } + for ( const task of o.tasks ) { + const ctor = this.operatorToTaskMap.get(task[0]); + if ( ctor === undefined ) { + this.invalid = true; + break; + } + const pselector = new ctor(task); + if ( pselector instanceof PSelectorIfTask && pselector.invalid ) { + this.invalid = true; + break; + } + this.tasks.push(pselector); + } + } + prime(input) { + const root = input || docRegister; + if ( this.selector === '' ) { return [ root ]; } + return Array.from(root.querySelectorAll(this.selector)); + } + exec(input) { + if ( this.invalid ) { return []; } + let nodes = this.prime(input); + for ( const task of this.tasks ) { + if ( nodes.length === 0 ) { break; } + const transposed = []; + for ( const node of nodes ) { + task.transpose(node, transposed); + } + nodes = transposed; + } + return nodes; + } + test(input) { + if ( this.invalid ) { return false; } + const nodes = this.prime(input); + for ( const node of nodes ) { + let output = [ node ]; + for ( const task of this.tasks ) { + const transposed = []; + for ( const node of output ) { + task.transpose(node, transposed); + } + output = transposed; + if ( output.length === 0 ) { break; } + } + if ( output.length !== 0 ) { return true; } + } + return false; + } +}; +PSelector.prototype.operatorToTaskMap = new Map([ + [ ':has', PSelectorIfTask ], + [ ':has-text', PSelectorHasTextTask ], + [ ':if', PSelectorIfTask ], + [ ':if-not', PSelectorIfNotTask ], + [ ':min-text-length', PSelectorMinTextLengthTask ], + [ ':not', PSelectorIfNotTask ], + [ ':nth-ancestor', PSelectorUpwardTask ], + [ ':spath', PSelectorSpathTask ], + [ ':upward', PSelectorUpwardTask ], + [ ':xpath', PSelectorXpathTask ], +]); +PSelector.prototype.invalid = false; + +const logOne = function(details, exception, selector) { + µBlock.filteringContext + .duplicate() + .fromTabId(details.tabId) + .setRealm('extended') + .setType('dom') + .setURL(details.url) + .setDocOriginFromURL(details.url) + .setFilter({ + source: 'extended', + raw: `${exception === 0 ? '##' : '#@#'}^${selector}` + }) + .toLogger(); +}; + +const applyProceduralSelector = function(details, selector) { + let pselector = pselectors.get(selector); + if ( pselector === undefined ) { + pselector = new PSelector(JSON.parse(selector)); + pselectors.set(selector, pselector); + } + const nodes = pselector.exec(); + let modified = false; + for ( const node of nodes ) { + node.remove(); + modified = true; + } + if ( modified && µb.logger.enabled ) { + logOne(details, 0, pselector.raw); + } + return modified; +}; + +const applyCSSSelector = function(details, selector) { + const nodes = docRegister.querySelectorAll(selector); + let modified = false; + for ( const node of nodes ) { + node.remove(); + modified = true; + } + if ( modified && µb.logger.enabled ) { + logOne(details, 0, selector); + } + return modified; +}; + +api.reset = function() { + filterDB.clear(); + pselectors.clear(); + duplicates.clear(); + acceptedCount = 0; + discardedCount = 0; +}; + +api.freeze = function() { + duplicates.clear(); + filterDB.collectGarbage(); +}; + +api.compile = function(parser, writer) { + const { raw, compiled, exception } = parser.result; + if ( compiled === undefined ) { + const who = writer.properties.get('name') || '?'; + µb.logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid HTML filter in ${who}: ##${raw}` + }); + return; + } + + writer.select(µb.compiledHTMLSection); + + // TODO: Mind negated hostnames, they are currently discarded. + + for ( const { hn, not, bad } of parser.extOptions() ) { + if ( bad ) { continue; } + let kind = 0; + if ( exception ) { + if ( not ) { continue; } + kind |= 0b01; + } + if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { + kind |= 0b10; + } + writer.push([ 64, hn, kind, compiled ]); + } +}; + +api.compileTemporary = function(parser) { + return { + session: sessionFilterDB, + selector: parser.result.compiled, + }; +}; + +api.fromCompiledContent = function(reader) { + // Don't bother loading filters if stream filtering is not supported. + if ( µb.canFilterResponseData === false ) { return; } + + reader.select(µb.compiledHTMLSection); + + while ( reader.next() ) { + acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( duplicates.has(fingerprint) ) { + discardedCount += 1; + continue; + } + duplicates.add(fingerprint); + const args = reader.args(); + filterDB.store(args[1], args[2], args[3]); + } +}; + +api.getSession = function() { + return sessionFilterDB; +}; + +api.retrieve = function(details) { + const hostname = details.hostname; + + const plains = new Set(); + const procedurals = new Set(); + const exceptions = new Set(); + + if ( sessionFilterDB.isNotEmpty ) { + sessionFilterDB.retrieve([ null, exceptions ]); + } + filterDB.retrieve( + hostname, + [ plains, exceptions, procedurals, exceptions ] + ); + const entity = details.entity !== '' + ? `${hostname.slice(0, -details.domain.length)}${details.entity}` + : '*'; + filterDB.retrieve( + entity, + [ plains, exceptions, procedurals, exceptions ], + 1 + ); + + if ( plains.size === 0 && procedurals.size === 0 ) { return; } + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not filter if the site is under an `allow` rule. + if ( + µb.userSettings.advancedUserEnabled && + µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 + ) { + return; + } + + const out = { plains, procedurals }; + + if ( exceptions.size === 0 ) { + return out; + } + + for ( const selector of exceptions ) { + if ( plains.has(selector) ) { + plains.delete(selector); + logOne(details, 1, selector); + continue; + } + if ( procedurals.has(selector) ) { + procedurals.delete(selector); + logOne(details, 1, JSON.parse(selector).raw); + continue; + } + } + + if ( plains.size !== 0 || procedurals.size !== 0 ) { + return out; + } +}; + +api.apply = function(doc, details) { + docRegister = doc; + let modified = false; + for ( const selector of details.selectors.plains ) { + if ( applyCSSSelector(details, selector) ) { + modified = true; + } + } + for ( const selector of details.selectors.procedurals ) { + if ( applyProceduralSelector(details, selector) ) { + modified = true; + } + } + docRegister = undefined; + return modified; +}; + +api.toSelfie = function() { + return filterDB.toSelfie(); +}; + +api.fromSelfie = function(selfie) { + filterDB.fromSelfie(selfie); + pselectors.clear(); +}; + +/******************************************************************************/ + +// Export + +µBlock.htmlFilteringEngine = api; /******************************************************************************/ diff --git a/src/js/httpheader-filtering.js b/src/js/httpheader-filtering.js index 52e78721a..19c6b40e7 100644 --- a/src/js/httpheader-filtering.js +++ b/src/js/httpheader-filtering.js @@ -23,8 +23,8 @@ /******************************************************************************/ -{ -// >>>>> start of local scope +import { entityFromDomain } from './uri-utils.js'; +import µBlock from './background.js'; /******************************************************************************/ @@ -157,7 +157,7 @@ api.apply = function(fctxt, headers) { if ( hostname === '' ) { return; } const domain = fctxt.getDomain(); - let entity = µb.URI.entityFromDomain(domain); + let entity = entityFromDomain(domain); if ( entity !== '' ) { entity = `${hostname.slice(0, -domain.length)}${entity}`; } else { @@ -218,11 +218,10 @@ api.fromSelfie = function(selfie) { filterDB.fromSelfie(selfie); }; +/******************************************************************************/ + +// Export + µb.httpheaderFilteringEngine = api; /******************************************************************************/ - -// <<<<< end of local scope -} - -/******************************************************************************/ diff --git a/src/js/logger-ui-inspector.js b/src/js/logger-ui-inspector.js index 17fdb5cad..26b892198 100644 --- a/src/js/logger-ui-inspector.js +++ b/src/js/logger-ui-inspector.js @@ -25,6 +25,10 @@ /******************************************************************************/ +import globals from './globals.js'; + +/******************************************************************************/ + (( ) => { /******************************************************************************/ @@ -43,7 +47,7 @@ if ( /******************************************************************************/ -var logger = self.logger; +const logger = globals.logger; var inspectorConnectionId; var inspectedTabId = 0; var inspectedURL = ''; diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index 2dd738d2f..88175b931 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -25,7 +25,8 @@ /******************************************************************************/ -(( ) => { +import globals from './globals.js'; +import { hostnameFromURI } from './uri-utils.js'; /******************************************************************************/ @@ -33,7 +34,7 @@ // accumulated over time. const messaging = vAPI.messaging; -const logger = self.logger = { ownerId: Date.now() }; +const logger = globals.logger = { ownerId: Date.now() }; const logDate = new Date(); const logDateTimezoneOffset = logDate.getTimezoneOffset() * 60000; const loggerEntries = []; @@ -1677,8 +1678,8 @@ const reloadTab = function(ev) { const aliasURL = text ? aliasURLFromID(text) : ''; if ( aliasURL !== '' ) { rows[8].children[1].textContent = - vAPI.hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' + - vAPI.hostnameFromURI(canonicalURL); + hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' + + hostnameFromURI(canonicalURL); rows[9].children[1].textContent = aliasURL; } else { rows[8].style.display = 'none'; @@ -2891,5 +2892,3 @@ if ( self.location.search.includes('popup=1') ) { } /******************************************************************************/ - -})(); diff --git a/src/js/logger.js b/src/js/logger.js index 6e3741402..eb417d306 100644 --- a/src/js/logger.js +++ b/src/js/logger.js @@ -22,70 +22,73 @@ 'use strict'; /******************************************************************************/ + +import µBlock from './background.js'; + /******************************************************************************/ -µBlock.logger = (function() { +let buffer = null; +let lastReadTime = 0; +let writePtr = 0; - let buffer = null; - let lastReadTime = 0; - let writePtr = 0; +// After 60 seconds without being read, a buffer will be considered +// unused, and thus removed from memory. +const logBufferObsoleteAfter = 30 * 1000; - // After 60 seconds without being read, a buffer will be considered - // unused, and thus removed from memory. - const logBufferObsoleteAfter = 30 * 1000; +const janitor = ( ) => { + if ( + buffer !== null && + lastReadTime < (Date.now() - logBufferObsoleteAfter) + ) { + api.enabled = false; + buffer = null; + writePtr = 0; + api.ownerId = undefined; + vAPI.messaging.broadcast({ what: 'loggerDisabled' }); + } + if ( buffer !== null ) { + vAPI.setTimeout(janitor, logBufferObsoleteAfter); + } +}; - const janitor = ( ) => { - if ( - buffer !== null && - lastReadTime < (Date.now() - logBufferObsoleteAfter) - ) { - api.enabled = false; - buffer = null; - writePtr = 0; - api.ownerId = undefined; - vAPI.messaging.broadcast({ what: 'loggerDisabled' }); +const boxEntry = function(details) { + if ( details.tstamp === undefined ) { + details.tstamp = Date.now(); + } + return JSON.stringify(details); +}; + +const api = { + enabled: false, + ownerId: undefined, + writeOne: function(details) { + if ( buffer === null ) { return; } + const box = boxEntry(details); + if ( writePtr === buffer.length ) { + buffer.push(box); + } else { + buffer[writePtr] = box; } - if ( buffer !== null ) { + writePtr += 1; + }, + readAll: function(ownerId) { + this.ownerId = ownerId; + if ( buffer === null ) { + this.enabled = true; + buffer = []; vAPI.setTimeout(janitor, logBufferObsoleteAfter); } - }; - - const boxEntry = function(details) { - if ( details.tstamp === undefined ) { - details.tstamp = Date.now(); - } - return JSON.stringify(details); - }; - - const api = { - enabled: false, - ownerId: undefined, - writeOne: function(details) { - if ( buffer === null ) { return; } - const box = boxEntry(details); - if ( writePtr === buffer.length ) { - buffer.push(box); - } else { - buffer[writePtr] = box; - } - writePtr += 1; - }, - readAll: function(ownerId) { - this.ownerId = ownerId; - if ( buffer === null ) { - this.enabled = true; - buffer = []; - vAPI.setTimeout(janitor, logBufferObsoleteAfter); - } - const out = buffer.slice(0, writePtr); - writePtr = 0; - lastReadTime = Date.now(); - return out; - }, - }; - - return api; - -})(); + const out = buffer.slice(0, writePtr); + writePtr = 0; + lastReadTime = Date.now(); + return out; + }, +}; + +/******************************************************************************/ + +// Export + +µBlock.logger = api; /******************************************************************************/ diff --git a/src/js/lz4.js b/src/js/lz4.js index efb8e7d6c..539fc8a94 100644 --- a/src/js/lz4.js +++ b/src/js/lz4.js @@ -23,6 +23,10 @@ 'use strict'; +/******************************************************************************/ + +import µBlock from './background.js'; + /******************************************************************************* Experimental support for storage compression. @@ -32,9 +36,6 @@ **/ -{ -// >>>> Start of private namespace - /******************************************************************************/ let lz4CodecInstance; @@ -198,6 +199,3 @@ const decodeValue = function(inputArray) { }; /******************************************************************************/ - -// <<<< End of private namespace -} diff --git a/src/js/messaging.js b/src/js/messaging.js index bd311a15d..ab161940f 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -19,10 +19,27 @@ Home: https://github.com/gorhill/uBlock */ -/******************************************************************************/ +'use strict'; + /******************************************************************************/ -'use strict'; +import '../lib/publicsuffixlist/publicsuffixlist.js'; +import '../lib/punycode.js'; + +import globals from './globals.js'; + +import { + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; + +import { StaticFilteringParser } from './static-filtering-parser.js'; +import µBlock from './background.js'; + +/******************************************************************************/ // https://github.com/uBlockOrigin/uBlock-issues/issues/710 // Listeners have a name and a "privileged" status. @@ -51,12 +68,11 @@ const clickToLoad = function(request, sender) { }; const getDomainNames = function(targets) { - const µburi = µb.URI; return targets.map(target => { if ( typeof target !== 'string' ) { return ''; } return target.indexOf('/') !== -1 - ? µburi.domainFromURI(target) || '' - : µburi.domainFromHostname(target) || target; + ? domainFromURI(target) || '' + : domainFromHostname(target) || target; }); }; @@ -98,8 +114,13 @@ const onMessage = function(request, sender, callback) { return; case 'sfneBenchmark': - µb.staticNetFilteringEngine.benchmark().then(result => { - callback(result); + µb.loadBenchmarkDataset().then(requests => { + µb.staticNetFilteringEngine.benchmark( + requests, + { redirectEngine: µb.redirectEngine } + ).then(result => { + callback(result); + }); }); return; @@ -244,7 +265,7 @@ const getHostnameDict = function(hostnameDetailsMap, out) { for ( const hnDetails of hostnameDetailsMap.values() ) { const hostname = hnDetails.hostname; if ( hnDict[hostname] !== undefined ) { continue; } - const domain = vAPI.domainFromHostname(hostname) || hostname; + const domain = domainFromHostname(hostname) || hostname; const dnDetails = hostnameDetailsMap.get(domain) || { counts: createCounts() }; if ( hnDict[domain] === undefined ) { @@ -336,7 +357,7 @@ const popupDataFromTabId = function(tabId, tabTitle) { getHostnameDict(pageStore.getAllHostnameDetails(), r); r.contentLastModified = pageStore.contentLastModified; getFirewallRules(rootHostname, r); - r.canElementPicker = µb.URI.isNetworkURI(r.rawURL); + r.canElementPicker = isNetworkURI(r.rawURL); r.noPopups = µb.sessionSwitches.evaluateZ( 'no-popups', rootHostname @@ -568,9 +589,9 @@ const retrieveContentScriptParameters = async function(sender, request) { request.tabId = tabId; request.frameId = frameId; - request.hostname = µb.URI.hostnameFromURI(request.url); - request.domain = µb.URI.domainFromHostname(request.hostname); - request.entity = µb.URI.entityFromDomain(request.domain); + request.hostname = hostnameFromURI(request.url); + request.domain = domainFromHostname(request.hostname); + request.entity = entityFromDomain(request.domain); response.specificCosmeticFilters = µb.cosmeticFilteringEngine.retrieveSpecificSelectors(request, response); @@ -597,7 +618,7 @@ const retrieveContentScriptParameters = async function(sender, request) { // effective URL is available here in `request.url`. if ( µb.canInjectScriptletsNow === false || - µb.URI.isNetworkURI(sender.frameURL) === false + isNetworkURI(sender.frameURL) === false ) { response.scriptlets = µb.scriptletFilteringEngine.retrieve(request); } @@ -1017,16 +1038,15 @@ const resetUserData = async function() { // Filter lists const prepListEntries = function(entries) { - const µburi = µb.URI; for ( const k in entries ) { if ( entries.hasOwnProperty(k) === false ) { continue; } const entry = entries[k]; if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { - entry.supportName = µburi.hostnameFromURI(entry.supportURL); + entry.supportName = hostnameFromURI(entry.supportURL); } else if ( typeof entry.homeURL === 'string' && entry.homeURL !== '' ) { - const hn = µburi.hostnameFromURI(entry.homeURL); + const hn = hostnameFromURI(entry.homeURL); entry.supportURL = `http://${hn}/`; - entry.supportName = µburi.domainFromHostname(hn); + entry.supportName = domainFromHostname(hn); } } }; @@ -1059,7 +1079,7 @@ const getLists = async function(callback) { // TODO: also return origin of embedded frames? const getOriginHints = function() { - const punycode = self.punycode; + const punycode = globals.punycode; const out = new Set(); for ( const tabId of µb.pageStores.keys() ) { if ( tabId === -1 ) { continue; } @@ -1088,7 +1108,7 @@ const getRules = function() { µb.sessionSwitches.toArray(), µb.sessionURLFiltering.toArray() ), - pslSelfie: self.publicSuffixList.toSelfie(), + pslSelfie: globals.publicSuffixList.toSelfie(), }; }; @@ -1393,7 +1413,7 @@ const getURLFilteringData = function(details) { }; const compileTemporaryException = function(filter) { - const parser = new vAPI.StaticFilteringParser(); + const parser = new StaticFilteringParser(); parser.analyze(filter); if ( parser.shouldDiscard() ) { return; } return µb.staticExtFilteringEngine.compileTemporary(parser); diff --git a/src/js/pagestore.js b/src/js/pagestore.js index 676632118..64d14c2b4 100644 --- a/src/js/pagestore.js +++ b/src/js/pagestore.js @@ -21,6 +21,16 @@ 'use strict'; +/******************************************************************************/ + +import { + domainFromHostname, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; + +import µBlock from './background.js'; + /******************************************************************************* A PageRequestStore object is used to store net requests in two ways: @@ -30,11 +40,6 @@ To create a log of net requests **/ -{ - -// start of private namespace -// >>>>> - /******************************************************************************/ const µb = µBlock; @@ -188,9 +193,8 @@ const FrameStore = class { this.clickToLoad = false; this.rawURL = frameURL; if ( frameURL !== undefined ) { - this.hostname = vAPI.hostnameFromURI(frameURL); - this.domain = - vAPI.domainFromHostname(this.hostname) || this.hostname; + this.hostname = hostnameFromURI(frameURL); + this.domain = domainFromHostname(this.hostname) || this.hostname; } // Evaluated on-demand // - 0b01: specific cosmetic filtering @@ -213,7 +217,7 @@ const FrameStore = class { } this._cosmeticFilteringBits = 0b11; { - const result = µb.staticNetFilteringEngine.matchStringReverse( + const result = µb.staticNetFilteringEngine.matchRequestReverse( 'specifichide', this.rawURL ); @@ -232,7 +236,7 @@ const FrameStore = class { } } { - const result = µb.staticNetFilteringEngine.matchStringReverse( + const result = µb.staticNetFilteringEngine.matchRequestReverse( 'generichide', this.rawURL ); @@ -600,7 +604,7 @@ const PageStore = class { getAllHostnameDetails() { if ( this.hostnameDetailsMap.has(this.tabHostname) === false && - µb.URI.isNetworkURI(this.rawURL) + isNetworkURI(this.rawURL) ) { this.hostnameDetailsMap.set( this.tabHostname, @@ -651,12 +655,12 @@ const PageStore = class { if ( this.journalLastUncommitted !== -1 && this.journalLastUncommitted < this.journalLastCommitted && - this.journalLastUncommittedOrigin === vAPI.hostnameFromURI(url) + this.journalLastUncommittedOrigin === hostnameFromURI(url) ) { this.journalLastCommitted = this.journalLastUncommitted; } } else if ( type === 'uncommitted' ) { - const newOrigin = vAPI.hostnameFromURI(url); + const newOrigin = hostnameFromURI(url); if ( this.journalLastUncommitted === -1 || this.journalLastUncommittedOrigin !== newOrigin @@ -807,7 +811,7 @@ const PageStore = class { // Static filtering has lowest precedence. const snfe = µb.staticNetFilteringEngine; if ( result === 0 || result === 3 ) { - result = snfe.matchString(fctxt); + result = snfe.matchRequest(fctxt); if ( result !== 0 ) { if ( loggerEnabled ) { fctxt.setFilter(snfe.toLogData()); @@ -912,7 +916,10 @@ const PageStore = class { } redirectBlockedRequest(fctxt) { - const directives = µb.staticNetFilteringEngine.redirectRequest(fctxt); + const directives = µb.staticNetFilteringEngine.redirectRequest( + µb.redirectEngine, + fctxt + ); if ( directives === undefined ) { return; } if ( µb.logger.enabled !== true ) { return; } fctxt.pushFilters(directives.map(a => a.logData())); @@ -1062,7 +1069,7 @@ const PageStore = class { } } if ( exceptCname === undefined ) { - const result = µb.staticNetFilteringEngine.matchStringReverse( + const result = µb.staticNetFilteringEngine.matchRequestReverse( 'cname', frameStore instanceof Object ? frameStore.rawURL @@ -1083,7 +1090,7 @@ const PageStore = class { } getBlockedResources(request, response) { - const normalURL = µb.normalizePageURL(this.tabId, request.frameURL); + const normalURL = µb.normalizeTabURL(this.tabId, request.frameURL); const resources = request.resources; const fctxt = µb.filteringContext; fctxt.fromTabId(this.tabId) @@ -1124,8 +1131,3 @@ PageStore.junkyardMax = 10; µb.PageStore = PageStore; /******************************************************************************/ - -// <<<<< -// end of private namespace - -} diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 59dae48c0..a07fddbe6 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -23,9 +23,9 @@ /******************************************************************************/ -µBlock.redirectEngine = (( ) => { +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; -/******************************************************************************/ /******************************************************************************/ // The resources referenced below are found in ./web_accessible_resources/ @@ -356,7 +356,7 @@ RedirectEngine.prototype.resourceContentFromName = function(name, mime) { // Append newlines to raw text to ensure processing of trailing resource. RedirectEngine.prototype.resourcesFromString = function(text) { - const lineIter = new µBlock.LineIterator( + const lineIter = new LineIterator( removeTopCommentBlock(text) + '\n\n' ); const reNonEmptyLine = /\S/; @@ -583,11 +583,10 @@ RedirectEngine.prototype.invalidateResourcesSelfie = function() { µBlock.assets.remove('compiled/redirectEngine/resources'); }; -/******************************************************************************/ /******************************************************************************/ -return new RedirectEngine(); +// Export + +µBlock.redirectEngine = new RedirectEngine(); /******************************************************************************/ - -})(); diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 52d79da8c..8549287a5 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -400,8 +400,8 @@ if ( if ( typeof rawFilter !== 'string' || rawFilter === '' ) { return; } const µb = µBlock; - const writer = new µb.CompiledLineIO.Writer(); - const parser = new vAPI.StaticFilteringParser(); + const writer = new µb.CompiledListWriter(); + const parser = new µb.StaticFilteringParser(); parser.setMaxTokenLength(µb.staticNetFilteringEngine.MAX_TOKEN_LENGTH); parser.analyze(rawFilter); @@ -435,20 +435,20 @@ if ( await initWorker(); const id = messageId++; - const hostname = µBlock.URI.hostnameFromURI(details.url); + const hostname = µBlock.hostnameFromURI(details.url); worker.postMessage({ what: 'fromCosmeticFilter', id: id, - domain: µBlock.URI.domainFromHostname(hostname), + domain: µBlock.domainFromHostname(hostname), hostname: hostname, ignoreGeneric: - µBlock.staticNetFilteringEngine.matchStringReverse( + µBlock.staticNetFilteringEngine.matchRequestReverse( 'generichide', details.url ) === 2, ignoreSpecific: - µBlock.staticNetFilteringEngine.matchStringReverse( + µBlock.staticNetFilteringEngine.matchRequestReverse( 'specifichide', details.url ) === 2, diff --git a/src/js/scriptlet-filtering.js b/src/js/scriptlet-filtering.js index ad41c508f..5da7a0b16 100644 --- a/src/js/scriptlet-filtering.js +++ b/src/js/scriptlet-filtering.js @@ -23,435 +23,447 @@ /******************************************************************************/ -µBlock.scriptletFilteringEngine = (function() { - const µb = µBlock; - const duplicates = new Set(); - const scriptletCache = new µb.MRUCache(32); - const reEscapeScriptArg = /[\\'"]/g; +import { + domainFromHostname, + entityFromDomain, + hostnameFromURI, +} from './uri-utils.js'; - const scriptletDB = new µb.staticExtFilteringEngine.HostnameBasedDB(1); - const sessionScriptletDB = new µb.staticExtFilteringEngine.SessionDB(); - - let acceptedCount = 0; - let discardedCount = 0; - - const api = { - get acceptedCount() { - return acceptedCount; - }, - get discardedCount() { - return discardedCount; - } - }; - - // Purpose of `contentscriptCode` below is too programmatically inject - // content script code which only purpose is to inject scriptlets. This - // essentially does the same as what uBO's declarative content script does, - // except that this allows to inject the scriptlets earlier than it is - // possible through the declarative content script. - // - // Declaratively: - // 1. Browser injects generic content script => - // 2. Content script queries scriptlets => - // 3. Main process sends scriptlets => - // 4. Content script injects scriptlets - // - // Programmatically: - // 1. uBO injects specific scriptlets-aware content script => - // 2. Content script injects scriptlets - // - // However currently this programmatic injection works well only on - // Chromium-based browsers, it does not work properly with Firefox. More - // investigations is needed to find out why this fails with Firefox. - // Consequently, the programmatic-injection code path is taken only with - // Chromium-based browsers. - - const contentscriptCode = (( ) => { - const parts = [ - '(', - function(hostname, scriptlets) { - if ( - document.location === null || - hostname !== document.location.hostname - ) { - return; - } - const injectScriptlets = function(d) { - let script; - try { - script = d.createElement('script'); - script.appendChild(d.createTextNode( - decodeURIComponent(scriptlets)) - ); - (d.head || d.documentElement).appendChild(script); - } catch (ex) { - } - if ( script ) { - if ( script.parentNode ) { - script.parentNode.removeChild(script); - } - script.textContent = ''; - } - }; - injectScriptlets(document); - }.toString(), - ')(', - '"', 'hostname-slot', '", ', - '"', 'scriptlets-slot', '"', - '); void 0;', - ]; - return { - parts: parts, - hostnameSlot: parts.indexOf('hostname-slot'), - scriptletsSlot: parts.indexOf('scriptlets-slot'), - assemble: function(hostname, scriptlets) { - this.parts[this.hostnameSlot] = hostname; - this.parts[this.scriptletsSlot] = - encodeURIComponent(scriptlets); - return this.parts.join(''); - } - }; - })(); - - // TODO: Probably should move this into StaticFilteringParser - // https://github.com/uBlockOrigin/uBlock-issues/issues/1031 - // Normalize scriptlet name to its canonical, unaliased name. - const normalizeRawFilter = function(rawFilter) { - const rawToken = rawFilter.slice(4, -1); - const rawEnd = rawToken.length; - let end = rawToken.indexOf(','); - if ( end === -1 ) { end = rawEnd; } - const token = rawToken.slice(0, end).trim(); - const alias = token.endsWith('.js') ? token.slice(0, -3) : token; - let normalized = µb.redirectEngine.aliases.get(`${alias}.js`); - normalized = normalized === undefined - ? alias - : normalized.slice(0, -3); - let beg = end + 1; - while ( beg < rawEnd ) { - end = rawToken.indexOf(',', beg); - if ( end === -1 ) { end = rawEnd; } - normalized += ', ' + rawToken.slice(beg, end).trim(); - beg = end + 1; - } - return `+js(${normalized})`; - }; - - const lookupScriptlet = function(rawToken, reng, toInject) { - if ( toInject.has(rawToken) ) { return; } - if ( scriptletCache.resetTime < reng.modifyTime ) { - scriptletCache.reset(); - } - let content = scriptletCache.lookup(rawToken); - if ( content === undefined ) { - const pos = rawToken.indexOf(','); - let token, args; - if ( pos === -1 ) { - token = rawToken; - } else { - token = rawToken.slice(0, pos).trim(); - args = rawToken.slice(pos + 1).trim(); - } - // TODO: The alias lookup can be removed once scriptlet resources - // with obsolete name are converted to their new name. - if ( reng.aliases.has(token) ) { - token = reng.aliases.get(token); - } else { - token = `${token}.js`; - } - content = reng.resourceContentFromName( - token, - 'application/javascript' - ); - if ( !content ) { return; } - if ( args ) { - content = patchScriptlet(content, args); - if ( !content ) { return; } - } - content = - 'try {\n' + - content + '\n' + - '} catch ( e ) { }'; - scriptletCache.add(rawToken, content); - } - toInject.set(rawToken, content); - }; - - // Fill-in scriptlet argument placeholders. - const patchScriptlet = function(content, args) { - let s = args; - let len = s.length; - let beg = 0, pos = 0; - let i = 1; - while ( beg < len ) { - pos = s.indexOf(',', pos); - // Escaped comma? If so, skip. - if ( pos > 0 && s.charCodeAt(pos - 1) === 0x5C /* '\\' */ ) { - s = s.slice(0, pos - 1) + s.slice(pos); - len -= 1; - continue; - } - if ( pos === -1 ) { pos = len; } - content = content.replace( - `{{${i}}}`, - s.slice(beg, pos).trim().replace(reEscapeScriptArg, '\\$&') - ); - beg = pos = pos + 1; - i++; - } - return content; - }; - - const logOne = function(isException, token, details) { - µBlock.filteringContext - .duplicate() - .fromTabId(details.tabId) - .setRealm('extended') - .setType('dom') - .setURL(details.url) - .setDocOriginFromURL(details.url) - .setFilter({ - source: 'extended', - raw: (isException ? '#@#' : '##') + `+js(${token})` - }) - .toLogger(); - }; - - api.reset = function() { - scriptletDB.clear(); - duplicates.clear(); - acceptedCount = 0; - discardedCount = 0; - }; - - api.freeze = function() { - duplicates.clear(); - scriptletDB.collectGarbage(); - }; - - api.compile = function(parser, writer) { - writer.select(µb.compiledScriptletSection); - - // Only exception filters are allowed to be global. - const { raw, exception } = parser.result; - const normalized = normalizeRawFilter(raw); - - // Tokenless is meaningful only for exception filters. - if ( normalized === '+js()' && exception === false ) { return; } - - if ( parser.hasOptions() === false ) { - if ( exception ) { - writer.push([ 32, '', 1, normalized ]); - } - return; - } - - // https://github.com/gorhill/uBlock/issues/3375 - // Ignore instances of exception filter with negated hostnames, - // because there is no way to create an exception to an exception. - - for ( const { hn, not, bad } of parser.extOptions() ) { - if ( bad ) { continue; } - let kind = 0; - if ( exception ) { - if ( not ) { continue; } - kind |= 1; - } else if ( not ) { - kind |= 1; - } - writer.push([ 32, hn, kind, normalized ]); - } - }; - - api.compileTemporary = function(parser) { - return { - session: sessionScriptletDB, - selector: parser.result.compiled, - }; - }; - - // 01234567890123456789 - // +js(token[, arg[, ...]]) - // ^ ^ - // 4 -1 - - api.fromCompiledContent = function(reader) { - reader.select(µb.compiledScriptletSection); - - while ( reader.next() ) { - acceptedCount += 1; - const fingerprint = reader.fingerprint(); - if ( duplicates.has(fingerprint) ) { - discardedCount += 1; - continue; - } - duplicates.add(fingerprint); - const args = reader.args(); - if ( args.length < 4 ) { continue; } - scriptletDB.store(args[1], args[2], args[3].slice(4, -1)); - } - }; - - api.getSession = function() { - return sessionScriptletDB; - }; - - const $scriptlets = new Set(); - const $exceptions = new Set(); - const $scriptletToCodeMap = new Map(); - - api.retrieve = function(request) { - if ( scriptletDB.size === 0 ) { return; } - - const reng = µb.redirectEngine; - if ( !reng ) { return; } - - const hostname = request.hostname; - - $scriptlets.clear(); - $exceptions.clear(); - - if ( sessionScriptletDB.isNotEmpty ) { - sessionScriptletDB.retrieve([ null, $exceptions ]); - } - scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]); - const entity = request.entity !== '' - ? `${hostname.slice(0, -request.domain.length)}${request.entity}` - : '*'; - scriptletDB.retrieve(entity, [ $scriptlets, $exceptions ], 1); - if ( $scriptlets.size === 0 ) { return; } - - // https://github.com/gorhill/uBlock/issues/2835 - // Do not inject scriptlets if the site is under an `allow` rule. - if ( - µb.userSettings.advancedUserEnabled && - µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 - ) { - return; - } - - const loggerEnabled = µb.logger.enabled; - - // Wholly disable scriptlet injection? - if ( $exceptions.has('') ) { - if ( loggerEnabled ) { - logOne(true, '', request); - } - return; - } - - $scriptletToCodeMap.clear(); - for ( const rawToken of $scriptlets ) { - lookupScriptlet(rawToken, reng, $scriptletToCodeMap); - } - if ( $scriptletToCodeMap.size === 0 ) { return; } - - // Return an array of scriptlets, and log results if needed. - const out = []; - for ( const [ rawToken, code ] of $scriptletToCodeMap ) { - const isException = $exceptions.has(rawToken); - if ( isException === false ) { - out.push(code); - } - if ( loggerEnabled ) { - logOne(isException, rawToken, request); - } - } - - if ( out.length === 0 ) { return; } - - if ( µb.hiddenSettings.debugScriptlets ) { - out.unshift('debugger;'); - } - - // https://github.com/uBlockOrigin/uBlock-issues/issues/156 - // Provide a private Map() object available for use by all - // scriptlets. - out.unshift( - '(function() {', - '// >>>> start of private namespace', - '' - ); - out.push( - '', - '// <<<< end of private namespace', - '})();' - ); - - return out.join('\n'); - }; - - api.hasScriptlet = function(hostname, exceptionBit, scriptlet) { - return scriptletDB.hasStr(hostname, exceptionBit, scriptlet); - }; - - api.injectNow = function(details) { - if ( typeof details.frameId !== 'number' ) { return; } - const request = { - tabId: details.tabId, - frameId: details.frameId, - url: details.url, - hostname: µb.URI.hostnameFromURI(details.url), - domain: undefined, - entity: undefined - }; - request.domain = µb.URI.domainFromHostname(request.hostname); - request.entity = µb.URI.entityFromDomain(request.domain); - const scriptlets = µb.scriptletFilteringEngine.retrieve(request); - if ( scriptlets === undefined ) { return; } - let code = contentscriptCode.assemble(request.hostname, scriptlets); - if ( µb.hiddenSettings.debugScriptletInjector ) { - code = 'debugger;\n' + code; - } - vAPI.tabs.executeScript(details.tabId, { - code, - frameId: details.frameId, - matchAboutBlank: true, - runAt: 'document_start', - }); - }; - - api.toSelfie = function() { - return scriptletDB.toSelfie(); - }; - - api.fromSelfie = function(selfie) { - scriptletDB.fromSelfie(selfie); - }; - - api.benchmark = async function() { - const requests = await µb.loadBenchmarkDataset(); - if ( Array.isArray(requests) === false || requests.length === 0 ) { - log.print('No requests found to benchmark'); - return; - } - log.print('Benchmarking scriptletFilteringEngine.retrieve()...'); - const details = { - domain: '', - entity: '', - hostname: '', - tabId: 0, - url: '', - }; - let count = 0; - const t0 = self.performance.now(); - for ( let i = 0; i < requests.length; i++ ) { - const request = requests[i]; - if ( request.cpt !== 'main_frame' ) { continue; } - count += 1; - details.url = request.url; - details.hostname = µb.URI.hostnameFromURI(request.url); - details.domain = µb.URI.domainFromHostname(details.hostname); - details.entity = µb.URI.entityFromDomain(details.domain); - void this.retrieve(details); - } - const t1 = self.performance.now(); - const dur = t1 - t0; - log.print(`Evaluated ${count} requests in ${dur.toFixed(0)} ms`); - log.print(`\tAverage: ${(dur / count).toFixed(3)} ms per request`); - }; - - return api; -})(); +import µBlock from './background.js'; + +/******************************************************************************/ + +const µb = µBlock; +const duplicates = new Set(); +const scriptletCache = new µb.MRUCache(32); +const reEscapeScriptArg = /[\\'"]/g; + +const scriptletDB = new µb.staticExtFilteringEngine.HostnameBasedDB(1); +const sessionScriptletDB = new µb.staticExtFilteringEngine.SessionDB(); + +let acceptedCount = 0; +let discardedCount = 0; + +const api = { + get acceptedCount() { + return acceptedCount; + }, + get discardedCount() { + return discardedCount; + } +}; + +// Purpose of `contentscriptCode` below is too programmatically inject +// content script code which only purpose is to inject scriptlets. This +// essentially does the same as what uBO's declarative content script does, +// except that this allows to inject the scriptlets earlier than it is +// possible through the declarative content script. +// +// Declaratively: +// 1. Browser injects generic content script => +// 2. Content script queries scriptlets => +// 3. Main process sends scriptlets => +// 4. Content script injects scriptlets +// +// Programmatically: +// 1. uBO injects specific scriptlets-aware content script => +// 2. Content script injects scriptlets +// +// However currently this programmatic injection works well only on +// Chromium-based browsers, it does not work properly with Firefox. More +// investigations is needed to find out why this fails with Firefox. +// Consequently, the programmatic-injection code path is taken only with +// Chromium-based browsers. + +const contentscriptCode = (( ) => { + const parts = [ + '(', + function(hostname, scriptlets) { + if ( + document.location === null || + hostname !== document.location.hostname + ) { + return; + } + const injectScriptlets = function(d) { + let script; + try { + script = d.createElement('script'); + script.appendChild(d.createTextNode( + decodeURIComponent(scriptlets)) + ); + (d.head || d.documentElement).appendChild(script); + } catch (ex) { + } + if ( script ) { + if ( script.parentNode ) { + script.parentNode.removeChild(script); + } + script.textContent = ''; + } + }; + injectScriptlets(document); + }.toString(), + ')(', + '"', 'hostname-slot', '", ', + '"', 'scriptlets-slot', '"', + '); void 0;', + ]; + return { + parts: parts, + hostnameSlot: parts.indexOf('hostname-slot'), + scriptletsSlot: parts.indexOf('scriptlets-slot'), + assemble: function(hostname, scriptlets) { + this.parts[this.hostnameSlot] = hostname; + this.parts[this.scriptletsSlot] = + encodeURIComponent(scriptlets); + return this.parts.join(''); + } + }; +})(); + +// TODO: Probably should move this into StaticFilteringParser +// https://github.com/uBlockOrigin/uBlock-issues/issues/1031 +// Normalize scriptlet name to its canonical, unaliased name. +const normalizeRawFilter = function(rawFilter) { + const rawToken = rawFilter.slice(4, -1); + const rawEnd = rawToken.length; + let end = rawToken.indexOf(','); + if ( end === -1 ) { end = rawEnd; } + const token = rawToken.slice(0, end).trim(); + const alias = token.endsWith('.js') ? token.slice(0, -3) : token; + let normalized = µb.redirectEngine.aliases.get(`${alias}.js`); + normalized = normalized === undefined + ? alias + : normalized.slice(0, -3); + let beg = end + 1; + while ( beg < rawEnd ) { + end = rawToken.indexOf(',', beg); + if ( end === -1 ) { end = rawEnd; } + normalized += ', ' + rawToken.slice(beg, end).trim(); + beg = end + 1; + } + return `+js(${normalized})`; +}; + +const lookupScriptlet = function(rawToken, reng, toInject) { + if ( toInject.has(rawToken) ) { return; } + if ( scriptletCache.resetTime < reng.modifyTime ) { + scriptletCache.reset(); + } + let content = scriptletCache.lookup(rawToken); + if ( content === undefined ) { + const pos = rawToken.indexOf(','); + let token, args; + if ( pos === -1 ) { + token = rawToken; + } else { + token = rawToken.slice(0, pos).trim(); + args = rawToken.slice(pos + 1).trim(); + } + // TODO: The alias lookup can be removed once scriptlet resources + // with obsolete name are converted to their new name. + if ( reng.aliases.has(token) ) { + token = reng.aliases.get(token); + } else { + token = `${token}.js`; + } + content = reng.resourceContentFromName( + token, + 'application/javascript' + ); + if ( !content ) { return; } + if ( args ) { + content = patchScriptlet(content, args); + if ( !content ) { return; } + } + content = + 'try {\n' + + content + '\n' + + '} catch ( e ) { }'; + scriptletCache.add(rawToken, content); + } + toInject.set(rawToken, content); +}; + +// Fill-in scriptlet argument placeholders. +const patchScriptlet = function(content, args) { + let s = args; + let len = s.length; + let beg = 0, pos = 0; + let i = 1; + while ( beg < len ) { + pos = s.indexOf(',', pos); + // Escaped comma? If so, skip. + if ( pos > 0 && s.charCodeAt(pos - 1) === 0x5C /* '\\' */ ) { + s = s.slice(0, pos - 1) + s.slice(pos); + len -= 1; + continue; + } + if ( pos === -1 ) { pos = len; } + content = content.replace( + `{{${i}}}`, + s.slice(beg, pos).trim().replace(reEscapeScriptArg, '\\$&') + ); + beg = pos = pos + 1; + i++; + } + return content; +}; + +const logOne = function(isException, token, details) { + µBlock.filteringContext + .duplicate() + .fromTabId(details.tabId) + .setRealm('extended') + .setType('dom') + .setURL(details.url) + .setDocOriginFromURL(details.url) + .setFilter({ + source: 'extended', + raw: (isException ? '#@#' : '##') + `+js(${token})` + }) + .toLogger(); +}; + +api.reset = function() { + scriptletDB.clear(); + duplicates.clear(); + acceptedCount = 0; + discardedCount = 0; +}; + +api.freeze = function() { + duplicates.clear(); + scriptletDB.collectGarbage(); +}; + +api.compile = function(parser, writer) { + writer.select(µb.compiledScriptletSection); + + // Only exception filters are allowed to be global. + const { raw, exception } = parser.result; + const normalized = normalizeRawFilter(raw); + + // Tokenless is meaningful only for exception filters. + if ( normalized === '+js()' && exception === false ) { return; } + + if ( parser.hasOptions() === false ) { + if ( exception ) { + writer.push([ 32, '', 1, normalized ]); + } + return; + } + + // https://github.com/gorhill/uBlock/issues/3375 + // Ignore instances of exception filter with negated hostnames, + // because there is no way to create an exception to an exception. + + for ( const { hn, not, bad } of parser.extOptions() ) { + if ( bad ) { continue; } + let kind = 0; + if ( exception ) { + if ( not ) { continue; } + kind |= 1; + } else if ( not ) { + kind |= 1; + } + writer.push([ 32, hn, kind, normalized ]); + } +}; + +api.compileTemporary = function(parser) { + return { + session: sessionScriptletDB, + selector: parser.result.compiled, + }; +}; + +// 01234567890123456789 +// +js(token[, arg[, ...]]) +// ^ ^ +// 4 -1 + +api.fromCompiledContent = function(reader) { + reader.select(µb.compiledScriptletSection); + + while ( reader.next() ) { + acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( duplicates.has(fingerprint) ) { + discardedCount += 1; + continue; + } + duplicates.add(fingerprint); + const args = reader.args(); + if ( args.length < 4 ) { continue; } + scriptletDB.store(args[1], args[2], args[3].slice(4, -1)); + } +}; + +api.getSession = function() { + return sessionScriptletDB; +}; + +const $scriptlets = new Set(); +const $exceptions = new Set(); +const $scriptletToCodeMap = new Map(); + +api.retrieve = function(request) { + if ( scriptletDB.size === 0 ) { return; } + + const reng = µb.redirectEngine; + if ( !reng ) { return; } + + const hostname = request.hostname; + + $scriptlets.clear(); + $exceptions.clear(); + + if ( sessionScriptletDB.isNotEmpty ) { + sessionScriptletDB.retrieve([ null, $exceptions ]); + } + scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]); + const entity = request.entity !== '' + ? `${hostname.slice(0, -request.domain.length)}${request.entity}` + : '*'; + scriptletDB.retrieve(entity, [ $scriptlets, $exceptions ], 1); + if ( $scriptlets.size === 0 ) { return; } + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not inject scriptlets if the site is under an `allow` rule. + if ( + µb.userSettings.advancedUserEnabled && + µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 + ) { + return; + } + + const loggerEnabled = µb.logger.enabled; + + // Wholly disable scriptlet injection? + if ( $exceptions.has('') ) { + if ( loggerEnabled ) { + logOne(true, '', request); + } + return; + } + + $scriptletToCodeMap.clear(); + for ( const rawToken of $scriptlets ) { + lookupScriptlet(rawToken, reng, $scriptletToCodeMap); + } + if ( $scriptletToCodeMap.size === 0 ) { return; } + + // Return an array of scriptlets, and log results if needed. + const out = []; + for ( const [ rawToken, code ] of $scriptletToCodeMap ) { + const isException = $exceptions.has(rawToken); + if ( isException === false ) { + out.push(code); + } + if ( loggerEnabled ) { + logOne(isException, rawToken, request); + } + } + + if ( out.length === 0 ) { return; } + + if ( µb.hiddenSettings.debugScriptlets ) { + out.unshift('debugger;'); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/156 + // Provide a private Map() object available for use by all + // scriptlets. + out.unshift( + '(function() {', + '// >>>> start of private namespace', + '' + ); + out.push( + '', + '// <<<< end of private namespace', + '})();' + ); + + return out.join('\n'); +}; + +api.hasScriptlet = function(hostname, exceptionBit, scriptlet) { + return scriptletDB.hasStr(hostname, exceptionBit, scriptlet); +}; + +api.injectNow = function(details) { + if ( typeof details.frameId !== 'number' ) { return; } + const request = { + tabId: details.tabId, + frameId: details.frameId, + url: details.url, + hostname: hostnameFromURI(details.url), + domain: undefined, + entity: undefined + }; + request.domain = domainFromHostname(request.hostname); + request.entity = entityFromDomain(request.domain); + const scriptlets = µb.scriptletFilteringEngine.retrieve(request); + if ( scriptlets === undefined ) { return; } + let code = contentscriptCode.assemble(request.hostname, scriptlets); + if ( µb.hiddenSettings.debugScriptletInjector ) { + code = 'debugger;\n' + code; + } + vAPI.tabs.executeScript(details.tabId, { + code, + frameId: details.frameId, + matchAboutBlank: true, + runAt: 'document_start', + }); +}; + +api.toSelfie = function() { + return scriptletDB.toSelfie(); +}; + +api.fromSelfie = function(selfie) { + scriptletDB.fromSelfie(selfie); +}; + +api.benchmark = async function() { + const requests = await µb.loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + log.print('No requests found to benchmark'); + return; + } + log.print('Benchmarking scriptletFilteringEngine.retrieve()...'); + const details = { + domain: '', + entity: '', + hostname: '', + tabId: 0, + url: '', + }; + let count = 0; + const t0 = self.performance.now(); + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + if ( request.cpt !== 'main_frame' ) { continue; } + count += 1; + details.url = request.url; + details.hostname = hostnameFromURI(request.url); + details.domain = domainFromHostname(details.hostname); + details.entity = entityFromDomain(details.domain); + void this.retrieve(details); + } + const t1 = self.performance.now(); + const dur = t1 - t0; + log.print(`Evaluated ${count} requests in ${dur.toFixed(0)} ms`); + log.print(`\tAverage: ${(dur / count).toFixed(3)} ms per request`); +}; + +/******************************************************************************/ + +// Export + +µBlock.scriptletFilteringEngine = api; /******************************************************************************/ diff --git a/src/js/start.js b/src/js/start.js index 3d48e903a..45b185435 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -23,6 +23,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + // Load all: executed once. (async ( ) => { @@ -33,7 +37,6 @@ const µb = µBlock; /******************************************************************************/ vAPI.app.onShutdown = function() { - const µb = µBlock; µb.staticFilteringReverseLookup.shutdown(); µb.assets.updateStop(); µb.staticNetFilteringEngine.reset(); @@ -317,7 +320,7 @@ try { } if ( µb.hiddenSettings.disableWebAssembly !== true ) { - µb.staticNetFilteringEngine.enableWASM().then(( ) => { + µb.staticNetFilteringEngine.enableWASM('/js').then(( ) => { log.info(`WASM modules ready ${Date.now()-vAPI.T0} ms after launch`); }); } @@ -445,7 +448,7 @@ if ( browser.runtime.onUpdateAvailable.addListener(details => { const toInt = vAPI.app.intFromVersion; if ( - µBlock.hiddenSettings.extensionUpdateForceReload === true || + µb.hiddenSettings.extensionUpdateForceReload === true || toInt(details.version) <= toInt(vAPI.app.version) ) { vAPI.app.restart(); diff --git a/src/js/static-ext-filtering.js b/src/js/static-ext-filtering.js index b9b9f2fd2..04b86795c 100644 --- a/src/js/static-ext-filtering.js +++ b/src/js/static-ext-filtering.js @@ -21,6 +21,10 @@ 'use strict'; +/******************************************************************************/ + +import µBlock from './background.js'; + /******************************************************************************* All static extended filters are of the form: @@ -48,326 +52,328 @@ **/ -µBlock.staticExtFilteringEngine = (( ) => { - const µb = µBlock; +const µb = µBlock; - //-------------------------------------------------------------------------- - // Public API - //-------------------------------------------------------------------------- +//-------------------------------------------------------------------------- +// Public API +//-------------------------------------------------------------------------- - const api = { - get acceptedCount() { - return µb.cosmeticFilteringEngine.acceptedCount + - µb.scriptletFilteringEngine.acceptedCount + - µb.httpheaderFilteringEngine.acceptedCount + - µb.htmlFilteringEngine.acceptedCount; - }, - get discardedCount() { - return µb.cosmeticFilteringEngine.discardedCount + - µb.scriptletFilteringEngine.discardedCount + - µb.httpheaderFilteringEngine.discardedCount + - µb.htmlFilteringEngine.discardedCount; - }, - }; +const api = { + get acceptedCount() { + return µb.cosmeticFilteringEngine.acceptedCount + + µb.scriptletFilteringEngine.acceptedCount + + µb.httpheaderFilteringEngine.acceptedCount + + µb.htmlFilteringEngine.acceptedCount; + }, + get discardedCount() { + return µb.cosmeticFilteringEngine.discardedCount + + µb.scriptletFilteringEngine.discardedCount + + µb.httpheaderFilteringEngine.discardedCount + + µb.htmlFilteringEngine.discardedCount; + }, +}; - //-------------------------------------------------------------------------- - // Public classes - //-------------------------------------------------------------------------- +//-------------------------------------------------------------------------- +// Public classes +//-------------------------------------------------------------------------- - api.HostnameBasedDB = class { - constructor(nBits, selfie = undefined) { - this.nBits = nBits; - this.timer = undefined; - this.strToIdMap = new Map(); - this.hostnameToSlotIdMap = new Map(); - // Array of integer pairs - this.hostnameSlots = []; - // Array of strings (selectors and pseudo-selectors) - this.strSlots = []; - this.size = 0; - if ( selfie !== undefined ) { - this.fromSelfie(selfie); +api.HostnameBasedDB = class { + constructor(nBits, selfie = undefined) { + this.nBits = nBits; + this.timer = undefined; + this.strToIdMap = new Map(); + this.hostnameToSlotIdMap = new Map(); + // Array of integer pairs + this.hostnameSlots = []; + // Array of strings (selectors and pseudo-selectors) + this.strSlots = []; + this.size = 0; + if ( selfie !== undefined ) { + this.fromSelfie(selfie); + } + } + + store(hn, bits, s) { + this.size += 1; + let iStr = this.strToIdMap.get(s); + if ( iStr === undefined ) { + iStr = this.strSlots.length; + this.strSlots.push(s); + this.strToIdMap.set(s, iStr); + if ( this.timer === undefined ) { + this.collectGarbage(true); } } - - store(hn, bits, s) { - this.size += 1; - let iStr = this.strToIdMap.get(s); - if ( iStr === undefined ) { - iStr = this.strSlots.length; - this.strSlots.push(s); - this.strToIdMap.set(s, iStr); - if ( this.timer === undefined ) { - this.collectGarbage(true); - } - } - const strId = iStr << this.nBits | bits; - let iHn = this.hostnameToSlotIdMap.get(hn); - if ( iHn === undefined ) { - this.hostnameToSlotIdMap.set(hn, this.hostnameSlots.length); - this.hostnameSlots.push(strId, 0); - return; - } - // Add as last item. - while ( this.hostnameSlots[iHn+1] !== 0 ) { - iHn = this.hostnameSlots[iHn+1]; - } - this.hostnameSlots[iHn+1] = this.hostnameSlots.length; + const strId = iStr << this.nBits | bits; + let iHn = this.hostnameToSlotIdMap.get(hn); + if ( iHn === undefined ) { + this.hostnameToSlotIdMap.set(hn, this.hostnameSlots.length); this.hostnameSlots.push(strId, 0); + return; } - - clear() { - this.hostnameToSlotIdMap.clear(); - this.hostnameSlots.length = 0; - this.strSlots.length = 0; - this.strToIdMap.clear(); - this.size = 0; + // Add as last item. + while ( this.hostnameSlots[iHn+1] !== 0 ) { + iHn = this.hostnameSlots[iHn+1]; } + this.hostnameSlots[iHn+1] = this.hostnameSlots.length; + this.hostnameSlots.push(strId, 0); + } - collectGarbage(later = false) { - if ( later === false ) { - if ( this.timer !== undefined ) { - self.cancelIdleCallback(this.timer); - this.timer = undefined; - } - this.strToIdMap.clear(); - return; + clear() { + this.hostnameToSlotIdMap.clear(); + this.hostnameSlots.length = 0; + this.strSlots.length = 0; + this.strToIdMap.clear(); + this.size = 0; + } + + collectGarbage(later = false) { + if ( later === false ) { + if ( this.timer !== undefined ) { + self.cancelIdleCallback(this.timer); + this.timer = undefined; } - if ( this.timer !== undefined ) { return; } - this.timer = self.requestIdleCallback( - ( ) => { - this.timer = undefined; - this.strToIdMap.clear(); - }, - { timeout: 5000 } - ); + this.strToIdMap.clear(); + return; } + if ( this.timer !== undefined ) { return; } + this.timer = self.requestIdleCallback( + ( ) => { + this.timer = undefined; + this.strToIdMap.clear(); + }, + { timeout: 5000 } + ); + } - // modifiers = 1: return only specific items - // modifiers = 2: return only generic items - // - retrieve(hostname, out, modifiers = 0) { - if ( modifiers === 2 ) { + // modifiers = 1: return only specific items + // modifiers = 2: return only generic items + // + retrieve(hostname, out, modifiers = 0) { + if ( modifiers === 2 ) { + hostname = ''; + } + const mask = out.length - 1; // out.length must be power of two + for (;;) { + let iHn = this.hostnameToSlotIdMap.get(hostname); + if ( iHn !== undefined ) { + do { + const strId = this.hostnameSlots[iHn+0]; + out[strId & mask].add( + this.strSlots[strId >>> this.nBits] + ); + iHn = this.hostnameSlots[iHn+1]; + } while ( iHn !== 0 ); + } + if ( hostname === '' ) { break; } + const pos = hostname.indexOf('.'); + if ( pos === -1 ) { + if ( modifiers === 1 ) { break; } + hostname = ''; + } else { + hostname = hostname.slice(pos + 1); + } + } + } + + hasStr(hostname, exceptionBit, value) { + let found = false; + for (;;) { + let iHn = this.hostnameToSlotIdMap.get(hostname); + if ( iHn !== undefined ) { + do { + const strId = this.hostnameSlots[iHn+0]; + const str = this.strSlots[strId >>> this.nBits]; + if ( (strId & exceptionBit) !== 0 ) { + if ( str === value || str === '' ) { return false; } + } + if ( str === value ) { found = true; } + iHn = this.hostnameSlots[iHn+1]; + } while ( iHn !== 0 ); + } + if ( hostname === '' ) { break; } + const pos = hostname.indexOf('.'); + if ( pos !== -1 ) { + hostname = hostname.slice(pos + 1); + } else if ( hostname !== '*' ) { + hostname = '*'; + } else { hostname = ''; } - const mask = out.length - 1; // out.length must be power of two - for (;;) { - let iHn = this.hostnameToSlotIdMap.get(hostname); - if ( iHn !== undefined ) { - do { - const strId = this.hostnameSlots[iHn+0]; - out[strId & mask].add( - this.strSlots[strId >>> this.nBits] - ); - iHn = this.hostnameSlots[iHn+1]; - } while ( iHn !== 0 ); - } - if ( hostname === '' ) { break; } - const pos = hostname.indexOf('.'); - if ( pos === -1 ) { - if ( modifiers === 1 ) { break; } - hostname = ''; - } else { - hostname = hostname.slice(pos + 1); - } + } + return found; + } + + toSelfie() { + return { + hostnameToSlotIdMap: Array.from(this.hostnameToSlotIdMap), + hostnameSlots: this.hostnameSlots, + strSlots: this.strSlots, + size: this.size + }; + } + + fromSelfie(selfie) { + if ( selfie === undefined ) { return; } + this.hostnameToSlotIdMap = new Map(selfie.hostnameToSlotIdMap); + this.hostnameSlots = selfie.hostnameSlots; + this.strSlots = selfie.strSlots; + this.size = selfie.size; + } +}; + +api.SessionDB = class { + constructor() { + this.db = new Map(); + } + compile(s) { + return s; + } + add(bits, s) { + const bucket = this.db.get(bits); + if ( bucket === undefined ) { + this.db.set(bits, new Set([ s ])); + } else { + bucket.add(s); + } + } + remove(bits, s) { + const bucket = this.db.get(bits); + if ( bucket === undefined ) { return; } + bucket.delete(s); + if ( bucket.size !== 0 ) { return; } + this.db.delete(bits); + } + retrieve(out) { + const mask = out.length - 1; + for ( const [ bits, bucket ] of this.db ) { + const i = bits & mask; + if ( out[i] instanceof Object === false ) { continue; } + for ( const s of bucket ) { + out[i].add(s); } } + } + has(bits, s) { + const selectors = this.db.get(bits); + return selectors !== undefined && selectors.has(s); + } + clear() { + this.db.clear(); + } + get isNotEmpty() { + return this.db.size !== 0; + } +}; - hasStr(hostname, exceptionBit, value) { - let found = false; - for (;;) { - let iHn = this.hostnameToSlotIdMap.get(hostname); - if ( iHn !== undefined ) { - do { - const strId = this.hostnameSlots[iHn+0]; - const str = this.strSlots[strId >>> this.nBits]; - if ( (strId & exceptionBit) !== 0 ) { - if ( str === value || str === '' ) { return false; } - } - if ( str === value ) { found = true; } - iHn = this.hostnameSlots[iHn+1]; - } while ( iHn !== 0 ); - } - if ( hostname === '' ) { break; } - const pos = hostname.indexOf('.'); - if ( pos !== -1 ) { - hostname = hostname.slice(pos + 1); - } else if ( hostname !== '*' ) { - hostname = '*'; - } else { - hostname = ''; - } - } - return found; - } +//-------------------------------------------------------------------------- +// Public methods +//-------------------------------------------------------------------------- - toSelfie() { - return { - hostnameToSlotIdMap: Array.from(this.hostnameToSlotIdMap), - hostnameSlots: this.hostnameSlots, - strSlots: this.strSlots, - size: this.size - }; - } +api.reset = function() { + µb.cosmeticFilteringEngine.reset(); + µb.scriptletFilteringEngine.reset(); + µb.httpheaderFilteringEngine.reset(); + µb.htmlFilteringEngine.reset(); +}; - fromSelfie(selfie) { - if ( selfie === undefined ) { return; } - this.hostnameToSlotIdMap = new Map(selfie.hostnameToSlotIdMap); - this.hostnameSlots = selfie.hostnameSlots; - this.strSlots = selfie.strSlots; - this.size = selfie.size; - } - }; +api.freeze = function() { + µb.cosmeticFilteringEngine.freeze(); + µb.scriptletFilteringEngine.freeze(); + µb.httpheaderFilteringEngine.freeze(); + µb.htmlFilteringEngine.freeze(); +}; - api.SessionDB = class { - constructor() { - this.db = new Map(); - } - compile(s) { - return s; - } - add(bits, s) { - const bucket = this.db.get(bits); - if ( bucket === undefined ) { - this.db.set(bits, new Set([ s ])); - } else { - bucket.add(s); - } - } - remove(bits, s) { - const bucket = this.db.get(bits); - if ( bucket === undefined ) { return; } - bucket.delete(s); - if ( bucket.size !== 0 ) { return; } - this.db.delete(bits); - } - retrieve(out) { - const mask = out.length - 1; - for ( const [ bits, bucket ] of this.db ) { - const i = bits & mask; - if ( out[i] instanceof Object === false ) { continue; } - for ( const s of bucket ) { - out[i].add(s); - } - } - } - has(bits, s) { - const selectors = this.db.get(bits); - return selectors !== undefined && selectors.has(s); - } - clear() { - this.db.clear(); - } - get isNotEmpty() { - return this.db.size !== 0; - } - }; +api.compile = function(parser, writer) { + if ( parser.category !== parser.CATStaticExtFilter ) { return false; } - //-------------------------------------------------------------------------- - // Public methods - //-------------------------------------------------------------------------- - - api.reset = function() { - µb.cosmeticFilteringEngine.reset(); - µb.scriptletFilteringEngine.reset(); - µb.httpheaderFilteringEngine.reset(); - µb.htmlFilteringEngine.reset(); - }; - - api.freeze = function() { - µb.cosmeticFilteringEngine.freeze(); - µb.scriptletFilteringEngine.freeze(); - µb.httpheaderFilteringEngine.freeze(); - µb.htmlFilteringEngine.freeze(); - }; - - api.compile = function(parser, writer) { - if ( parser.category !== parser.CATStaticExtFilter ) { return false; } - - if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) { - const who = writer.properties.get('assetKey') || '?'; - µb.logger.writeOne({ - realm: 'message', - type: 'error', - text: `Invalid extended filter in ${who}: ${parser.raw}` - }); - return true; - } - - // Scriptlet injection - if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { - µb.scriptletFilteringEngine.compile(parser, writer); - return true; - } - - // Response header filtering - if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { - µb.httpheaderFilteringEngine.compile(parser, writer); - return true; - } - - // HTML filtering - // TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML - // filtering syntax. - if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { - µb.htmlFilteringEngine.compile(parser, writer); - return true; - } - - // Cosmetic filtering - µb.cosmeticFilteringEngine.compile(parser, writer); - return true; - }; - - api.compileTemporary = function(parser) { - if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { - return µb.scriptletFilteringEngine.compileTemporary(parser); - } - if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { - return µb.httpheaderFilteringEngine.compileTemporary(parser); - } - if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { - return µb.htmlFilteringEngine.compileTemporary(parser); - } - return µb.cosmeticFilteringEngine.compileTemporary(parser); - }; - - api.fromCompiledContent = function(reader, options) { - µb.cosmeticFilteringEngine.fromCompiledContent(reader, options); - µb.scriptletFilteringEngine.fromCompiledContent(reader, options); - µb.httpheaderFilteringEngine.fromCompiledContent(reader, options); - µb.htmlFilteringEngine.fromCompiledContent(reader, options); - }; - - api.toSelfie = function(path) { - return µBlock.assets.put( - `${path}/main`, - JSON.stringify({ - cosmetic: µb.cosmeticFilteringEngine.toSelfie(), - scriptlets: µb.scriptletFilteringEngine.toSelfie(), - httpHeaders: µb.httpheaderFilteringEngine.toSelfie(), - html: µb.htmlFilteringEngine.toSelfie(), - }) - ); - }; - - api.fromSelfie = function(path) { - return µBlock.assets.get(`${path}/main`).then(details => { - let selfie; - try { - selfie = JSON.parse(details.content); - } catch (ex) { - } - if ( selfie instanceof Object === false ) { return false; } - µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); - µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); - µb.httpheaderFilteringEngine.fromSelfie(selfie.httpHeaders); - µb.htmlFilteringEngine.fromSelfie(selfie.html); - return true; + if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) { + const who = writer.properties.get('name') || '?'; + µb.logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid extended filter in ${who}: ${parser.raw}` }); - }; + return true; + } - return api; -})(); + // Scriptlet injection + if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { + µb.scriptletFilteringEngine.compile(parser, writer); + return true; + } + + // Response header filtering + if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { + µb.httpheaderFilteringEngine.compile(parser, writer); + return true; + } + + // HTML filtering + // TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML + // filtering syntax. + if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { + µb.htmlFilteringEngine.compile(parser, writer); + return true; + } + + // Cosmetic filtering + µb.cosmeticFilteringEngine.compile(parser, writer); + return true; +}; + +api.compileTemporary = function(parser) { + if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { + return µb.scriptletFilteringEngine.compileTemporary(parser); + } + if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { + return µb.httpheaderFilteringEngine.compileTemporary(parser); + } + if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { + return µb.htmlFilteringEngine.compileTemporary(parser); + } + return µb.cosmeticFilteringEngine.compileTemporary(parser); +}; + +api.fromCompiledContent = function(reader, options) { + µb.cosmeticFilteringEngine.fromCompiledContent(reader, options); + µb.scriptletFilteringEngine.fromCompiledContent(reader, options); + µb.httpheaderFilteringEngine.fromCompiledContent(reader, options); + µb.htmlFilteringEngine.fromCompiledContent(reader, options); +}; + +api.toSelfie = function(path) { + return µBlock.assets.put( + `${path}/main`, + JSON.stringify({ + cosmetic: µb.cosmeticFilteringEngine.toSelfie(), + scriptlets: µb.scriptletFilteringEngine.toSelfie(), + httpHeaders: µb.httpheaderFilteringEngine.toSelfie(), + html: µb.htmlFilteringEngine.toSelfie(), + }) + ); +}; + +api.fromSelfie = function(path) { + return µBlock.assets.get(`${path}/main`).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( selfie instanceof Object === false ) { return false; } + µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); + µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); + µb.httpheaderFilteringEngine.fromSelfie(selfie.httpHeaders); + µb.htmlFilteringEngine.fromSelfie(selfie.html); + return true; + }); +}; + +/******************************************************************************/ + +// Export + +µBlock.staticExtFilteringEngine = api; /******************************************************************************/ diff --git a/src/js/static-filtering-io.js b/src/js/static-filtering-io.js new file mode 100644 index 000000000..5240c3bc2 --- /dev/null +++ b/src/js/static-filtering-io.js @@ -0,0 +1,147 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://www.reddit.com/r/uBlockOrigin/comments/oq6kt5/ubo_loads_generic_filter_instead_of_specific/ +// Ensure blocks of content are sorted in ascending id order, such that the +// specific cosmetic filters will be found (and thus reported) before the +// generic ones. + +const serialize = JSON.stringify; +const unserialize = JSON.parse; + +const blockStartPrefix = '#block-start-'; // ensure no special regex characters +const blockEndPrefix = '#block-end-'; // ensure no special regex characters + +class CompiledListWriter { + constructor() { + this.blockId = undefined; + this.block = undefined; + this.blocks = new Map(); + this.properties = new Map(); + } + push(args) { + this.block.push(serialize(args)); + } + last() { + if ( Array.isArray(this.block) && this.block.length !== 0 ) { + return this.block[this.block.length - 1]; + } + } + select(blockId) { + if ( blockId === this.blockId ) { return; } + this.blockId = blockId; + this.block = this.blocks.get(blockId); + if ( this.block === undefined ) { + this.blocks.set(blockId, (this.block = [])); + } + return this; + } + toString() { + const result = []; + const sortedBlocks = + Array.from(this.blocks).sort((a, b) => a[0] - b[0]); + for ( const [ id, lines ] of sortedBlocks ) { + if ( lines.length === 0 ) { continue; } + result.push( + blockStartPrefix + id, + lines.join('\n'), + blockEndPrefix + id + ); + } + return result.join('\n'); + } + static serialize(arg) { + return serialize(arg); + } +} + +class CompiledListReader { + constructor(raw, blockId) { + this.block = ''; + this.len = 0; + this.offset = 0; + this.line = ''; + this.blocks = new Map(); + this.properties = new Map(); + const reBlockStart = new RegExp(`^${blockStartPrefix}(\\d+)\\n`, 'gm'); + let match = reBlockStart.exec(raw); + while ( match !== null ) { + let beg = match.index + match[0].length; + let end = raw.indexOf(blockEndPrefix + match[1], beg); + this.blocks.set(parseInt(match[1], 10), raw.slice(beg, end)); + reBlockStart.lastIndex = end; + match = reBlockStart.exec(raw); + } + if ( blockId !== undefined ) { + this.select(blockId); + } + } + next() { + if ( this.offset === this.len ) { + this.line = ''; + return false; + } + let pos = this.block.indexOf('\n', this.offset); + if ( pos !== -1 ) { + this.line = this.block.slice(this.offset, pos); + this.offset = pos + 1; + } else { + this.line = this.block.slice(this.offset); + this.offset = this.len; + } + return true; + } + select(blockId) { + this.block = this.blocks.get(blockId) || ''; + this.len = this.block.length; + this.offset = 0; + return this; + } + fingerprint() { + return this.line; + } + args() { + return unserialize(this.line); + } + static unserialize(arg) { + return unserialize(arg); + } +} + +CompiledListWriter.prototype.NETWORK_SECTION = +CompiledListReader.prototype.NETWORK_SECTION = 100; + +CompiledListWriter.blockStartPrefix = +CompiledListReader.blockStartPrefix = blockStartPrefix; + +CompiledListWriter.blockEndPrefix = +CompiledListReader.blockEndPrefix = blockEndPrefix; + +/******************************************************************************/ + +export { + CompiledListReader, + CompiledListWriter, +}; diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index c4112e92f..a07d0bd73 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -21,6 +21,12 @@ 'use strict'; +/******************************************************************************/ + +import '../lib/regexanalyzer/regex.js'; + +import globals from './globals.js'; + /******************************************************************************* The goal is for the static filtering parser to avoid external @@ -66,9 +72,6 @@ **/ -{ -// >>>>> start of local scope - /******************************************************************************/ const Parser = class { @@ -116,7 +119,7 @@ const Parser = class { // https://github.com/uBlockOrigin/uBlock-issues/issues/1146 // From https://codemirror.net/doc/manual.html#option_specialChars this.reInvalidCharacters = /[\x00-\x1F\x7F-\x9F\xAD\u061C\u200B-\u200F\u2028\u2029\uFEFF\uFFF9-\uFFFC]/; - this.punycoder = new URL(self.location); + this.punycoder = new URL(globals.location); // TODO: mind maxTokenLength this.reGoodRegexToken = /[^\x01%0-9A-Za-z][%0-9A-Za-z]{7,}|[^\x01%0-9A-Za-z][%0-9A-Za-z]{1,6}[^\x01%0-9A-Za-z]/; @@ -1299,7 +1302,11 @@ Parser.prototype.SelectorCompiler = class { [ 'matches-css-before', ':matches-css-before' ], ]); this.reSimpleSelector = /^[#.][A-Za-z_][\w-]*$/; - this.div = document.createElement('div'); + this.div = (( ) => { + if ( typeof document !== 'object' ) { return null; } + if ( document instanceof Object === false ) { return null; } + return document.createElement('div'); + })(); this.rePseudoElement = /:(?::?after|:?before|:-?[a-z][a-z-]*[a-z])$/; this.reProceduralOperator = new RegExp([ '^(?:', @@ -1424,6 +1431,7 @@ Parser.prototype.SelectorCompiler = class { if ( pos !== -1 ) { return this.cssSelectorType(s.slice(0, pos)) === 1 ? 3 : 0; } + if ( this.div === null ) { return 1; } try { this.div.matches(`${s}, ${s}:not(#foo)`); } catch (ex) { @@ -1533,6 +1541,7 @@ Parser.prototype.SelectorCompiler = class { // https://github.com/uBlockOrigin/uBlock-issues/issues/668 compileStyleProperties(s) { if ( /url\(|\\/i.test(s) ) { return; } + if ( this.div === null ) { return s; } this.div.style.cssText = s; if ( this.div.style.cssText === '' ) { return; } this.div.style.cssText = ''; @@ -2874,7 +2883,7 @@ Parser.regexUtils = Parser.prototype.regexUtils = (( ) => { return '\x01'; }; - const Regex = self.Regex; + const Regex = globals.Regex; if ( Regex instanceof Object === false || Regex.Analyzer instanceof Object === false @@ -2914,13 +2923,6 @@ Parser.regexUtils = Parser.prototype.regexUtils = (( ) => { /******************************************************************************/ -if ( typeof vAPI === 'object' && vAPI !== null ) { - vAPI.StaticFilteringParser = Parser; -} else { - self.StaticFilteringParser = Parser; -} +const StaticFilteringParser = Parser; -/******************************************************************************/ - -// <<<<< end of local scope -} +export { StaticFilteringParser }; diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 81d282f5b..f71844ed6 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -23,11 +23,38 @@ /******************************************************************************/ -µBlock.staticNetFilteringEngine = (( ) => { +import globals from './globals.js'; +import { sparseBase64 } from './base64-custom.js'; +import { BidiTrieContainer } from './biditrie.js'; +import { HNTrieContainer } from './hntrie.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; +import { CompiledListReader } from './static-filtering-io.js'; + +import { + domainFromHostname, + hostnameFromNetworkURL, +} from './uri-utils.js'; + + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility +// +// This import would be best done dynamically, but since dynamic imports are +// not supported by older browsers, for now a static import is necessary. +import { FilteringContext } from './filtering-context.js'; /******************************************************************************/ -const µb = µBlock; +// Access to a key-val store is optional and useful only for optimal +// initialization at module load time. Probably could re-arrange code +// to export an init() function with optimization parameters which would +// need to be called by module clients. For now, I want modularizing with +// minimal amount of changes. + +const keyvalStore = typeof vAPI !== 'undefined' + ? vAPI.localStorage + : { getItem() { return null; }, setItem() {} }; + +/******************************************************************************/ // fedcba9876543210 // || | || | @@ -41,7 +68,7 @@ const µb = µBlock; // |+-------------- bit 10: headers-based filters // +--------------- bit 11-15: unused -const CategoryCount = 1 << 0xb; // shift left to first unused bit +const CategoryCount = 1 << 0xb; // shift left to first unused bit const RealmBitsMask = 0b00000000111; const ActionBitsMask = 0b00000000011; @@ -57,7 +84,7 @@ const AnyParty = 0b00000000000; const FirstParty = 0b00000001000; const ThirdParty = 0b00000010000; const AllParties = 0b00000011000; -const Headers = 0b10000000000; +const HEADERS = 0b10000000000; const typeNameToTypeValue = { 'no_type': 0 << TypeBitsOffset, @@ -141,6 +168,8 @@ const typeValueToTypeName = [ const MAX_TOKEN_LENGTH = 7; +const COMPILED_BAD_SECTION = 1; + /******************************************************************************/ // See the following as short-lived registers, used during evaluation. They are @@ -365,14 +394,14 @@ const bidiTrieMatchExtra = function(l, r, ix) { return 0; }; -const bidiTrie = new µb.BidiTrieContainer(bidiTrieMatchExtra); +const bidiTrie = new BidiTrieContainer(bidiTrieMatchExtra); const bidiTriePrime = function() { - bidiTrie.reset(vAPI.localStorage.getItem('SNFE.bidiTrie')); + bidiTrie.reset(keyvalStore.getItem('SNFE.bidiTrie')); }; const bidiTrieOptimize = function(shrink = false) { - vAPI.localStorage.setItem('SNFE.bidiTrie', bidiTrie.optimize(shrink)); + keyvalStore.setItem('SNFE.bidiTrie', bidiTrie.optimize(shrink)); }; /******************************************************************************* @@ -1223,7 +1252,7 @@ const domainOptIterator = new DomainOptIterator(''); const filterOrigin = (( ) => { const FilterOrigin = class { constructor() { - this.trieContainer = new µb.HNTrieContainer(); + this.trieContainer = new HNTrieContainer(); } compile(domainOptList, prepend, units) { @@ -1298,7 +1327,7 @@ const filterOrigin = (( ) => { prime() { this.trieContainer.reset( - vAPI.localStorage.getItem('SNFE.filterOrigin.trieDetails') + keyvalStore.getItem('SNFE.filterOrigin.trieDetails') ); } @@ -1307,7 +1336,7 @@ const filterOrigin = (( ) => { } optimize() { - vAPI.localStorage.setItem( + keyvalStore.setItem( 'SNFE.filterOrigin.trieDetails', this.trieContainer.optimize() ); @@ -1631,7 +1660,7 @@ const FilterModifier = class { } logData(details) { - let opt = vAPI.StaticFilteringParser.netOptionTokenNames.get(this.type); + let opt = StaticFilteringParser.netOptionTokenNames.get(this.type); if ( this.value !== '' ) { opt += `=${this.value}`; } @@ -1933,7 +1962,7 @@ const FilterHostnameDict = class { static prime() { return FilterHostnameDict.trieContainer.reset( - vAPI.localStorage.getItem('SNFE.FilterHostnameDict.trieDetails') + keyvalStore.getItem('SNFE.FilterHostnameDict.trieDetails') ); } @@ -1942,7 +1971,7 @@ const FilterHostnameDict = class { } static optimize() { - vAPI.localStorage.setItem( + keyvalStore.setItem( 'SNFE.FilterHostnameDict.trieDetails', FilterHostnameDict.trieContainer.optimize() ); @@ -1953,7 +1982,7 @@ const FilterHostnameDict = class { } }; -FilterHostnameDict.trieContainer = new µb.HNTrieContainer(); +FilterHostnameDict.trieContainer = new HNTrieContainer(); registerFilterClass(FilterHostnameDict); @@ -2402,7 +2431,7 @@ const FilterOnHeaders = class { match() { if ( this.parsed === undefined ) { this.parsed = - vAPI.StaticFilteringParser.parseHeaderValue(this.headerOpt); + StaticFilteringParser.parseHeaderValue(this.headerOpt); } const { bad, name, not, re, value } = this.parsed; if ( bad ) { return false; } @@ -2556,14 +2585,14 @@ const urlTokenizer = new (class { } toSelfie() { - return µBlock.base64.encode( + return sparseBase64.encode( this.knownTokens.buffer, this.knownTokens.byteLength ); } fromSelfie(selfie) { - return µBlock.base64.decode(selfie, this.knownTokens.buffer); + return sparseBase64.decode(selfie, this.knownTokens.buffer); } // https://github.com/chrisaljoudi/uBlock/issues/1118 @@ -3183,7 +3212,7 @@ const FilterParser = class { // Mind `\b` directives: `/\bads\b/` should result in token being `ads`, // not `bads`. extractTokenFromRegex(pattern) { - pattern = vAPI.StaticFilteringParser.regexUtils.toTokenizableStr(pattern); + pattern = StaticFilteringParser.regexUtils.toTokenizableStr(pattern); this.reToken.lastIndex = 0; let bestToken; let bestBadness = 0x7FFFFFFF; @@ -3278,7 +3307,7 @@ FilterParser.parse = (( ) => { parser = undefined; return; } - ttlTimer = vAPI.setTimeout(ttlProcess, 10007); + ttlTimer = globals.setTimeout(ttlProcess, 10007); }; return p => { @@ -3287,7 +3316,7 @@ FilterParser.parse = (( ) => { } last = Date.now(); if ( ttlTimer === undefined ) { - ttlTimer = vAPI.setTimeout(ttlProcess, 10007); + ttlTimer = globals.setTimeout(ttlProcess, 10007); } return parser.parse(p); }; @@ -3351,7 +3380,7 @@ FilterContainer.prototype.reset = function() { // Cancel potentially pending optimization run. if ( this.optimizeTimerId !== undefined ) { - self.cancelIdleCallback(this.optimizeTimerId); + globals.cancelIdleCallback(this.optimizeTimerId); this.optimizeTimerId = undefined; } @@ -3365,7 +3394,7 @@ FilterContainer.prototype.reset = function() { FilterContainer.prototype.freeze = function() { const filterBucketId = FilterBucket.fid; - const unserialize = µb.CompiledLineIO.unserialize; + const unserialize = CompiledListReader.unserialize; const t0 = Date.now(); @@ -3452,17 +3481,24 @@ FilterContainer.prototype.freeze = function() { // Optimizing is not critical for the static network filtering engine to // work properly, so defer this until later to allow for reduced delay to // readiness when no valid selfie is available. - this.optimizeTimerId = self.requestIdleCallback(( ) => { - this.optimizeTimerId = undefined; - this.optimize(); - }, { timeout: 5000 }); + if ( this.optimizeTimerId === undefined ) { + this.optimizeTimerId = globals.requestIdleCallback(( ) => { + this.optimizeTimerId = undefined; + this.optimize(); + }, { timeout: 5000 }); + } - log.info(`staticNetFilteringEngine.freeze() took ${Date.now()-t0} ms`); + console.info(`staticNetFilteringEngine.freeze() took ${Date.now()-t0} ms`); }; /******************************************************************************/ FilterContainer.prototype.optimize = function() { + if ( this.optimizeTimerId !== undefined ) { + globals.cancelIdleCallback(this.optimizeTimerId); + this.optimizeTimerId = undefined; + } + const t0 = Date.now(); for ( let bits = 0, n = this.categories.length; bits < n; bits++ ) { @@ -3488,12 +3524,19 @@ FilterContainer.prototype.optimize = function() { filterUnits[i] = null; } - log.info(`staticNetFilteringEngine.optimize() took ${Date.now()-t0} ms`); + console.info(`staticNetFilteringEngine.optimize() took ${Date.now()-t0} ms`); }; /******************************************************************************/ -FilterContainer.prototype.toSelfie = function(path) { +FilterContainer.prototype.toSelfie = function(storage, path) { + if ( + storage instanceof Object === false || + storage.put instanceof Function === false + ) { + return Promise.resolve(); + } + const categoriesToSelfie = ( ) => { const selfie = []; for ( let bits = 0, n = this.categories.length; bits < n; bits++ ) { @@ -3508,26 +3551,26 @@ FilterContainer.prototype.toSelfie = function(path) { filterOrigin.optimize(); return Promise.all([ - µb.assets.put( + storage.put( `${path}/FilterHostnameDict.trieContainer`, - FilterHostnameDict.trieContainer.serialize(µb.base64) + FilterHostnameDict.trieContainer.serialize(sparseBase64) ), - µb.assets.put( + storage.put( `${path}/FilterOrigin.trieContainer`, - filterOrigin.trieContainer.serialize(µb.base64) + filterOrigin.trieContainer.serialize(sparseBase64) ), - µb.assets.put( + storage.put( `${path}/bidiTrie`, - bidiTrie.serialize(µb.base64) + bidiTrie.serialize(sparseBase64) ), - µb.assets.put( + storage.put( `${path}/filterSequences`, - µb.base64.encode( + sparseBase64.encode( Uint32Array.from(filterSequences).buffer, filterSequenceWritePtr << 2 ) ), - µb.assets.put( + storage.put( `${path}/main`, JSON.stringify({ processedFilterCount: this.processedFilterCount, @@ -3548,38 +3591,45 @@ FilterContainer.prototype.toSelfie = function(path) { /******************************************************************************/ -FilterContainer.prototype.fromSelfie = function(path) { +FilterContainer.prototype.fromSelfie = function(storage, path) { + if ( + storage instanceof Object === false || + storage.get instanceof Function === false + ) { + return Promise.resolve(); + } + return Promise.all([ - µb.assets.get(`${path}/FilterHostnameDict.trieContainer`).then(details => + storage.get(`${path}/FilterHostnameDict.trieContainer`).then(details => FilterHostnameDict.trieContainer.unserialize( details.content, - µb.base64 + sparseBase64 ) ), - µb.assets.get(`${path}/FilterOrigin.trieContainer`).then(details => + storage.get(`${path}/FilterOrigin.trieContainer`).then(details => filterOrigin.trieContainer.unserialize( details.content, - µb.base64 + sparseBase64 ) ), - µb.assets.get(`${path}/bidiTrie`).then(details => + storage.get(`${path}/bidiTrie`).then(details => bidiTrie.unserialize( details.content, - µb.base64 + sparseBase64 ) ), - µb.assets.get(`${path}/filterSequences`).then(details => { - const size = µb.base64.decodeSize(details.content) >> 2; + storage.get(`${path}/filterSequences`).then(details => { + const size = sparseBase64.decodeSize(details.content) >> 2; if ( size === 0 ) { return false; } filterSequenceBufferResize(size); filterSequenceWritePtr = size; - const buf32 = µb.base64.decode(details.content); + const buf32 = sparseBase64.decode(details.content); for ( let i = 0; i < size; i++ ) { filterSequences[i] = buf32[i]; } return true; }), - µb.assets.get(`${path}/main`).then(details => { + storage.get(`${path}/main`).then(details => { let selfie; try { selfie = JSON.parse(details.content); @@ -3615,7 +3665,7 @@ FilterContainer.prototype.fromSelfie = function(path) { /******************************************************************************/ FilterContainer.prototype.compile = function(parser, writer) { - // ORDER OF TESTS IS IMPORTANT! + this.error = undefined; const parsed = FilterParser.parse(parser); @@ -3624,19 +3674,15 @@ FilterContainer.prototype.compile = function(parser, writer) { // Ignore filters with unsupported options if ( parsed.unsupported ) { - const who = writer.properties.get('assetKey') || '?'; - µb.logger.writeOne({ - realm: 'message', - type: 'error', - text: `Invalid network filter in ${who}: ${parser.raw}` - }); + const who = writer.properties.get('name') || '?'; + this.error = `Invalid network filter in ${who}: ${parser.raw}`; return false; } writer.select( parsed.badFilter - ? µb.compiledNetworkSection + µb.compiledBadSubsection - : µb.compiledNetworkSection + ? writer.NETWORK_SECTION + COMPILED_BAD_SECTION + : writer.NETWORK_SECTION ); // Reminder: @@ -3747,7 +3793,7 @@ FilterContainer.prototype.compileParsed = function(parsed, writer) { // Header if ( parsed.headerOpt !== undefined ) { units.push(FilterOnHeaders.compile(parsed)); - parsed.action |= Headers; + parsed.action |= HEADERS; } // Modifier @@ -3805,8 +3851,8 @@ FilterContainer.prototype.compileToAtomicFilter = function( /******************************************************************************/ -FilterContainer.prototype.fromCompiledContent = function(reader) { - reader.select(µb.compiledNetworkSection); +FilterContainer.prototype.fromCompiled = function(reader) { + reader.select(reader.NETWORK_SECTION); while ( reader.next() ) { this.acceptedCount += 1; if ( this.goodFilters.has(reader.line) ) { @@ -3816,7 +3862,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader) { } } - reader.select(µb.compiledNetworkSection + µb.compiledBadSubsection); + reader.select(reader.NETWORK_SECTION + COMPILED_BAD_SECTION); while ( reader.next() ) { this.badFilters.add(reader.line); } @@ -3863,7 +3909,7 @@ FilterContainer.prototype.matchAndFetchModifiers = function( const results = []; const env = { - modifier: vAPI.StaticFilteringParser.netOptionTokenIds.get(modifierType) || 0, + modifier: StaticFilteringParser.netOptionTokenIds.get(modifierType) || 0, bits: 0, th: 0, iunit: 0, @@ -4118,7 +4164,7 @@ FilterContainer.prototype.realmMatchString = function( // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ // Add support for `specifichide`. -FilterContainer.prototype.matchStringReverse = function(type, url) { +FilterContainer.prototype.matchRequestReverse = function(type, url) { const typeBits = typeNameToTypeValue[type] | 0x80000000; // Prime tokenizer: we get a normalized URL in return. @@ -4127,8 +4173,8 @@ FilterContainer.prototype.matchStringReverse = function(type, url) { this.$filterUnit = 0; // These registers will be used by various filters - $docHostname = $requestHostname = vAPI.hostnameFromNetworkURL(url); - $docDomain = vAPI.domainFromHostname($docHostname); + $docHostname = $requestHostname = hostnameFromNetworkURL(url); + $docDomain = domainFromHostname($docHostname); $docEntity.reset(); // Exception filters @@ -4165,7 +4211,7 @@ FilterContainer.prototype.matchStringReverse = function(type, url) { * * @returns {integer} 0=no match, 1=block, 2=allow (exeption) */ -FilterContainer.prototype.matchString = function(fctxt, modifiers = 0) { +FilterContainer.prototype.matchRequest = function(fctxt, modifiers = 0) { let typeValue = typeNameToTypeValue[fctxt.type]; if ( modifiers === 0 ) { if ( typeValue === undefined ) { @@ -4227,10 +4273,10 @@ FilterContainer.prototype.matchHeaders = function(fctxt, headers) { $httpHeaders.init(headers); let r = 0; - if ( this.realmMatchString(Headers | BlockImportant, typeValue, partyBits) ) { + if ( this.realmMatchString(HEADERS | BlockImportant, typeValue, partyBits) ) { r = 1; - } else if ( this.realmMatchString(Headers | BlockAction, typeValue, partyBits) ) { - r = this.realmMatchString(Headers | AllowAction, typeValue, partyBits) + } else if ( this.realmMatchString(HEADERS | BlockAction, typeValue, partyBits) ) { + r = this.realmMatchString(HEADERS | AllowAction, typeValue, partyBits) ? 2 : 1; } @@ -4242,21 +4288,23 @@ FilterContainer.prototype.matchHeaders = function(fctxt, headers) { /******************************************************************************/ -FilterContainer.prototype.redirectRequest = function(fctxt) { +FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) { const directives = this.matchAndFetchModifiers(fctxt, 'redirect-rule'); // No directive is the most common occurrence. if ( directives === undefined ) { return; } const highest = directives.length - 1; // More than a single directive means more work. if ( highest !== 0 ) { - directives.sort(FilterContainer.compareRedirectRequests); + directives.sort( + FilterContainer.compareRedirectRequests.bind(this, redirectEngine) + ); } // Redirect to highest-ranked directive const directive = directives[highest]; if ( (directive.bits & AllowAction) === 0 ) { const { token } = FilterContainer.parseRedirectRequestValue(directive.modifier); - fctxt.redirectURL = µb.redirectEngine.tokenToURL(fctxt, token); + fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token); if ( fctxt.redirectURL === undefined ) { return; } } return directives; @@ -4265,18 +4313,18 @@ FilterContainer.prototype.redirectRequest = function(fctxt) { FilterContainer.parseRedirectRequestValue = function(modifier) { if ( modifier.cache === undefined ) { modifier.cache = - vAPI.StaticFilteringParser.parseRedirectValue(modifier.value); + StaticFilteringParser.parseRedirectValue(modifier.value); } return modifier.cache; }; -FilterContainer.compareRedirectRequests = function(a, b) { +FilterContainer.compareRedirectRequests = function(redirectEngine, a, b) { const { token: atok, priority: aint, bits: abits } = FilterContainer.parseRedirectRequestValue(a.modifier); - if ( µb.redirectEngine.hasToken(atok) === false ) { return -1; } + if ( redirectEngine.hasToken(atok) === false ) { return -1; } const { token: btok, priority: bint, bits: bbits } = FilterContainer.parseRedirectRequestValue(b.modifier); - if ( µb.redirectEngine.hasToken(btok) === false ) { return 1; } + if ( redirectEngine.hasToken(btok) === false ) { return 1; } if ( abits !== bbits ) { if ( (abits & Important) !== 0 ) { return 1; } if ( (bbits & Important) !== 0 ) { return -1; } @@ -4299,7 +4347,9 @@ FilterContainer.prototype.filterQuery = function(fctxt) { if ( qpos === -1 ) { return; } let hpos = url.indexOf('#', qpos + 1); if ( hpos === -1 ) { hpos = url.length; } - const params = new Map(new self.URLSearchParams(url.slice(qpos + 1, hpos))); + const params = new Map( + new globals.URLSearchParams(url.slice(qpos + 1, hpos)) + ); const inParamCount = params.size; const out = []; for ( const directive of directives ) { @@ -4363,7 +4413,7 @@ FilterContainer.prototype.filterQuery = function(fctxt) { FilterContainer.prototype.parseQueryPruneValue = function(modifier) { if ( modifier.cache === undefined ) { modifier.cache = - vAPI.StaticFilteringParser.parseQueryPruneValue(modifier.value); + StaticFilteringParser.parseQueryPruneValue(modifier.value); } return modifier.cache; }; @@ -4406,11 +4456,11 @@ FilterContainer.prototype.getFilterCount = function() { /******************************************************************************/ -FilterContainer.prototype.enableWASM = function() { +FilterContainer.prototype.enableWASM = function(modulePath) { return Promise.all([ - bidiTrie.enableWASM(), - filterOrigin.trieContainer.enableWASM(), - FilterHostnameDict.trieContainer.enableWASM(), + bidiTrie.enableWASM(modulePath), + filterOrigin.trieContainer.enableWASM(modulePath), + FilterHostnameDict.trieContainer.enableWASM(modulePath), ]); }; @@ -4418,8 +4468,8 @@ FilterContainer.prototype.enableWASM = function() { // action: 1=test, 2=record -FilterContainer.prototype.benchmark = async function(action, target) { - const requests = await µb.loadBenchmarkDataset(); +FilterContainer.prototype.benchmark = async function(requests, options = {}) { + const { action, target, redirectEngine } = options; if ( Array.isArray(requests) === false || requests.length === 0 ) { const text = 'No dataset found to benchmark'; @@ -4429,15 +4479,16 @@ FilterContainer.prototype.benchmark = async function(action, target) { const print = log.print; - print(`Benchmarking staticNetFilteringEngine.matchString()...`); - const fctxt = µb.filteringContext.duplicate(); + print(`Benchmarking staticNetFilteringEngine.matchRequest()...`); + + const fctxt = new FilteringContext(); if ( typeof target === 'number' ) { const request = requests[target]; fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); print(`Result=${r}:`); print(`\ttype=${fctxt.type}`); print(`\turl=${fctxt.url}`); @@ -4452,7 +4503,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { if ( action === 1 ) { try { expected = JSON.parse( - vAPI.localStorage.getItem('FilterContainer.benchmark.results') + keyvalStore.getItem('FilterContainer.benchmark.results') ); } catch(ex) { } @@ -4461,7 +4512,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { recorded = []; } - const t0 = self.performance.now(); + const t0 = globals.performance.now(); let matchCount = 0; for ( let i = 0; i < requests.length; i++ ) { const request = requests[i]; @@ -4469,7 +4520,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); this.redirectURL = undefined; - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); matchCount += 1; if ( recorded !== undefined ) { recorded.push(r); } if ( expected !== undefined && r !== expected[i] ) { @@ -4487,15 +4538,15 @@ FilterContainer.prototype.benchmark = async function(action, target) { this.matchAndFetchModifiers(fctxt, 'csp'); } this.matchHeaders(fctxt, []); - } else { - this.redirectRequest(fctxt); + } else if ( redirectEngine !== undefined ) { + this.redirectRequest(redirectEngine, fctxt); } } - const t1 = self.performance.now(); + const t1 = globals.performance.now(); const dur = t1 - t0; if ( recorded !== undefined ) { - vAPI.localStorage.setItem( + keyvalStore.setItem( 'FilterContainer.benchmark.results', JSON.stringify(recorded) ); @@ -4519,12 +4570,12 @@ FilterContainer.prototype.benchmark = async function(action, target) { /******************************************************************************/ -FilterContainer.prototype.test = function(docURL, type, url) { - const fctxt = µb.filteringContext.duplicate(); +FilterContainer.prototype.test = async function(docURL, type, url) { + const fctxt = new FilteringContext(); fctxt.setDocOriginFromURL(docURL); fctxt.setType(type); fctxt.setURL(url); - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); console.log(`${r}`); if ( r !== 0 ) { console.log(this.toLogData()); @@ -4687,17 +4738,15 @@ FilterContainer.prototype.filterClassHistogram = function() { /******************************************************************************/ -FilterContainer.prototype.tokenHistograms = async function() { - const requests = await µb.loadBenchmarkDataset(); - +FilterContainer.prototype.tokenHistograms = async function(requests) { if ( Array.isArray(requests) === false || requests.length === 0 ) { console.info('No requests found to benchmark'); return; } console.info(`Computing token histograms...`); - const fctxt = µb.filteringContext.duplicate(); + const fctxt = new FilteringContext(); const missTokenMap = new Map(); const hitTokenMap = new Map(); const reTokens = /[0-9a-z%]{2,}/g; @@ -4707,7 +4756,7 @@ FilterContainer.prototype.tokenHistograms = async function() { fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); for ( let [ keyword ] of request.url.toLowerCase().matchAll(reTokens) ) { const token = keyword; if ( r === 0 ) { @@ -4729,8 +4778,8 @@ FilterContainer.prototype.tokenHistograms = async function() { /******************************************************************************/ -return new FilterContainer(); +const staticNetFilteringEngine = new FilterContainer(); /******************************************************************************/ -})(); +export { staticNetFilteringEngine }; diff --git a/src/js/storage.js b/src/js/storage.js index e0fb1668c..5ca65a30d 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -19,12 +19,27 @@ Home: https://github.com/gorhill/uBlock */ -/* global punycode, publicSuffixList */ - 'use strict'; /******************************************************************************/ +import '../lib/publicsuffixlist/publicsuffixlist.js'; +import '../lib/punycode.js'; + +import globals from './globals.js'; +import { hostnameFromURI } from './uri-utils.js'; +import { sparseBase64 } from './base64-custom.js'; +import { LineIterator } from './text-iterators.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; +import µBlock from './background.js'; + +import { + CompiledListReader, + CompiledListWriter, +} from './static-filtering-io.js'; + +/******************************************************************************/ + µBlock.getBytesInUse = async function() { const promises = []; let bytesInUse; @@ -242,7 +257,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µBlock.hiddenSettingsFromString = function(raw) { const out = Object.assign({}, this.hiddenSettingsDefault); - const lineIter = new this.LineIterator(raw); + const lineIter = new LineIterator(raw); while ( lineIter.eot() === false ) { const line = lineIter.next(); const matches = /^\s*(\S+)\s+(.+)$/.exec(line); @@ -561,7 +576,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { // https://github.com/gorhill/uBlock/issues/1786 if ( details.docURL === undefined ) { return; } this.cosmeticFilteringEngine.removeFromSelectorCache( - vAPI.hostnameFromURI(details.docURL) + hostnameFromURI(details.docURL) ); }; @@ -929,12 +944,12 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ µBlock.compileFilters = function(rawText, details = {}) { - const writer = new this.CompiledLineIO.Writer(); + const writer = new CompiledListWriter(); // Populate the writer with information potentially useful to the // client compilers. if ( details.assetKey ) { - writer.properties.set('assetKey', details.assetKey); + writer.properties.set('name', details.assetKey); } const expertMode = details.assetKey !== this.userFiltersPath || @@ -944,8 +959,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { // https://adblockplus.org/en/filters const staticNetFilteringEngine = this.staticNetFilteringEngine; const staticExtFilteringEngine = this.staticExtFilteringEngine; - const lineIter = new this.LineIterator(this.preparseDirectives.prune(rawText)); - const parser = new vAPI.StaticFilteringParser({ expertMode }); + const lineIter = new LineIterator(this.preparseDirectives.prune(rawText)); + const parser = new StaticFilteringParser({ expertMode }); parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); @@ -973,7 +988,14 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { if ( parser.patternHasUnicode() && parser.toASCII() === false ) { continue; } - staticNetFilteringEngine.compile(parser, writer); + if ( staticNetFilteringEngine.compile(parser, writer) ) { continue; } + if ( staticNetFilteringEngine.error !== undefined ) { + this.logger.writeOne({ + realm: 'message', + type: 'error', + text: staticNetFilteringEngine.error + }); + } } // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 @@ -993,8 +1015,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µBlock.applyCompiledFilters = function(rawText, firstparty) { if ( rawText === '' ) { return; } - const reader = new this.CompiledLineIO.Reader(rawText); - this.staticNetFilteringEngine.fromCompiledContent(reader); + const reader = new CompiledListReader(rawText); + this.staticNetFilteringEngine.fromCompiled(reader); this.staticExtFilteringEngine.fromCompiledContent(reader, { skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters, skipCosmetic: !firstparty && !this.userSettings.parseAllABPHideFilters @@ -1162,13 +1184,14 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ µBlock.loadPublicSuffixList = async function() { + const psl = globals.publicSuffixList; if ( this.hiddenSettings.disableWebAssembly !== true ) { - publicSuffixList.enableWASM(); + psl.enableWASM('/lib/publicsuffixlist'); } try { const result = await this.assets.get(`compiled/${this.pslAssetKey}`); - if ( publicSuffixList.fromSelfie(result.content, this.base64) ) { + if ( psl.fromSelfie(result.content, sparseBase64) ) { return; } } catch (ex) { @@ -1182,11 +1205,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { }; µBlock.compilePublicSuffixList = function(content) { - publicSuffixList.parse(content, punycode.toASCII); - this.assets.put( - 'compiled/' + this.pslAssetKey, - publicSuffixList.toSelfie(µBlock.base64) - ); + const psl = globals.publicSuffixList; + psl.parse(content, globals.punycode.toASCII); + this.assets.put(`compiled/${this.pslAssetKey}`, psl.toSelfie(sparseBase64)); }; /******************************************************************************/ @@ -1218,6 +1239,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { 'selfie/staticExtFilteringEngine' ), µb.staticNetFilteringEngine.toSelfie( + µb.assets, 'selfie/staticNetFilteringEngine' ), ]); @@ -1261,6 +1283,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { 'selfie/staticExtFilteringEngine' ), µb.staticNetFilteringEngine.fromSelfie( + µb.assets, 'selfie/staticNetFilteringEngine' ), ]); diff --git a/src/js/tab.js b/src/js/tab.js index b63b72863..046df4ddf 100644 --- a/src/js/tab.js +++ b/src/js/tab.js @@ -21,6 +21,17 @@ 'use strict'; +/******************************************************************************/ + +import { + domainFromHostname, + hostnameFromURI, + isNetworkURI, + originFromURI, +} from './uri-utils.js'; + +import µBlock from './background.js'; + /******************************************************************************/ /******************************************************************************/ @@ -33,26 +44,30 @@ // hostname. This way, for a specific scheme you can create scope with // rules which will apply only to that scheme. -µBlock.normalizePageURL = function(tabId, pageURL) { - if ( tabId < 0 ) { - return 'http://behind-the-scene/'; - } - const uri = this.URI.set(pageURL); - const scheme = uri.scheme; - if ( scheme === 'https' || scheme === 'http' ) { - return uri.normalizedURI(); - } +µBlock.normalizeTabURL = (( ) => { + const tabURLNormalizer = new URL('about:blank'); - let fakeHostname = scheme + '-scheme'; + return (tabId, tabURL) => { + if ( tabId < 0 ) { + return 'http://behind-the-scene/'; + } + tabURLNormalizer.href = tabURL; + const protocol = tabURLNormalizer.protocol.slice(0, -1); + if ( protocol === 'https' || protocol === 'http' ) { + return tabURLNormalizer.href; + } - if ( uri.hostname !== '' ) { - fakeHostname = uri.hostname + '.' + fakeHostname; - } else if ( scheme === 'about' && uri.path !== '' ) { - fakeHostname = uri.path + '.' + fakeHostname; - } + let fakeHostname = protocol + '-scheme'; - return `http://${fakeHostname}/`; -}; + if ( tabURLNormalizer.hostname !== '' ) { + fakeHostname = tabURLNormalizer.hostname + '.' + fakeHostname; + } else if ( protocol === 'about' && protocol.pathname !== '' ) { + fakeHostname = tabURLNormalizer.pathname + '.' + fakeHostname; + } + + return `http://${fakeHostname}/`; + }; +})(); /******************************************************************************/ @@ -114,7 +129,7 @@ // Don't block if uBO is turned off in popup's context if ( µb.getNetFilteringSwitch(targetURL) === false || - µb.getNetFilteringSwitch(µb.normalizePageURL(0, targetURL)) === false + µb.getNetFilteringSwitch(µb.normalizeTabURL(0, targetURL)) === false ) { return 0; } @@ -192,7 +207,7 @@ } fctxt.type = popupType; - const result = µb.staticNetFilteringEngine.matchString(fctxt, 0b0001); + const result = µb.staticNetFilteringEngine.matchRequest(fctxt, 0b0001); if ( result !== 0 ) { fctxt.filter = µb.staticNetFilteringEngine.toLogData(); return result; @@ -257,7 +272,7 @@ // For now, a "broad" filter is one which does not touch any part of // the hostname part of the opener URL. let popunderURL = rootOpenerURL, - popunderHostname = µb.URI.hostnameFromURI(popunderURL); + popunderHostname = hostnameFromURI(popunderURL); if ( popunderHostname === '' ) { return 0; } result = mapPopunderResult( @@ -270,7 +285,7 @@ // https://github.com/gorhill/uBlock/issues/1598 // Try to find a match against origin part of the opener URL. - popunderURL = µb.URI.originFromURI(popunderURL); + popunderURL = originFromURI(popunderURL); if ( popunderURL === '' ) { return 0; } return mapPopunderResult( @@ -305,7 +320,7 @@ // https://github.com/gorhill/uBlock/issues/1538 if ( µb.getNetFilteringSwitch( - µb.normalizePageURL(openerTabId, rootOpenerURL) + µb.normalizeTabURL(openerTabId, rootOpenerURL) ) === false ) { return; @@ -662,11 +677,11 @@ housekeep itself. } const stackEntry = this.stack[this.stack.length - 1]; this.rawURL = stackEntry.url; - this.normalURL = µb.normalizePageURL(this.tabId, this.rawURL); - this.origin = µb.URI.originFromURI(this.normalURL); - this.rootHostname = µb.URI.hostnameFromURI(this.origin); + this.normalURL = µb.normalizeTabURL(this.tabId, this.rawURL); + this.origin = originFromURI(this.normalURL); + this.rootHostname = hostnameFromURI(this.origin); this.rootDomain = - µb.URI.domainFromHostname(this.rootHostname) || + domainFromHostname(this.rootHostname) || this.rootHostname; }; @@ -794,10 +809,10 @@ housekeep itself. const entry = new TabContext(vAPI.noTabId); entry.stack.push(new StackEntry('', true)); entry.rawURL = ''; - entry.normalURL = µb.normalizePageURL(entry.tabId); - entry.origin = µb.URI.originFromURI(entry.normalURL); - entry.rootHostname = µb.URI.hostnameFromURI(entry.origin); - entry.rootDomain = µb.URI.domainFromHostname(entry.rootHostname); + entry.normalURL = µb.normalizeTabURL(entry.tabId); + entry.origin = originFromURI(entry.normalURL); + entry.rootHostname = hostnameFromURI(entry.origin); + entry.rootDomain = domainFromHostname(entry.rootHostname); } // Context object, typically to be used to feed filtering engines. @@ -894,7 +909,7 @@ vAPI.Tabs = class extends vAPI.Tabs { pageStore.setFrameURL(details); if ( µb.canInjectScriptletsNow && - µb.URI.isNetworkURI(url) && + isNetworkURI(url) && pageStore.getNetFilteringSwitch() ) { µb.scriptletFilteringEngine.injectNow(details); diff --git a/src/js/text-encode.js b/src/js/text-encode.js index 614db6481..3eb6dc166 100644 --- a/src/js/text-encode.js +++ b/src/js/text-encode.js @@ -23,6 +23,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + µBlock.textEncode = (function() { if ( µBlock.canFilterResponseData !== true ) { return; } diff --git a/src/js/text-iterators.js b/src/js/text-iterators.js new file mode 100644 index 000000000..e2986ddae --- /dev/null +++ b/src/js/text-iterators.js @@ -0,0 +1,92 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +class LineIterator { + constructor(text, offset) { + this.text = text; + this.textLen = this.text.length; + this.offset = offset || 0; + } + next(offset) { + if ( offset !== undefined ) { + this.offset += offset; + } + let lineEnd = this.text.indexOf('\n', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.text.indexOf('\r', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.textLen; + } + } + const line = this.text.slice(this.offset, lineEnd); + this.offset = lineEnd + 1; + return line; + } + peek(n) { + const offset = this.offset; + return this.text.slice(offset, offset + n); + } + charCodeAt(offset) { + return this.text.charCodeAt(this.offset + offset); + } + eot() { + return this.offset >= this.textLen; + } +} + +/******************************************************************************/ + +// The field iterator is less CPU-intensive than when using native +// String.split(). + +class FieldIterator { + constructor(sep) { + this.text = ''; + this.sep = sep; + this.sepLen = sep.length; + this.offset = 0; + } + first(text) { + this.text = text; + this.offset = 0; + return this.next(); + } + next() { + let end = this.text.indexOf(this.sep, this.offset); + if ( end === -1 ) { + end = this.text.length; + } + const field = this.text.slice(this.offset, end); + this.offset = end + this.sepLen; + return field; + } + remainder() { + return this.text.slice(this.offset); + } +} + +/******************************************************************************/ + +export { LineIterator, FieldIterator }; diff --git a/src/js/traffic.js b/src/js/traffic.js index 0f383aa0d..d76805ca3 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -23,9 +23,12 @@ /******************************************************************************/ -// Start isolation from global scope +import { + entityFromDomain, + isNetworkURI, +} from './uri-utils.js'; -µBlock.webRequest = (( ) => { +import µBlock from './background.js'; /******************************************************************************/ @@ -37,20 +40,11 @@ let dontCacheResponseHeaders = vAPI.webextFlavor.soup.has('firefox'); -// https://github.com/gorhill/uMatrix/issues/967#issuecomment-373002011 -// This can be removed once Firefox 60 ESR is released. -let cantMergeCSPHeaders = - vAPI.webextFlavor.soup.has('firefox') && vAPI.webextFlavor.major < 59; - - // The real actual webextFlavor value may not be set in stone, so listen // for possible future changes. window.addEventListener('webextFlavor', function() { dontCacheResponseHeaders = vAPI.webextFlavor.soup.has('firefox'); - cantMergeCSPHeaders = - vAPI.webextFlavor.soup.has('firefox') && - vAPI.webextFlavor.major < 59; }, { once: true }); // https://github.com/uBlockOrigin/uBlock-issues/issues/1553 @@ -269,7 +263,7 @@ const shouldStrictBlock = function(fctxt, loggerEnabled) { const snfe = µb.staticNetFilteringEngine; // Explicit filtering: `document` option - const rs = snfe.matchString(fctxt, 0b0011); + const rs = snfe.matchRequest(fctxt, 0b0011); const is = rs === 1 && snfe.isBlockImportant(); let lds; if ( rs !== 0 || loggerEnabled ) { @@ -291,7 +285,7 @@ const shouldStrictBlock = function(fctxt, loggerEnabled) { // Implicit filtering: no `document` option fctxt.type = 'no_type'; - let rg = snfe.matchString(fctxt, 0b0011); + let rg = snfe.matchRequest(fctxt, 0b0011); fctxt.type = 'main_frame'; const ig = rg === 1 && snfe.isBlockImportant(); let ldg; @@ -382,7 +376,7 @@ const onBeforeBehindTheSceneRequest = function(fctxt) { if ( fctxt.tabOrigin.endsWith('-scheme') === false && - µb.URI.isNetworkURI(fctxt.tabOrigin) || + isNetworkURI(fctxt.tabOrigin) || µb.userSettings.advancedUserEnabled || fctxt.itype === fctxt.CSP_REPORT ) { @@ -836,7 +830,7 @@ const filterDocument = (( ) => { url: fctxt.url, hostname: hostname, domain: domain, - entity: µb.URI.entityFromDomain(domain), + entity: entityFromDomain(domain), selectors: undefined, buffer: null, mime: 'text/html', @@ -997,17 +991,6 @@ const injectCSP = function(fctxt, pageStore, responseHeaders) { // Firefox 58/webext and less can't merge CSP headers, so we will merge // them here. - if ( cantMergeCSPHeaders ) { - const i = headerIndexFromName( - 'content-security-policy', - responseHeaders - ); - if ( i !== -1 ) { - cspSubsets.unshift(responseHeaders[i].value.trim()); - responseHeaders.splice(i, 1); - } - } - responseHeaders.push({ name: 'Content-Security-Policy', value: cspSubsets.join(', ') @@ -1139,7 +1122,9 @@ const strictBlockBypasser = { /******************************************************************************/ -return { +// Export + +µBlock.webRequest = { start: (( ) => { vAPI.net = new vAPI.Net(); vAPI.net.suspend(); @@ -1162,7 +1147,3 @@ return { }; /******************************************************************************/ - -})(); - -/******************************************************************************/ diff --git a/src/js/ublock.js b/src/js/ublock.js index 4284d15b4..bef20f8a3 100644 --- a/src/js/ublock.js +++ b/src/js/ublock.js @@ -21,13 +21,13 @@ 'use strict'; +/******************************************************************************/ + +import { hostnameFromURI } from './uri-utils.js'; +import µBlock from './background.js'; + /******************************************************************************/ /******************************************************************************/ - -{ - -// ***************************************************************************** -// start of local namespace // https://github.com/chrisaljoudi/uBlock/issues/405 // Be more flexible with whitelist syntax @@ -91,7 +91,7 @@ const matchBucket = function(url, hostname, bucket, start) { /******************************************************************************/ µBlock.getNetFilteringSwitch = function(url) { - const hostname = this.URI.hostnameFromURI(url); + const hostname = hostnameFromURI(url); let key = hostname; for (;;) { if ( matchBucket(url, hostname, this.netWhitelist.get(key)) !== -1 ) { @@ -121,7 +121,7 @@ const matchBucket = function(url, hostname, bucket, start) { const netWhitelist = this.netWhitelist; const pos = url.indexOf('#'); let targetURL = pos !== -1 ? url.slice(0, pos) : url; - const targetHostname = this.URI.hostnameFromURI(targetURL); + const targetHostname = hostnameFromURI(targetURL); let key = targetHostname; let directive = scope === 'page' ? targetURL : targetHostname; @@ -281,12 +281,6 @@ const matchBucket = function(url, hostname, bucket, start) { µBlock.reWhitelistBadHostname = /[^a-z0-9.\-_\[\]:]/; µBlock.reWhitelistHostnameExtractor = /([a-z0-9.\-_\[\]]+)(?::[\d*]+)?\/(?:[^\x00-\x20\/]|$)[^\x00-\x20]*$/; -// end of local namespace -// ***************************************************************************** - -} - -/******************************************************************************/ /******************************************************************************/ µBlock.changeUserSettings = function(name, value) { diff --git a/src/js/uri-utils.js b/src/js/uri-utils.js new file mode 100644 index 000000000..99cbfdd7c --- /dev/null +++ b/src/js/uri-utils.js @@ -0,0 +1,126 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import '../lib/publicsuffixlist/publicsuffixlist.js'; +import '../lib/punycode.js'; + +import globals from './globals.js'; + +/******************************************************************************/ + +// Originally: +// https://github.com/gorhill/uBlock/blob/8b5733a58d3acf9fb62815e14699c986bd1c2fdc/src/js/uritools.js + +const psl = globals.publicSuffixList; +const punycode = globals.punycode; + +const reCommonHostnameFromURL = + /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//; +const reAuthorityFromURI = + /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; +const reHostFromNakedAuthority = + /^[0-9a-z._-]+[0-9a-z]$/i; +const reHostFromAuthority = + /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; +const reIPv6FromAuthority = + /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; +const reMustNormalizeHostname = + /[^0-9a-z._-]/; +const reOriginFromURI = + /^(?:[^:\/?#]+:)\/\/[^\/?#]+/; +const reHostnameFromNetworkURL = + /^(?:http|ws|ftp)s?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])(?::\d+)?\//; +const reIPAddressNaive = + /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; + +/******************************************************************************/ + +const domainFromHostname = function(hostname) { + return reIPAddressNaive.test(hostname) + ? hostname + : psl.getDomain(hostname); +}; + +const domainFromURI = function(uri) { + if ( !uri ) { return ''; } + return domainFromHostname(hostnameFromURI(uri)); +}; + +const entityFromDomain = function(domain) { + const pos = domain.indexOf('.'); + return pos !== -1 ? domain.slice(0, pos) + '.*' : ''; +}; + +const hostnameFromURI = function(uri) { + let matches = reCommonHostnameFromURL.exec(uri); + if ( matches !== null ) { return matches[1]; } + matches = reAuthorityFromURI.exec(uri); + if ( matches === null ) { return ''; } + const authority = matches[1].slice(2); + if ( reHostFromNakedAuthority.test(authority) ) { + return authority.toLowerCase(); + } + matches = reHostFromAuthority.exec(authority); + if ( matches === null ) { + matches = reIPv6FromAuthority.exec(authority); + if ( matches === null ) { return ''; } + } + let hostname = matches[1]; + while ( hostname.endsWith('.') ) { + hostname = hostname.slice(0, -1); + } + if ( reMustNormalizeHostname.test(hostname) ) { + hostname = punycode.toASCII(hostname.toLowerCase()); + } + return hostname; +}; + +const hostnameFromNetworkURL = function(url) { + const matches = reHostnameFromNetworkURL.exec(url); + return matches !== null ? matches[1] : ''; +}; + +const originFromURI = function(uri) { + const matches = reOriginFromURI.exec(uri); + return matches !== null ? matches[0].toLowerCase() : ''; +}; + +const isNetworkURI = function(uri) { + return reNetworkURI.test(uri); +}; + +const reNetworkURI = /^(?:ftps?|https?|wss?):\/\//; + +/******************************************************************************/ + +export { + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromNetworkURL, + hostnameFromURI, + isNetworkURI, + originFromURI, +}; diff --git a/src/js/uritools.js b/src/js/uritools.js deleted file mode 100644 index cb0a17883..000000000 --- a/src/js/uritools.js +++ /dev/null @@ -1,380 +0,0 @@ -/******************************************************************************* - - uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-present Raymond Hill - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see {http://www.gnu.org/licenses/}. - - Home: https://github.com/gorhill/uBlock -*/ - -'use strict'; - -/******************************************************************************* - -RFC 3986 as reference: http://tools.ietf.org/html/rfc3986#appendix-A - -Naming convention from https://en.wikipedia.org/wiki/URI_scheme#Examples - -*/ - -/******************************************************************************/ - -µBlock.URI = (( ) => { - -/******************************************************************************/ - -// Favorite regex tool: http://regex101.com/ - -// Ref: -// I removed redundant capture groups: capture less = peform faster. See -// -// Performance improvements welcomed. -// jsperf: -const reRFC3986 = /^([^:\/?#]+:)?(\/\/[^\/?#]*)?([^?#]*)(\?[^#]*)?(#.*)?/; - -// Derived -const reSchemeFromURI = /^[^:\/?#]+:/; -const reOriginFromURI = /^(?:[^:\/?#]+:)\/\/[^\/?#]+/; -const rePathFromURI = /^(?:[^:\/?#]+:)?(?:\/\/[^\/?#]*)?([^?#]*)/; - -// These are to parse authority field, not parsed by above official regex -// IPv6 is seen as an exception: a non-compatible IPv6 is first tried, and -// if it fails, the IPv6 compatible regex istr used. This helps -// peformance by avoiding the use of a too complicated regex first. - -// https://github.com/gorhill/httpswitchboard/issues/211 -// "While a hostname may not contain other characters, such as the -// "underscore character (_), other DNS names may contain the underscore" -const reHostPortFromAuthority = /^(?:[^@]*@)?([^:]*)(:\d*)?$/; -const reIPv6PortFromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]*\])(:\d*)?$/i; - -const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i; - -// Coarse (but fast) tests -const reValidHostname = /^([a-z\d]+(-*[a-z\d]+)*)(\.[a-z\d]+(-*[a-z\d])*)*$/; - -/******************************************************************************/ - -const reset = function(o) { - o.scheme = ''; - o.hostname = ''; - o._ipv4 = undefined; - o._ipv6 = undefined; - o.port = ''; - o.path = ''; - o.query = ''; - o.fragment = ''; - return o; -}; - -const resetAuthority = function(o) { - o.hostname = ''; - o._ipv4 = undefined; - o._ipv6 = undefined; - o.port = ''; - return o; -}; - -/******************************************************************************/ - -// This will be exported - -const URI = { - scheme: '', - authority: '', - hostname: '', - _ipv4: undefined, - _ipv6: undefined, - port: '', - domain: undefined, - path: '', - query: '', - fragment: '', - schemeBit: (1 << 0), - userBit: (1 << 1), - passwordBit: (1 << 2), - hostnameBit: (1 << 3), - portBit: (1 << 4), - pathBit: (1 << 5), - queryBit: (1 << 6), - fragmentBit: (1 << 7), - allBits: (0xFFFF) -}; - -URI.authorityBit = (URI.userBit | URI.passwordBit | URI.hostnameBit | URI.portBit); -URI.normalizeBits = (URI.schemeBit | URI.hostnameBit | URI.pathBit | URI.queryBit); - -/******************************************************************************/ - -// See: https://en.wikipedia.org/wiki/URI_scheme#Examples -// URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] -// -// foo://example.com:8042/over/there?name=ferret#nose -// \_/ \______________/\_________/ \_________/ \__/ -// | | | | | -// scheme authority path query fragment -// | _____________________|__ -// / \ / \ -// urn:example:animal:ferret:nose - -URI.set = function(uri) { - if ( uri === undefined ) { - return reset(URI); - } - let matches = reRFC3986.exec(uri); - if ( !matches ) { - return reset(URI); - } - this.scheme = matches[1] !== undefined ? matches[1].slice(0, -1) : ''; - this.authority = matches[2] !== undefined ? matches[2].slice(2).toLowerCase() : ''; - this.path = matches[3] !== undefined ? matches[3] : ''; - - // - // "In general, a URI that uses the generic syntax for authority - // "with an empty path should be normalized to a path of '/'." - if ( this.authority !== '' && this.path === '' ) { - this.path = '/'; - } - this.query = matches[4] !== undefined ? matches[4].slice(1) : ''; - this.fragment = matches[5] !== undefined ? matches[5].slice(1) : ''; - - // Assume very simple authority, i.e. just a hostname (highest likelihood - // case for µBlock) - if ( reHostFromNakedAuthority.test(this.authority) ) { - this.hostname = this.authority; - this.port = ''; - return this; - } - // Authority contains more than just a hostname - matches = reHostPortFromAuthority.exec(this.authority); - if ( !matches ) { - matches = reIPv6PortFromAuthority.exec(this.authority); - if ( !matches ) { - return resetAuthority(URI); - } - } - this.hostname = matches[1] !== undefined ? matches[1] : ''; - // http://en.wikipedia.org/wiki/FQDN - if ( this.hostname.endsWith('.') ) { - this.hostname = this.hostname.slice(0, -1); - } - this.port = matches[2] !== undefined ? matches[2].slice(1) : ''; - return this; -}; - -/******************************************************************************/ - -// URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] -// -// foo://example.com:8042/over/there?name=ferret#nose -// \_/ \______________/\_________/ \_________/ \__/ -// | | | | | -// scheme authority path query fragment -// | _____________________|__ -// / \ / \ -// urn:example:animal:ferret:nose - -URI.assemble = function(bits) { - if ( bits === undefined ) { - bits = this.allBits; - } - const s = []; - if ( this.scheme && (bits & this.schemeBit) ) { - s.push(this.scheme, ':'); - } - if ( this.hostname && (bits & this.hostnameBit) ) { - s.push('//', this.hostname); - } - if ( this.port && (bits & this.portBit) ) { - s.push(':', this.port); - } - if ( this.path && (bits & this.pathBit) ) { - s.push(this.path); - } - if ( this.query && (bits & this.queryBit) ) { - s.push('?', this.query); - } - if ( this.fragment && (bits & this.fragmentBit) ) { - s.push('#', this.fragment); - } - return s.join(''); -}; - -/******************************************************************************/ - -URI.originFromURI = function(uri) { - const matches = reOriginFromURI.exec(uri); - return matches !== null ? matches[0].toLowerCase() : ''; -}; - -/******************************************************************************/ - -URI.schemeFromURI = function(uri) { - const matches = reSchemeFromURI.exec(uri); - if ( !matches ) { return ''; } - return matches[0].slice(0, -1).toLowerCase(); -}; - -/******************************************************************************/ - -URI.hostnameFromURI = vAPI.hostnameFromURI; -URI.domainFromHostname = vAPI.domainFromHostname; - -URI.domain = function() { - return this.domainFromHostname(this.hostname); -}; - -/******************************************************************************/ - -URI.entityFromDomain = function(domain) { - const pos = domain.indexOf('.'); - return pos !== -1 ? domain.slice(0, pos) + '.*' : ''; -}; - -/******************************************************************************/ - -URI.pathFromURI = function(uri) { - const matches = rePathFromURI.exec(uri); - return matches !== null ? matches[1] : ''; -}; - -/******************************************************************************/ - -URI.domainFromURI = function(uri) { - if ( !uri ) { return ''; } - return this.domainFromHostname(this.hostnameFromURI(uri)); -}; - -/******************************************************************************/ - -URI.isNetworkURI = function(uri) { - return reNetworkURI.test(uri); -}; - -const reNetworkURI = /^(?:ftps?|https?|wss?):\/\//; - -/******************************************************************************/ - -URI.isNetworkScheme = function(scheme) { - return reNetworkScheme.test(scheme); -}; - -const reNetworkScheme = /^(?:ftps?|https?|wss?)$/; - -/******************************************************************************/ - -// Normalize the way µBlock expects it - -URI.normalizedURI = function() { - // Will be removed: - // - port - // - user id/password - // - fragment - return this.assemble(this.normalizeBits); -}; - -/******************************************************************************/ - -URI.rootURL = function() { - if ( !this.hostname ) { return ''; } - return this.assemble(this.schemeBit | this.hostnameBit); -}; - -/******************************************************************************/ - -URI.isValidHostname = function(hostname) { - try { - return reValidHostname.test(hostname); - } - catch (e) { - } - return false; -}; - -/******************************************************************************/ - -// Return the parent domain. For IP address, there is no parent domain. - -URI.parentHostnameFromHostname = function(hostname) { - // `locahost` => `` - // `example.org` => `example.org` - // `www.example.org` => `example.org` - // `tomato.www.example.org` => `example.org` - const domain = this.domainFromHostname(hostname); - - // `locahost` === `` => bye - // `example.org` === `example.org` => bye - // `www.example.org` !== `example.org` => stay - // `tomato.www.example.org` !== `example.org` => stay - if ( domain === '' || domain === hostname ) { return; } - - // Parent is hostname minus first label - return hostname.slice(hostname.indexOf('.') + 1); -}; - -/******************************************************************************/ - -// Return all possible parent hostnames which can be derived from `hostname`, -// ordered from direct parent up to domain inclusively. - -URI.parentHostnamesFromHostname = function(hostname) { - // TODO: I should create an object which is optimized to receive - // the list of hostnames by making it reusable (junkyard etc.) and which - // has its own element counter property in order to avoid memory - // alloc/dealloc. - const domain = this.domainFromHostname(hostname); - if ( domain === '' || domain === hostname ) { - return []; - } - const nodes = []; - for (;;) { - const pos = hostname.indexOf('.'); - if ( pos < 0 ) { break; } - hostname = hostname.slice(pos + 1); - nodes.push(hostname); - if ( hostname === domain ) { break; } - } - return nodes; -}; - -/******************************************************************************/ - -// Return all possible hostnames which can be derived from `hostname`, -// ordered from self up to domain inclusively. - -URI.allHostnamesFromHostname = function(hostname) { - const nodes = this.parentHostnamesFromHostname(hostname); - nodes.unshift(hostname); - return nodes; -}; - -/******************************************************************************/ - -URI.toString = function() { - return this.assemble(); -}; - -/******************************************************************************/ - -// Export - -return URI; - -/******************************************************************************/ - -})(); - -/******************************************************************************/ - diff --git a/src/js/url-net-filtering.js b/src/js/url-net-filtering.js index aa6a9e232..c5f1e2283 100644 --- a/src/js/url-net-filtering.js +++ b/src/js/url-net-filtering.js @@ -23,21 +23,20 @@ /******************************************************************************/ -// The purpose of log filtering is to create ad hoc filtering rules, to -// diagnose and assist in the creation of custom filters. - -µBlock.URLNetFiltering = (( ) => { +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; /******************************************************************************* -buckets: map of [hostname + type] - bucket: array of rule entries, sorted from shorter to longer url -rule entry: { url, action } + The purpose of log filtering is to create ad hoc filtering rules, to + diagnose and assist in the creation of custom filters. + + buckets: map of [hostname + type] + bucket: array of rule entries, sorted from shorter to longer url + rule entry: { url, action } *******************************************************************************/ -/******************************************************************************/ - const actionToNameMap = { 1: 'block', 2: 'allow', @@ -330,7 +329,7 @@ URLNetFiltering.prototype.toString = function() { URLNetFiltering.prototype.fromString = function(text) { this.reset(); - const lineIter = new µBlock.LineIterator(text); + const lineIter = new LineIterator(text); while ( lineIter.eot() === false ) { this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); } @@ -371,15 +370,11 @@ URLNetFiltering.prototype.removeFromRuleParts = function(parts) { /******************************************************************************/ -return URLNetFiltering; - -/******************************************************************************/ - -})(); - -/******************************************************************************/ - -µBlock.sessionURLFiltering = new µBlock.URLNetFiltering(); -µBlock.permanentURLFiltering = new µBlock.URLNetFiltering(); +// Export + +µBlock.URLNetFiltering = URLNetFiltering; + +µBlock.sessionURLFiltering = new URLNetFiltering(); +µBlock.permanentURLFiltering = new URLNetFiltering(); /******************************************************************************/ diff --git a/src/js/utils.js b/src/js/utils.js index 47e84d557..955ce8ceb 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -23,6 +23,11 @@ /******************************************************************************/ +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; + +/******************************************************************************/ + µBlock.formatCount = function(count) { if ( typeof count !== 'number' ) { return ''; @@ -57,183 +62,6 @@ /******************************************************************************/ -µBlock.LineIterator = class { - constructor(text, offset) { - this.text = text; - this.textLen = this.text.length; - this.offset = offset || 0; - } - next(offset) { - if ( offset !== undefined ) { - this.offset += offset; - } - let lineEnd = this.text.indexOf('\n', this.offset); - if ( lineEnd === -1 ) { - lineEnd = this.text.indexOf('\r', this.offset); - if ( lineEnd === -1 ) { - lineEnd = this.textLen; - } - } - const line = this.text.slice(this.offset, lineEnd); - this.offset = lineEnd + 1; - return line; - } - peek(n) { - const offset = this.offset; - return this.text.slice(offset, offset + n); - } - charCodeAt(offset) { - return this.text.charCodeAt(this.offset + offset); - } - eot() { - return this.offset >= this.textLen; - } -}; - -/******************************************************************************/ - -// The field iterator is less CPU-intensive than when using native -// String.split(). - -µBlock.FieldIterator = class { - constructor(sep) { - this.text = ''; - this.sep = sep; - this.sepLen = sep.length; - this.offset = 0; - } - first(text) { - this.text = text; - this.offset = 0; - return this.next(); - } - next() { - let end = this.text.indexOf(this.sep, this.offset); - if ( end === -1 ) { - end = this.text.length; - } - const field = this.text.slice(this.offset, end); - this.offset = end + this.sepLen; - return field; - } - remainder() { - return this.text.slice(this.offset); - } -}; - -/******************************************************************************/ - -// https://www.reddit.com/r/uBlockOrigin/comments/oq6kt5/ubo_loads_generic_filter_instead_of_specific/ -// Ensure blocks of content are sorted in ascending id order, such that the -// specific cosmetic filters will be found (and thus reported) before the -// generic ones. - -µBlock.CompiledLineIO = { - serialize: JSON.stringify, - unserialize: JSON.parse, - blockStartPrefix: '#block-start-', // ensure no special regex characters - blockEndPrefix: '#block-end-', // ensure no special regex characters - - Writer: class { - constructor() { - this.io = µBlock.CompiledLineIO; - this.blockId = undefined; - this.block = undefined; - this.stringifier = this.io.serialize; - this.blocks = new Map(); - this.properties = new Map(); - } - push(args) { - this.block.push(this.stringifier(args)); - } - last() { - if ( Array.isArray(this.block) && this.block.length !== 0 ) { - return this.block[this.block.length - 1]; - } - } - select(blockId) { - if ( blockId === this.blockId ) { return; } - this.blockId = blockId; - this.block = this.blocks.get(blockId); - if ( this.block === undefined ) { - this.blocks.set(blockId, (this.block = [])); - } - return this; - } - toString() { - const result = []; - const sortedBlocks = - Array.from(this.blocks).sort((a, b) => a[0] - b[0]); - for ( const [ id, lines ] of sortedBlocks ) { - if ( lines.length === 0 ) { continue; } - result.push( - this.io.blockStartPrefix + id, - lines.join('\n'), - this.io.blockEndPrefix + id - ); - } - return result.join('\n'); - } - }, - - Reader: class { - constructor(raw, blockId) { - this.io = µBlock.CompiledLineIO; - this.block = ''; - this.len = 0; - this.offset = 0; - this.line = ''; - this.parser = this.io.unserialize; - this.blocks = new Map(); - this.properties = new Map(); - let reBlockStart = new RegExp( - `^${this.io.blockStartPrefix}(\\d+)\\n`, - 'gm' - ); - let match = reBlockStart.exec(raw); - while ( match !== null ) { - let beg = match.index + match[0].length; - let end = raw.indexOf(this.io.blockEndPrefix + match[1], beg); - this.blocks.set(parseInt(match[1], 10), raw.slice(beg, end)); - reBlockStart.lastIndex = end; - match = reBlockStart.exec(raw); - } - if ( blockId !== undefined ) { - this.select(blockId); - } - } - next() { - if ( this.offset === this.len ) { - this.line = ''; - return false; - } - let pos = this.block.indexOf('\n', this.offset); - if ( pos !== -1 ) { - this.line = this.block.slice(this.offset, pos); - this.offset = pos + 1; - } else { - this.line = this.block.slice(this.offset); - this.offset = this.len; - } - return true; - } - select(blockId) { - this.block = this.blocks.get(blockId) || ''; - this.len = this.block.length; - this.offset = 0; - return this; - } - fingerprint() { - return this.line; - } - args() { - return this.parser(this.line); - } - } -}; - -/******************************************************************************/ - µBlock.openNewTab = function(details) { if ( details.url.startsWith('logger-ui.html') ) { if ( details.shiftKey ) { @@ -374,228 +202,6 @@ /******************************************************************************/ -// Custom base64 codecs. These codecs are meant to encode/decode typed arrays -// to/from strings. - -// https://github.com/uBlockOrigin/uBlock-issues/issues/461 -// Provide a fallback encoding for Chromium 59 and less by issuing a plain -// JSON string. The fallback can be removed once min supported version is -// above 59. - -// TODO: rename µBlock.base64 to µBlock.SparseBase64, now that -// µBlock.DenseBase64 has been introduced. -// TODO: Should no longer need to test presence of TextEncoder/TextDecoder. - -{ - const valToDigit = new Uint8Array(64); - const digitToVal = new Uint8Array(128); - { - const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@%'; - for ( let i = 0, n = chars.length; i < n; i++ ) { - const c = chars.charCodeAt(i); - valToDigit[i] = c; - digitToVal[c] = i; - } - } - - // The sparse base64 codec is best for buffers which contains a lot of - // small u32 integer values. Those small u32 integer values are better - // represented with stringified integers, because small values can be - // represented with fewer bits than the usual base64 codec. For example, - // 0 become '0 ', i.e. 16 bits instead of 48 bits with official base64 - // codec. - - µBlock.base64 = { - magic: 'Base64_1', - - encode: function(arrbuf, arrlen) { - const inputLength = (arrlen + 3) >>> 2; - const inbuf = new Uint32Array(arrbuf, 0, inputLength); - const outputLength = this.magic.length + 7 + inputLength * 7; - const outbuf = new Uint8Array(outputLength); - // magic bytes - let j = 0; - for ( let i = 0; i < this.magic.length; i++ ) { - outbuf[j++] = this.magic.charCodeAt(i); - } - // array size - let v = inputLength; - do { - outbuf[j++] = valToDigit[v & 0b111111]; - v >>>= 6; - } while ( v !== 0 ); - outbuf[j++] = 0x20 /* ' ' */; - // array content - for ( let i = 0; i < inputLength; i++ ) { - v = inbuf[i]; - do { - outbuf[j++] = valToDigit[v & 0b111111]; - v >>>= 6; - } while ( v !== 0 ); - outbuf[j++] = 0x20 /* ' ' */; - } - if ( typeof TextDecoder === 'undefined' ) { - return JSON.stringify( - Array.from(new Uint32Array(outbuf.buffer, 0, j >>> 2)) - ); - } - const textDecoder = new TextDecoder(); - return textDecoder.decode(new Uint8Array(outbuf.buffer, 0, j)); - }, - - decode: function(instr, arrbuf) { - if ( instr.charCodeAt(0) === 0x5B /* '[' */ ) { - const inbuf = JSON.parse(instr); - if ( arrbuf instanceof ArrayBuffer === false ) { - return new Uint32Array(inbuf); - } - const outbuf = new Uint32Array(arrbuf); - outbuf.set(inbuf); - return outbuf; - } - if ( instr.startsWith(this.magic) === false ) { - throw new Error('Invalid µBlock.base64 encoding'); - } - const inputLength = instr.length; - const outputLength = this.decodeSize(instr) >> 2; - const outbuf = arrbuf instanceof ArrayBuffer === false - ? new Uint32Array(outputLength) - : new Uint32Array(arrbuf); - let i = instr.indexOf(' ', this.magic.length) + 1; - if ( i === -1 ) { - throw new Error('Invalid µBlock.base64 encoding'); - } - // array content - let j = 0; - for (;;) { - if ( j === outputLength || i >= inputLength ) { break; } - let v = 0, l = 0; - for (;;) { - const c = instr.charCodeAt(i++); - if ( c === 0x20 /* ' ' */ ) { break; } - v += digitToVal[c] << l; - l += 6; - } - outbuf[j++] = v; - } - if ( i < inputLength || j < outputLength ) { - throw new Error('Invalid µBlock.base64 encoding'); - } - return outbuf; - }, - - decodeSize: function(instr) { - if ( instr.startsWith(this.magic) === false ) { return 0; } - let v = 0, l = 0, i = this.magic.length; - for (;;) { - const c = instr.charCodeAt(i++); - if ( c === 0x20 /* ' ' */ ) { break; } - v += digitToVal[c] << l; - l += 6; - } - return v << 2; - }, - }; - - // The dense base64 codec is best for typed buffers which values are - // more random. For example, buffer contents as a result of compression - // contain less repetitive values and thus the content is more - // random-looking. - - // TODO: Investigate that in Firefox, creating a new Uint8Array from the - // ArrayBuffer fails, the content of the resulting Uint8Array is - // non-sensical. WASM-related? - - µBlock.denseBase64 = { - magic: 'DenseBase64_1', - - encode: function(input) { - const m = input.length % 3; - const n = input.length - m; - let outputLength = n / 3 * 4; - if ( m !== 0 ) { - outputLength += m + 1; - } - const output = new Uint8Array(outputLength); - let j = 0; - for ( let i = 0; i < n; i += 3) { - const i1 = input[i+0]; - const i2 = input[i+1]; - const i3 = input[i+2]; - output[j+0] = valToDigit[ i1 >>> 2]; - output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; - output[j+2] = valToDigit[i2 << 2 & 0b111100 | i3 >>> 6]; - output[j+3] = valToDigit[i3 & 0b111111 ]; - j += 4; - } - if ( m !== 0 ) { - const i1 = input[n]; - output[j+0] = valToDigit[i1 >>> 2]; - if ( m === 1 ) { // 1 value - output[j+1] = valToDigit[i1 << 4 & 0b110000]; - } else { // 2 values - const i2 = input[n+1]; - output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; - output[j+2] = valToDigit[i2 << 2 & 0b111100 ]; - } - } - const textDecoder = new TextDecoder(); - const b64str = textDecoder.decode(output); - return this.magic + b64str; - }, - - decode: function(instr, arrbuf) { - if ( instr.startsWith(this.magic) === false ) { - throw new Error('Invalid µBlock.denseBase64 encoding'); - } - const outputLength = this.decodeSize(instr); - const outbuf = arrbuf instanceof ArrayBuffer === false - ? new Uint8Array(outputLength) - : new Uint8Array(arrbuf); - const inputLength = instr.length - this.magic.length; - let i = this.magic.length; - let j = 0; - const m = inputLength & 3; - const n = i + inputLength - m; - while ( i < n ) { - const i1 = digitToVal[instr.charCodeAt(i+0)]; - const i2 = digitToVal[instr.charCodeAt(i+1)]; - const i3 = digitToVal[instr.charCodeAt(i+2)]; - const i4 = digitToVal[instr.charCodeAt(i+3)]; - i += 4; - outbuf[j+0] = i1 << 2 | i2 >>> 4; - outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; - outbuf[j+2] = i3 << 6 & 0b11000000 | i4; - j += 3; - } - if ( m !== 0 ) { - const i1 = digitToVal[instr.charCodeAt(i+0)]; - const i2 = digitToVal[instr.charCodeAt(i+1)]; - outbuf[j+0] = i1 << 2 | i2 >>> 4; - if ( m === 3 ) { - const i3 = digitToVal[instr.charCodeAt(i+2)]; - outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; - } - } - return outbuf; - }, - - decodeSize: function(instr) { - if ( instr.startsWith(this.magic) === false ) { return 0; } - const inputLength = instr.length - this.magic.length; - const m = inputLength & 3; - const n = inputLength - m; - let outputLength = (n >>> 2) * 3; - if ( m !== 0 ) { - outputLength += m - 1; - } - return outputLength; - }, - }; -} - -/******************************************************************************/ - // The requests.json.gz file can be downloaded from: // https://cdn.cliqz.com/adblocking/requests_top500.json.gz // @@ -654,7 +260,7 @@ datasetPromise = µBlock.assets.fetchText(datasetURL).then(details => { console.info(`Parsing benchmark dataset...`); const requests = []; - const lineIter = new µBlock.LineIterator(details.content); + const lineIter = new LineIterator(details.content); while ( lineIter.eot() === false ) { let request; try { diff --git a/src/lib/publicsuffixlist/publicsuffixlist.js b/src/lib/publicsuffixlist/publicsuffixlist.js index 9963c1f88..a073d49ac 100644 --- a/src/lib/publicsuffixlist/publicsuffixlist.js +++ b/src/lib/publicsuffixlist/publicsuffixlist.js @@ -16,6 +16,8 @@ /* jshint browser:true, esversion:6, laxbreak:true, undef:true, unused:true */ /* globals WebAssembly, console, exports:true, module */ +'use strict'; + /******************************************************************************* Reference: @@ -45,8 +47,6 @@ (function(context) { // >>>>>>>> start of anonymous namespace -'use strict'; - /******************************************************************************* Tree encoding in array buffer: @@ -81,13 +81,11 @@ let hostnameArg = EMPTY_STRING; /******************************************************************************/ const fireChangedEvent = function() { - if ( - window instanceof Object && - window.dispatchEvent instanceof Function && - window.CustomEvent instanceof Function - ) { - window.dispatchEvent(new CustomEvent('publicSuffixListChanged')); - } + if ( typeof window !== 'object' ) { return; } + if ( window instanceof Object === false ) { return; } + if ( window.dispatchEvent instanceof Function === false ) { return; } + if ( window.CustomEvent instanceof Function === false ) { return; } + window.dispatchEvent(new CustomEvent('publicSuffixListChanged')); }; /******************************************************************************/ @@ -531,22 +529,9 @@ const fromSelfie = function(selfie, decoder) { // used should the WASM module be unavailable for whatever reason. const enableWASM = (function() { - // The directory from which the current script was fetched should also - // contain the related WASM file. The script is fetched from a trusted - // location, and consequently so will be the related WASM file. - let workingDir; - { - const url = new URL(document.currentScript.src); - const match = /[^\/]+$/.exec(url.pathname); - if ( match !== null ) { - url.pathname = url.pathname.slice(0, match.index); - } - workingDir = url.href; - } - let memory; - return function() { + return function(modulePath) { if ( getPublicSuffixPosWASM instanceof Function ) { return Promise.resolve(true); } @@ -568,7 +553,7 @@ const enableWASM = (function() { } return fetch( - workingDir + 'wasm/publicsuffixlist.wasm', + `${modulePath}/wasm/publicsuffixlist.wasm`, { mode: 'same-origin' } ).then(response => { const pageCount = pslBuffer8 !== undefined @@ -624,8 +609,6 @@ const disableWASM = function() { /******************************************************************************/ -context = context || window; - context.publicSuffixList = { version: '2.0', parse, @@ -644,4 +627,14 @@ if ( typeof module !== 'undefined' ) { /******************************************************************************/ // <<<<<<<< end of anonymous namespace -})(this); +})( + (root => { + if ( root !== undefined ) { return root; } + // jshint ignore:start + if ( typeof self !== 'undefined' ) { return self; } + if ( typeof window !== 'undefined' ) { return window; } + if ( typeof global !== 'undefined' ) { return global; } + // jshint ignore:end + throw new Error('unable to locate global object'); + })(this) +); diff --git a/src/lib/punycode.js b/src/lib/punycode.js index f3866b8e6..18a94eac6 100644 --- a/src/lib/punycode.js +++ b/src/lib/punycode.js @@ -6,7 +6,7 @@ !exports.nodeType && exports; var freeModule = typeof module == 'object' && module && !module.nodeType && module; - var freeGlobal = typeof global == 'object' && global; + var freeGlobal = typeof global == 'object' && global || self; if ( freeGlobal.global === freeGlobal || freeGlobal.window === freeGlobal || diff --git a/src/lib/regexanalyzer/regex.js b/src/lib/regexanalyzer/regex.js index 788f03ee8..1decf2534 100644 --- a/src/lib/regexanalyzer/regex.js +++ b/src/lib/regexanalyzer/regex.js @@ -9,9 +9,7 @@ **/ !function( root, name, factory ){ "use strict"; -if ( ('undefined'!==typeof Components)&&('object'===typeof Components.classes)&&('object'===typeof Components.classesByID)&&Components.utils&&('function'===typeof Components.utils['import']) ) /* XPCOM */ - (root.$deps = root.$deps||{}) && (root.EXPORTED_SYMBOLS = [name]) && (root[name] = root.$deps[name] = factory.call(root)); -else if ( ('object'===typeof module)&&module.exports ) /* CommonJS */ +if ( ('object'===typeof module)&&module.exports ) /* CommonJS */ (module.$deps = module.$deps||{}) && (module.exports = module.$deps[name] = factory.call(root)); else if ( ('undefined'!==typeof System)&&('function'===typeof System.register)&&('function'===typeof System['import']) ) /* ES6 module */ System.register(name,[],function($__export){$__export(name, factory.call(root));}); @@ -19,7 +17,11 @@ else if ( ('function'===typeof define)&&define.amd&&('function'===typeof require define(name,['module'],function(module){factory.moduleUri = module.uri; return factory.call(root);}); else if ( !(name in root) ) /* Browser/WebWorker/.. */ (root[name] = factory.call(root)||1)&&('function'===typeof(define))&&define.amd&&define(function(){return root[name];} ); -}( /* current root */ 'undefined' !== typeof self ? self : this, +}( /* current root */ (( ) => { + if ( typeof globalThis !== 'undefined' ) { return globalThis; } + if ( typeof self !== 'undefined' ) { return self; } + if ( typeof global !== 'undefined' ) { return global; } + })(), /* module name */ "Regex", /* module factory */ function ModuleFactory__Regex( undef ){ "use strict"; diff --git a/src/logger-ui.html b/src/logger-ui.html index 884a01339..7ea8f46ad 100644 --- a/src/logger-ui.html +++ b/src/logger-ui.html @@ -212,8 +212,8 @@ - - + + diff --git a/src/web_accessible_resources/epicker-ui.html b/src/web_accessible_resources/epicker-ui.html index 8ab76e32b..a99e0d764 100644 --- a/src/web_accessible_resources/epicker-ui.html +++ b/src/web_accessible_resources/epicker-ui.html @@ -61,15 +61,12 @@ - - - - + diff --git a/tools/copy-common-files.sh b/tools/copy-common-files.sh index 7500512ea..83da91b91 100644 --- a/tools/copy-common-files.sh +++ b/tools/copy-common-files.sh @@ -8,7 +8,11 @@ bash ./tools/make-assets.sh $DES cp -R src/css $DES/ cp -R src/img $DES/ -cp -R src/js $DES/ +mkdir $DES/js +cp -R src/js/*.js $DES/js/ +cp -R src/js/codemirror $DES/js/ +cp -R src/js/scriptlets $DES/js/ +cp -R src/js/wasm $DES/js/ cp -R src/lib $DES/ cp -R src/web_accessible_resources $DES/ cp -R src/_locales $DES/ diff --git a/tools/import-war.py b/tools/import-war.py deleted file mode 100755 index e4f4f57fe..000000000 --- a/tools/import-war.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 - -import base64 -import hashlib -import os -import re -import sys - -if len(sys.argv) == 1 or not sys.argv[1]: - raise SystemExit('Build dir missing.') - -# resource_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..') -build_dir = os.path.abspath(sys.argv[1]) - -# Read list of resource tokens to convert -to_import = set() -with open('./src/web_accessible_resources/to-import.txt', 'r') as f: - for line in f: - line = line.strip() - if len(line) != 0 and line[0] != '#': - to_import.add(line) - -# https://github.com/gorhill/uBlock/issues/3636 -safe_exts = { 'javascript': 'js', 'plain': 'txt' } - -imported = [] - -# scan the file until a resource to import is found -def find_next_resource(f): - for line in f: - line = line.strip() - if len(line) == 0 or line[0] == '#': - continue - parts = line.partition(' ') - if parts[0] in to_import: - return (parts[0], parts[2].strip()) - return ('', '') - -def safe_filename_from_token(token, mime): - h = hashlib.md5() - h.update(bytes(token, 'utf-8')) - name = h.hexdigest() - # extract file extension from mime - match = re.search('^[^/]+/([^\s;]+)', mime) - if match: - ext = match.group(1) - if ext in safe_exts: - ext = safe_exts[ext] - name += '.' + ext - return name - -def import_resource(f, token, mime): - isBinary = mime.endswith(';base64') - lines = [] - for line in f: - if line.strip() == '': - break - if line.lstrip()[0] == '#': - continue - if isBinary: - line = line.strip() - lines.append(line) - filename = safe_filename_from_token(token, mime) - filepath = os.path.join(build_dir, 'web_accessible_resources', filename) - filedata = ''.join(lines) - if isBinary: - filedata = base64.b64decode(filedata) - else: - filedata = bytes(filedata, 'utf-8') - with open(filepath, 'wb') as fo: - fo.write(filedata) - imported.append(token + '\n\t' + filename) - -# Read content of the resources to convert -# - At this point, it is assumed resources.txt has been imported into the -# package. -resources_filename = os.path.join(build_dir, 'assets/ublock/resources.txt') -with open(resources_filename, 'r') as f: - while True: - token, mime = find_next_resource(f) - if token == '': - break - import_resource(f, token, mime) - -# Output associations -content = '' -with open('./src/web_accessible_resources/imported.txt', 'r') as f: - content = f.read() + '\n'.join(imported) - filename = os.path.join(build_dir, 'web_accessible_resources/imported.txt') - with open(filename, 'w') as f: - f.write(content) - diff --git a/tools/make-browser.sh b/tools/make-browser.sh new file mode 100755 index 000000000..700157d19 --- /dev/null +++ b/tools/make-browser.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +DES=dist/build/uBlock0.browser + +mkdir -p $DES/js +cp src/js/base64-custom.js $DES/js +cp src/js/biditrie.js $DES/js +cp src/js/filtering-context.js $DES/js +cp src/js/globals.js $DES/js +cp src/js/hntrie.js $DES/js +cp src/js/static-filtering-parser.js $DES/js +cp src/js/static-net-filtering.js $DES/js +cp src/js/static-filtering-io.js $DES/js +cp src/js/text-iterators.js $DES/js +cp src/js/uri-utils.js $DES/js + +mkdir -p $DES/js/wasm +cp -R src/js/wasm $DES/js/ + +mkdir -p $DES/lib +cp -R src/lib/punycode.js $DES/lib/ +cp -R src/lib/publicsuffixlist $DES/lib/ +cp -R src/lib/regexanalyzer $DES/lib/ + +mkdir -p $DES/data +cp -R ../uAssets/thirdparties/publicsuffix.org/list/* \ + $DES/data +cp -R ../uAssets/thirdparties/easylist-downloads.adblockplus.org/* \ + $DES/data + +cp platform/browser/*.html $DES/ +cp platform/browser/*.js $DES/ +cp LICENSE.txt $DES/ diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh index f32bd2330..047146026 100755 --- a/tools/make-chromium.sh +++ b/tools/make-chromium.sh @@ -13,9 +13,9 @@ bash ./tools/copy-common-files.sh $DES # Chromium-specific echo "*** uBlock0.chromium: Copying chromium-specific files" -cp platform/chromium/*.js $DES/js/ -cp platform/chromium/*.html $DES/ -cp platform/chromium/*.json $DES/ +cp platform/chromium/*.js $DES/js/ +cp platform/chromium/*.html $DES/ +cp platform/chromium/*.json $DES/ # Chrome store-specific cp -R $DES/_locales/nb $DES/_locales/no diff --git a/tools/make-firefox.sh b/tools/make-firefox.sh index 31ec7ae20..b50eae886 100755 --- a/tools/make-firefox.sh +++ b/tools/make-firefox.sh @@ -14,11 +14,11 @@ bash ./tools/copy-common-files.sh $DES # Firefox-specific echo "*** uBlock0.firefox: Copying firefox-specific files" -cp platform/firefox/*.json $DES/ -cp platform/firefox/*.js $DES/js/ +cp platform/firefox/*.json $DES/ +cp platform/firefox/*.js $DES/js/ # Firefox store-specific -cp -R $DES/_locales/nb $DES/_locales/no +cp -R $DES/_locales/nb $DES/_locales/no # Firefox/webext-specific rm $DES/img/icon_128.png diff --git a/tools/make-nodejs.sh b/tools/make-nodejs.sh new file mode 100755 index 000000000..d217ba953 --- /dev/null +++ b/tools/make-nodejs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +DES=dist/build/uBlock0.nodejs + +mkdir -p $DES/js +cp src/js/base64-custom.js $DES/js +cp src/js/biditrie.js $DES/js +cp src/js/filtering-context.js $DES/js +cp src/js/globals.js $DES/js +cp src/js/hntrie.js $DES/js +cp src/js/static-filtering-parser.js $DES/js +cp src/js/static-net-filtering.js $DES/js +cp src/js/static-filtering-io.js $DES/js +cp src/js/text-iterators.js $DES/js +cp src/js/uri-utils.js $DES/js + +mkdir -p $DES/lib +cp -R src/lib/punycode.js $DES/lib/ +cp -R src/lib/publicsuffixlist $DES/lib/ +cp -R src/lib/regexanalyzer $DES/lib/ + +mkdir -p $DES/data +cp -R ../uAssets/thirdparties/publicsuffix.org/list/* \ + $DES/data +cp -R ../uAssets/thirdparties/easylist-downloads.adblockplus.org/* \ + $DES/data + +cp platform/nodejs/*.js $DES/ +cp platform/nodejs/*.json $DES/ +cp LICENSE.txt $DES/ diff --git a/tools/make-opera.sh b/tools/make-opera.sh index 8a6c7443a..0e95e1a41 100755 --- a/tools/make-opera.sh +++ b/tools/make-opera.sh @@ -13,12 +13,12 @@ bash ./tools/copy-common-files.sh $DES # Chromium-specific echo "*** uBlock0.opera: Copying chromium-specific files" -cp platform/chromium/*.js $DES/js/ -cp platform/chromium/*.html $DES/ +cp platform/chromium/*.js $DES/js/ +cp platform/chromium/*.html $DES/ # Opera-specific echo "*** uBlock0.opera: Copying opera-specific files" -cp platform/opera/manifest.json $DES/ +cp platform/opera/manifest.json $DES/ rm -r $DES/_locales/az rm -r $DES/_locales/cv diff --git a/tools/make-safari-meta.py b/tools/make-safari-meta.py deleted file mode 100644 index ad6a63cd4..000000000 --- a/tools/make-safari-meta.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -import os -import json -import sys -from io import open -from time import time -from shutil import rmtree -from collections import OrderedDict - -if len(sys.argv) == 1 or not sys.argv[1]: - raise SystemExit('Build dir missing.') - - -def mkdirs(path): - try: - os.makedirs(path) - finally: - return os.path.exists(path) - -pj = os.path.join -build_dir = os.path.abspath(sys.argv[1]) - -description = '' - -# locales -locale_dir = pj(build_dir, '_locales') - -for alpha2 in sorted(os.listdir(locale_dir)): - locale_path = pj(locale_dir, alpha2, 'messages.json') - with open(locale_path, encoding='utf-8') as f: - string_data = json.load(f, object_pairs_hook=OrderedDict) - - if alpha2 == 'en': - description = string_data['extShortDesc']['message'] - - for string_name in string_data: - string_data[string_name] = string_data[string_name]['message'] - - rmtree(pj(locale_dir, alpha2)) - - alpha2 = alpha2.replace('_', '-') - locale_path = pj(locale_dir, alpha2 + '.json') - - mkdirs(pj(locale_dir)) - - with open(locale_path, 'wb') as f: - f.write(json.dumps(string_data, ensure_ascii=False).encode('utf8')) - - -# update Info.plist -proj_dir = pj(os.path.split(os.path.abspath(__file__))[0], '..') -chromium_manifest = pj(proj_dir, 'platform', 'chromium', 'manifest.json') - -with open(chromium_manifest, encoding='utf-8') as m: - manifest = json.load(m) - -manifest['buildNumber'] = int(time()) -manifest['description'] = description - -info_plist = pj(build_dir, 'Info.plist') - -with open(info_plist, 'r+t', encoding='utf-8', newline='\n') as f: - info_plist = f.read() - f.seek(0) - - f.write(info_plist.format(**manifest)) - -# update Update.plist -update_plist = pj(proj_dir, 'platform', 'safari', 'Update.plist') -update_plist_build = pj(build_dir, '..', os.path.basename(update_plist)) - -with open(update_plist_build, 'wt', encoding='utf-8', newline='\n') as f: - with open(update_plist, encoding='utf-8') as u: - update_plist = u.read() - - f.write(update_plist.format(**manifest)) diff --git a/tools/make-safari.sh b/tools/make-safari.sh deleted file mode 100755 index db96811eb..000000000 --- a/tools/make-safari.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# This script assumes an OS X or *NIX environment - -echo "*** uBlock.safariextension: Copying files..." - -DES=dist/build/uBlock.safariextension -rm -rf $DES -mkdir -p $DES - -cp -R assets $DES/ -rm $DES/assets/*.sh -cp -R src/css $DES/ -cp -R src/img $DES/ -cp -R src/js $DES/ -cp -R src/lib $DES/ -cp -R src/_locales $DES/ -cp src/*.html $DES/ -mv $DES/img/icon_128.png $DES/Icon.png -cp platform/safari/*.js $DES/js/ -cp -R platform/safari/img $DES/ -cp platform/safari/Info.plist $DES/ -cp platform/safari/Settings.plist $DES/ -cp LICENSE.txt $DES/ - -echo "*** uBlock.safariextension: Generating Info.plist..." -python tools/make-safari-meta.py $DES/ - -if [ "$1" = all ]; then - echo "*** Use Safari's Extension Builder to create the signed uBlock extension package -- can't automate it." -fi - -echo "*** uBlock.safariextension: Done." diff --git a/tools/make-thunderbird.sh b/tools/make-thunderbird.sh index 16a17121f..590a2bce9 100755 --- a/tools/make-thunderbird.sh +++ b/tools/make-thunderbird.sh @@ -13,7 +13,7 @@ echo "*** uBlock0.thunderbird: copying common files" bash ./tools/copy-common-files.sh $DES echo "*** uBlock0.firefox: Copying firefox-specific files" -cp platform/firefox/*.js $DES/js/ +cp platform/firefox/*.js $DES/js/ echo "*** uBlock0.firefox: Copying thunderbird-specific files" cp platform/thunderbird/manifest.json $DES/ diff --git a/tools/make-webext-meta.py b/tools/make-webext-meta.py deleted file mode 100644 index 8639d2b37..000000000 --- a/tools/make-webext-meta.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -import os -import json -import re -import sys - -if len(sys.argv) == 1 or not sys.argv[1]: - raise SystemExit('Build dir missing.') - -proj_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..') -build_dir = os.path.abspath(sys.argv[1]) - -version = '' -with open(os.path.join(proj_dir, 'dist', 'version')) as f: - version = f.read().strip() - -webext_manifest = {} -webext_manifest_file = os.path.join(build_dir, 'manifest.json') -with open(webext_manifest_file, encoding='utf-8') as f2: - webext_manifest = json.load(f2) - -webext_manifest['version'] = version - -match = re.search('^\d+\.\d+\.\d+\.\d+$', version) -if match: - webext_manifest['name'] += ' development build' - webext_manifest['short_name'] += ' dev build' - webext_manifest['browser_action']['default_title'] += ' dev build' -else: - # https://bugzilla.mozilla.org/show_bug.cgi?id=1459007 - # By design Firefox opens the sidebar with new installation of - # uBO when sidebar_action is present in the manifest. - # Remove sidebarAction support for stable release of uBO. - del webext_manifest['sidebar_action'] - -with open(webext_manifest_file, mode='w', encoding='utf-8') as f2: - json.dump(webext_manifest, f2, indent=2, separators=(',', ': '), sort_keys=True) - f2.write('\n') diff --git a/tools/make-webext.sh b/tools/make-webext.sh deleted file mode 100755 index 80e66a0f4..000000000 --- a/tools/make-webext.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# This script assumes a linux environment - -# https://github.com/uBlockOrigin/uBlock-issues/issues/217 -set -e - -echo "*** uBlock0.webext: Creating web store package" - -DES=dist/build/uBlock0.webext -rm -rf $DES -mkdir -p $DES - -echo "*** uBlock0.webext: copying common files" -bash ./tools/copy-common-files.sh $DES - -cp -R $DES/_locales/nb $DES/_locales/no - -cp platform/webext/manifest.json $DES/ - -# https://github.com/uBlockOrigin/uBlock-issues/issues/407 -echo "*** uBlock0.webext: concatenating vapi-webrequest.js" -cat platform/chromium/vapi-webrequest.js > /tmp/vapi-webrequest.js -grep -v "^'use strict';$" platform/firefox/vapi-webrequest.js >> /tmp/vapi-webrequest.js -mv /tmp/vapi-webrequest.js $DES/js/vapi-webrequest.js - -echo "*** uBlock0.webext: Generating meta..." -python3 tools/make-webext-meta.py $DES/ - -if [ "$1" = all ]; then - echo "*** uBlock0.webext: Creating package..." - pushd $DES > /dev/null - zip ../$(basename $DES).xpi -qr * - popd > /dev/null -elif [ -n "$1" ]; then - echo "*** uBlock0.webext: Creating versioned package..." - pushd $DES > /dev/null - zip ../$(basename $DES).xpi -qr * -O ../uBlock0_"$1".webext.xpi - popd > /dev/null -fi - -echo "*** uBlock0.webext: Package done."