From 22022f636fb179f32a9aa3cbe3c3335ba0a3d52f Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Sun, 25 Jul 2021 10:55:35 -0400 Subject: [PATCH] Modularize codebase with export/import Related issue: - https://github.com/uBlockOrigin/uBlock-issues/issues/1664 The changes are enough to fulfill the related issue. A new platform has been added in order to allow for building a NodeJS package. From the root of the project: ./tools/make-nodejs This will create new uBlock0.nodejs directory in the ./dist/build directory, which is a valid NodeJS package. From the root of the package, you can try: node test This will instantiate a static network filtering engine, populated by easylist and easyprivacy, which can be used to match network requests by filling the appropriate filtering context object. The test.js file contains code which is typical example of usage of the package. Limitations: the NodeJS package can't execute the WASM versions of the code since the WASM module requires the use of fetch(), which is not available in NodeJS. This is a first pass at modularizing the codebase, and while at it a number of opportunistic small rewrites have also been made. This commit requires the minimum supported version for Chromium and Firefox be raised to 61 and 60 respectively. --- dist/firefox/updates.json | 4 +- dist/version | 2 +- platform/browser/main.js | 125 +++ platform/browser/test.html | 71 ++ platform/chromium/manifest.json | 2 +- platform/common/vapi-background.js | 12 - platform/common/vapi-common.js | 53 -- platform/firefox/vapi-background-ext.js | 39 +- platform/nodejs/main.js | 127 +++ platform/nodejs/package.json | 25 + platform/nodejs/test.js | 107 +++ platform/opera/manifest.json | 2 +- platform/thunderbird/manifest.json | 2 +- src/1p-filters.html | 5 +- src/asset-viewer.html | 5 +- src/background.html | 65 +- src/dyna-rules.html | 6 +- src/js/1p-filters.js | 4 +- src/js/asset-viewer.js | 4 + src/js/assets.js | 10 +- src/js/background.js | 520 ++++++----- src/js/base64-custom.js | 246 ++++++ src/js/{strie.js => biditrie.js} | 37 +- src/js/cachestorage.js | 4 + src/js/codemirror/ubo-static-filtering.js | 14 +- src/js/commands.js | 9 +- src/js/contextmenu.js | 4 + src/js/cosmetic-filtering.js | 40 +- src/js/dyna-rules.js | 40 +- src/js/dynamic-net-filtering.js | 22 +- src/js/epicker-ui.js | 11 +- src/js/filtering-context.js | 113 +-- src/js/globals.js | 53 ++ src/js/hnswitches.js | 73 +- src/js/hntrie.js | 33 +- src/js/html-filtering.js | 837 +++++++++--------- src/js/httpheader-filtering.js | 15 +- src/js/logger-ui-inspector.js | 6 +- src/js/logger-ui.js | 11 +- src/js/logger.js | 117 +-- src/js/lz4.js | 10 +- src/js/messaging.js | 60 +- src/js/pagestore.js | 46 +- src/js/redirect-engine.js | 13 +- src/js/reverselookup.js | 12 +- src/js/scriptlet-filtering.js | 870 ++++++++++--------- src/js/start.js | 9 +- src/js/static-ext-filtering.js | 600 ++++++------- src/js/static-filtering-io.js | 147 ++++ src/js/static-filtering-parser.js | 32 +- src/js/static-net-filtering.js | 259 +++--- src/js/storage.js | 59 +- src/js/tab.js | 77 +- src/js/text-encode.js | 4 + src/js/text-iterators.js | 92 ++ src/js/traffic.js | 43 +- src/js/ublock.js | 20 +- src/js/uri-utils.js | 126 +++ src/js/uritools.js | 380 -------- src/js/url-net-filtering.js | 35 +- src/js/utils.js | 406 +-------- src/lib/publicsuffixlist/publicsuffixlist.js | 47 +- src/lib/punycode.js | 2 +- src/lib/regexanalyzer/regex.js | 10 +- src/logger-ui.html | 4 +- src/web_accessible_resources/epicker-ui.html | 5 +- tools/copy-common-files.sh | 6 +- tools/import-war.py | 92 -- tools/make-browser.sh | 35 + tools/make-chromium.sh | 6 +- tools/make-firefox.sh | 6 +- tools/make-nodejs.sh | 32 + tools/make-opera.sh | 6 +- tools/make-safari-meta.py | 77 -- tools/make-safari.sh | 33 - tools/make-thunderbird.sh | 2 +- tools/make-webext-meta.py | 39 - tools/make-webext.sh | 42 - 78 files changed, 3380 insertions(+), 3239 deletions(-) create mode 100644 platform/browser/main.js create mode 100644 platform/browser/test.html create mode 100644 platform/nodejs/main.js create mode 100644 platform/nodejs/package.json create mode 100644 platform/nodejs/test.js create mode 100644 src/js/base64-custom.js rename src/js/{strie.js => biditrie.js} (97%) create mode 100644 src/js/globals.js create mode 100644 src/js/static-filtering-io.js create mode 100644 src/js/text-iterators.js create mode 100644 src/js/uri-utils.js delete mode 100644 src/js/uritools.js delete mode 100755 tools/import-war.py create mode 100755 tools/make-browser.sh create mode 100755 tools/make-nodejs.sh delete mode 100644 tools/make-safari-meta.py delete mode 100755 tools/make-safari.sh delete mode 100644 tools/make-webext-meta.py delete mode 100755 tools/make-webext.sh diff --git a/dist/firefox/updates.json b/dist/firefox/updates.json index d23a8da59..63a7f9422 100644 --- a/dist/firefox/updates.json +++ b/dist/firefox/updates.json @@ -3,9 +3,9 @@ "uBlock0@raymondhill.net": { "updates": [ { - "version": "1.37.1.1", + "version": "1.37.1.2", "browser_specific_settings": { "gecko": { "strict_min_version": "57" } }, - "update_link": "https://github.com/gorhill/uBlock/releases/download/1.37.1b1/uBlock0_1.37.1b1.firefox.signed.xpi" + "update_link": "https://github.com/gorhill/uBlock/releases/download/1.37.1b2/uBlock0_1.37.1b2.firefox.signed.xpi" } ] } diff --git a/dist/version b/dist/version index 29ae45efb..e56539b5d 100644 --- a/dist/version +++ b/dist/version @@ -1 +1 @@ -1.37.1.1 +1.37.1.2 diff --git a/platform/browser/main.js b/platform/browser/main.js new file mode 100644 index 000000000..aaacfe9a2 --- /dev/null +++ b/platform/browser/main.js @@ -0,0 +1,125 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import './lib/publicsuffixlist/publicsuffixlist.js'; +import './lib/punycode.js'; + +import globals from './js/globals.js'; +import { FilteringContext } from './js/filtering-context.js'; +import { LineIterator } from './js/text-iterators.js'; +import { StaticFilteringParser } from './js/static-filtering-parser.js'; +import { staticNetFilteringEngine } from './js/static-net-filtering.js'; + +import { + CompiledListReader, + CompiledListWriter +} from './js/static-filtering-io.js'; + +/******************************************************************************/ + +function compileList(rawText, writer) { + const lineIter = new LineIterator(rawText); + const parser = new StaticFilteringParser(true); + + parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); + + while ( lineIter.eot() === false ) { + let line = lineIter.next(); + + while ( line.endsWith(' \\') ) { + if ( lineIter.peek(4) !== ' ' ) { break; } + line = line.slice(0, -2).trim() + lineIter.next().trim(); + } + parser.analyze(line); + + if ( parser.shouldIgnore() ) { continue; } + if ( parser.category !== parser.CATStaticNetFilter ) { continue; } + if ( parser.patternHasUnicode() && parser.toASCII() === false ) { + continue; + } + if ( staticNetFilteringEngine.compile(parser, writer) ) { continue; } + if ( staticNetFilteringEngine.error !== undefined ) { + console.info(JSON.stringify({ + realm: 'message', + type: 'error', + text: staticNetFilteringEngine.error + })); + } + } + + return writer.toString(); +} + +function applyList(name, raw) { + const writer = new CompiledListWriter(); + writer.properties.set('name', name); + const compiled = compileList(raw, writer); + const reader = new CompiledListReader(compiled); + staticNetFilteringEngine.fromCompiled(reader); +} + +function enableWASM(path) { + return Promise.all([ + globals.publicSuffixList.enableWASM(`${path}/lib/publicsuffixlist`), + staticNetFilteringEngine.enableWASM(`${path}/js`), + ]); +} + +function pslInit(raw) { + if ( typeof raw !== 'string' || raw.trim() === '' ) { + console.info('Unable to populate public suffix list'); + return; + } + globals.publicSuffixList.parse(raw, globals.punycode.toASCII); + console.info('Public suffix list populated'); +} + +function restart(lists) { + // Remove all filters + reset(); + + if ( Array.isArray(lists) && lists.length !== 0 ) { + // Populate filtering engine with filter lists + for ( const { name, raw } of lists ) { + applyList(name, raw); + } + // Commit changes + staticNetFilteringEngine.freeze(); + staticNetFilteringEngine.optimize(); + } + + return staticNetFilteringEngine; +} + +function reset() { + staticNetFilteringEngine.reset(); +} + +export { + FilteringContext, + enableWASM, + pslInit, + restart, +}; diff --git a/platform/browser/test.html b/platform/browser/test.html new file mode 100644 index 000000000..32b1aba8e --- /dev/null +++ b/platform/browser/test.html @@ -0,0 +1,71 @@ + + + + +uBO Static Network Filtering Engine + + + + + diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 9a515985a..f9ab901be 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -70,7 +70,7 @@ }, "incognito": "split", "manifest_version": 2, - "minimum_chrome_version": "55.0", + "minimum_chrome_version": "61.0", "name": "uBlock Origin", "options_ui": { "page": "dashboard.html", diff --git a/platform/common/vapi-background.js b/platform/common/vapi-background.js index 489f231fe..b9f7e2721 100644 --- a/platform/common/vapi-background.js +++ b/platform/common/vapi-background.js @@ -26,12 +26,6 @@ /******************************************************************************/ -{ -// >>>>> start of local scope - -/******************************************************************************/ -/******************************************************************************/ - const browser = self.browser; const manifest = browser.runtime.getManifest(); @@ -1719,9 +1713,3 @@ vAPI.cloud = (( ) => { })(); /******************************************************************************/ -/******************************************************************************/ - -// <<<<< end of local scope -} - -/******************************************************************************/ diff --git a/platform/common/vapi-common.js b/platform/common/vapi-common.js index e8de131d2..184260681 100644 --- a/platform/common/vapi-common.js +++ b/platform/common/vapi-common.js @@ -100,59 +100,6 @@ vAPI.webextFlavor = { /******************************************************************************/ -{ - const punycode = self.punycode; - const reCommonHostnameFromURL = /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//; - const reAuthorityFromURI = /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; - const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i; - const reHostFromAuthority = /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; - const reIPv6FromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; - const reMustNormalizeHostname = /[^0-9a-z._-]/; - - vAPI.hostnameFromURI = function(uri) { - let matches = reCommonHostnameFromURL.exec(uri); - if ( matches !== null ) { return matches[1]; } - matches = reAuthorityFromURI.exec(uri); - if ( matches === null ) { return ''; } - const authority = matches[1].slice(2); - if ( reHostFromNakedAuthority.test(authority) ) { - return authority.toLowerCase(); - } - matches = reHostFromAuthority.exec(authority); - if ( matches === null ) { - matches = reIPv6FromAuthority.exec(authority); - if ( matches === null ) { return ''; } - } - let hostname = matches[1]; - while ( hostname.endsWith('.') ) { - hostname = hostname.slice(0, -1); - } - if ( reMustNormalizeHostname.test(hostname) ) { - hostname = punycode.toASCII(hostname.toLowerCase()); - } - return hostname; - }; - - const reHostnameFromNetworkURL = - /^(?:http|ws|ftp)s?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])(?::\d+)?\//; - - vAPI.hostnameFromNetworkURL = function(url) { - const matches = reHostnameFromNetworkURL.exec(url); - return matches !== null ? matches[1] : ''; - }; - - const psl = self.publicSuffixList; - const reIPAddressNaive = /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; - - vAPI.domainFromHostname = function(hostname) { - return reIPAddressNaive.test(hostname) - ? hostname - : psl.getDomain(hostname); - }; -} - -/******************************************************************************/ - vAPI.download = function(details) { if ( !details.url ) { return; } const a = document.createElement('a'); diff --git a/platform/firefox/vapi-background-ext.js b/platform/firefox/vapi-background-ext.js index c083cabfe..69609f216 100644 --- a/platform/firefox/vapi-background-ext.js +++ b/platform/firefox/vapi-background-ext.js @@ -25,30 +25,17 @@ /******************************************************************************/ +import { + domainFromHostname, + hostnameFromNetworkURL, +} from './uri-utils.js'; + +/******************************************************************************/ + (( ) => { // https://github.com/uBlockOrigin/uBlock-issues/issues/407 if ( vAPI.webextFlavor.soup.has('firefox') === false ) { return; } - // https://github.com/gorhill/uBlock/issues/2950 - // Firefox 56 does not normalize URLs to ASCII, uBO must do this itself. - // https://bugzilla.mozilla.org/show_bug.cgi?id=945240 - const evalMustPunycode = ( ) => { - return vAPI.webextFlavor.soup.has('firefox') && - vAPI.webextFlavor.major < 57; - }; - - let mustPunycode = evalMustPunycode(); - - // The real actual webextFlavor value may not be set in stone, so listen - // for possible future changes. - window.addEventListener('webextFlavor', ( ) => { - mustPunycode = evalMustPunycode(); - }, { once: true }); - - const punycode = self.punycode; - const reAsciiHostname = /^https?:\/\/[0-9a-z_.:@-]+[/?#]/; - const parsedURL = new URL('about:blank'); - // Canonical name-uncloaking feature. let cnameUncloakEnabled = browser.dns instanceof Object; let cnameUncloakProxied = false; @@ -144,14 +131,6 @@ } } normalizeDetails(details) { - if ( mustPunycode && !reAsciiHostname.test(details.url) ) { - parsedURL.href = details.url; - details.url = details.url.replace( - parsedURL.hostname, - punycode.toASCII(parsedURL.hostname) - ); - } - const type = details.type; if ( type === 'imageset' ) { @@ -231,7 +210,7 @@ if ( cname !== '' && this.cnameIgnore1stParty && - vAPI.domainFromHostname(cname) === vAPI.domainFromHostname(hn) + domainFromHostname(cname) === domainFromHostname(hn) ) { cname = ''; } @@ -284,7 +263,7 @@ ) { return; } - const hn = vAPI.hostnameFromNetworkURL(details.url); + const hn = hostnameFromNetworkURL(details.url); const cname = this.cnames.get(hn); if ( cname === '' ) { return; } if ( cname !== undefined ) { diff --git a/platform/nodejs/main.js b/platform/nodejs/main.js new file mode 100644 index 000000000..a104fae84 --- /dev/null +++ b/platform/nodejs/main.js @@ -0,0 +1,127 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import './lib/punycode.js'; +import './lib/publicsuffixlist/publicsuffixlist.js'; + +import globals from './js/globals.js'; +import { FilteringContext } from './js/filtering-context.js'; +import { LineIterator } from './js/text-iterators.js'; +import { StaticFilteringParser } from './js/static-filtering-parser.js'; +import { staticNetFilteringEngine } from './js/static-net-filtering.js'; + +import { + CompiledListReader, + CompiledListWriter, +} from './js/static-filtering-io.js'; + +/******************************************************************************/ + +function compileList(rawText, writer) { + const lineIter = new LineIterator(rawText); + const parser = new StaticFilteringParser(true); + + parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); + + while ( lineIter.eot() === false ) { + let line = lineIter.next(); + + while ( line.endsWith(' \\') ) { + if ( lineIter.peek(4) !== ' ' ) { break; } + line = line.slice(0, -2).trim() + lineIter.next().trim(); + } + parser.analyze(line); + + if ( parser.shouldIgnore() ) { continue; } + if ( parser.category !== parser.CATStaticNetFilter ) { continue; } + if ( parser.patternHasUnicode() && parser.toASCII() === false ) { + continue; + } + if ( staticNetFilteringEngine.compile(parser, writer) ) { continue; } + if ( staticNetFilteringEngine.error !== undefined ) { + console.info(JSON.stringify({ + realm: 'message', + type: 'error', + text: staticNetFilteringEngine.error + })); + } + } + + return writer.toString(); +} + +function applyList(name, raw) { + const writer = new CompiledListWriter(); + writer.properties.set('name', name); + const compiled = compileList(raw, writer); + const reader = new CompiledListReader(compiled); + staticNetFilteringEngine.fromCompiled(reader); +} + +function enableWASM(path) { + return Promise.all([ + globals.publicSuffixList.enableWASM(`${path}/lib/publicsuffixlist`), + staticNetFilteringEngine.enableWASM(`${path}/js`), + ]); +} + +function pslInit(raw) { + if ( typeof raw !== 'string' || raw.trim() === '' ) { + console.info('Unable to populate public suffix list'); + return; + } + globals.publicSuffixList.parse(raw, globals.punycode.toASCII); + console.info('Public suffix list populated'); +} + +function restart(lists) { + // Remove all filters + reset(); + + if ( Array.isArray(lists) && lists.length !== 0 ) { + // Populate filtering engine with filter lists + for ( const { name, raw } of lists ) { + applyList(name, raw); + } + // Commit changes + staticNetFilteringEngine.freeze(); + staticNetFilteringEngine.optimize(); + } + + console.info('Static network filtering engine populated'); + + return staticNetFilteringEngine; +} + +function reset() { + staticNetFilteringEngine.reset(); +} + +export { + FilteringContext, + enableWASM, + pslInit, + restart, +}; diff --git a/platform/nodejs/package.json b/platform/nodejs/package.json new file mode 100644 index 000000000..37dbc7755 --- /dev/null +++ b/platform/nodejs/package.json @@ -0,0 +1,25 @@ +{ + "name": "uBO-snfe", + "version": "0.1.0", + "description": "To create a working instance of uBlock's static network filtering engine", + "type": "module", + "main": "main.js", + "scripts": { + "test": "node test.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/gorhill/uBlock.git" + }, + "keywords": [ + "uBlock", + "uBO", + "adblock" + ], + "author": "Raymond Hill", + "license": "GPL-3.0-or-later", + "bugs": { + "url": "https://github.com/gorhill/uBlock/issues" + }, + "homepage": "https://github.com/gorhill/uBlock#readme" +} diff --git a/platform/nodejs/test.js b/platform/nodejs/test.js new file mode 100644 index 000000000..0d544e9d1 --- /dev/null +++ b/platform/nodejs/test.js @@ -0,0 +1,107 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +/* globals process */ + +'use strict'; + +/******************************************************************************/ + +import { readFile } from 'fs'; + +import { + FilteringContext, + pslInit, + restart, +} from './main.js'; + +/******************************************************************************/ + +function fetch(path) { + return new Promise((resolve, reject) => { + readFile(path, 'utf8', (err, data) => { + if ( err ) { + reject(err); + } else { + resolve(data); + } + }); + }); +} + +(async ( ) => { + /* + * WASM require fetch(), not present in Node + try { + await enableWASM('//ublock/dist/build/uBlock0.nodejs'); + } catch(ex) { + } + */ + + await fetch('./data/effective_tld_names.dat').then(pslRaw => { + pslInit(pslRaw); + }); + + const snfe = await Promise.all([ + fetch('./data/easylist.txt'), + fetch('./data/easyprivacy.txt'), + ]).then(rawLists => { + return restart([ + { name: 'easylist', raw: rawLists[0] }, + { name: 'easyprivacy', raw: rawLists[1] }, + ]); + }); + + // Reuse filtering context: it's what uBO does + const fctxt = new FilteringContext(); + + // Tests + // Not blocked + fctxt.setDocOriginFromURL('https://www.bloomberg.com/'); + fctxt.setURL('https://www.bloomberg.com/tophat/assets/v2.6.1/that.css'); + fctxt.setType('stylesheet'); + if ( snfe.matchRequest(fctxt) !== 0 ) { + console.log(snfe.toLogData()); + } + + // Blocked + fctxt.setDocOriginFromURL('https://www.bloomberg.com/'); + fctxt.setURL('https://securepubads.g.doubleclick.net/tag/js/gpt.js'); + fctxt.setType('script'); + if ( snfe.matchRequest(fctxt) !== 0 ) { + console.log(snfe.toLogData()); + } + + // Unblocked + fctxt.setDocOriginFromURL('https://www.bloomberg.com/'); + fctxt.setURL('https://sourcepointcmp.bloomberg.com/ccpa.js'); + fctxt.setType('script'); + if ( snfe.matchRequest(fctxt) !== 0 ) { + console.log(snfe.toLogData()); + } + + // Remove all filters + restart(); + + process.exit(); +})(); + +/******************************************************************************/ diff --git a/platform/opera/manifest.json b/platform/opera/manifest.json index 69af37e96..2545566ee 100644 --- a/platform/opera/manifest.json +++ b/platform/opera/manifest.json @@ -69,7 +69,7 @@ }, "incognito": "split", "manifest_version": 2, - "minimum_opera_version": "42.0", + "minimum_opera_version": "48.0", "name": "uBlock Origin", "options_page": "dashboard.html", "permissions": [ diff --git a/platform/thunderbird/manifest.json b/platform/thunderbird/manifest.json index a0762a129..3f909edbe 100644 --- a/platform/thunderbird/manifest.json +++ b/platform/thunderbird/manifest.json @@ -2,7 +2,7 @@ "applications": { "gecko": { "id": "uBlock0@raymondhill.net", - "strict_min_version": "65.0" + "strict_min_version": "78.0" } }, "author": "Raymond Hill & contributors", diff --git a/src/1p-filters.html b/src/1p-filters.html index 26fce7f6e..29d615501 100644 --- a/src/1p-filters.html +++ b/src/1p-filters.html @@ -50,11 +50,9 @@ - - @@ -65,8 +63,7 @@ - - + diff --git a/src/asset-viewer.html b/src/asset-viewer.html index d94fc0bdc..126cbc6e4 100644 --- a/src/asset-viewer.html +++ b/src/asset-viewer.html @@ -33,11 +33,9 @@ - - @@ -46,8 +44,7 @@ - - + diff --git a/src/background.html b/src/background.html index 078824442..21429406f 100644 --- a/src/background.html +++ b/src/background.html @@ -7,45 +7,36 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/dyna-rules.html b/src/dyna-rules.html index bb9b28260..752f997f4 100644 --- a/src/dyna-rules.html +++ b/src/dyna-rules.html @@ -51,10 +51,6 @@ - - - - @@ -64,7 +60,7 @@ - + diff --git a/src/js/1p-filters.js b/src/js/1p-filters.js index 86a5b707d..e52beaedf 100644 --- a/src/js/1p-filters.js +++ b/src/js/1p-filters.js @@ -25,7 +25,7 @@ /******************************************************************************/ -(( ) => { +import './codemirror/ubo-static-filtering.js'; /******************************************************************************/ @@ -352,5 +352,3 @@ cmEditor.on('changes', userFiltersChanged); CodeMirror.commands.save = applyChanges; /******************************************************************************/ - -})(); diff --git a/src/js/asset-viewer.js b/src/js/asset-viewer.js index ecdef01c2..ef0824d6d 100644 --- a/src/js/asset-viewer.js +++ b/src/js/asset-viewer.js @@ -25,6 +25,10 @@ /******************************************************************************/ +import './codemirror/ubo-static-filtering.js'; + +/******************************************************************************/ + (async ( ) => { const subscribeURL = new URL(document.location); const subscribeParams = subscribeURL.searchParams; diff --git a/src/js/assets.js b/src/js/assets.js index f63596efb..d7e144e55 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -23,7 +23,7 @@ /******************************************************************************/ -µBlock.assets = (( ) => { +import µBlock from './background.js'; /******************************************************************************/ @@ -1048,10 +1048,8 @@ api.isUpdating = function() { /******************************************************************************/ -return api; - -/******************************************************************************/ - -})(); +// Export + +µBlock.assets = api; /******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js index 0139b2286..b6794f5ac 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -19,212 +19,340 @@ Home: https://github.com/gorhill/uBlock */ - 'use strict'; /******************************************************************************/ +import globals from './globals.js'; + +import { + domainFromHostname, + hostnameFromURI, + originFromURI, +} from './uri-utils.js'; + +import { FilteringContext } from './filtering-context.js'; +import { CompiledListWriter } from './static-filtering-io.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; +import { staticNetFilteringEngine } from './static-net-filtering.js'; + +/******************************************************************************/ + // Not all platforms may have properly declared vAPI.webextFlavor. if ( vAPI.webextFlavor === undefined ) { vAPI.webextFlavor = { major: 0, soup: new Set([ 'ublock' ]) }; } +/******************************************************************************/ + +const hiddenSettingsDefault = { + allowGenericProceduralFilters: false, + assetFetchTimeout: 30, + autoCommentFilterTemplate: '{{date}} {{origin}}', + autoUpdateAssetFetchPeriod: 120, + autoUpdateDelayAfterLaunch: 180, + autoUpdatePeriod: 4, + benchmarkDatasetURL: 'unset', + blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', + cacheStorageAPI: 'unset', + cacheStorageCompression: true, + cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', + cloudStorageCompression: true, + cnameIgnoreList: 'unset', + cnameIgnore1stParty: true, + cnameIgnoreExceptions: true, + cnameIgnoreRootDocument: true, + cnameMaxTTL: 120, + cnameReplayFullURL: false, + cnameUncloak: true, + cnameUncloakProxied: false, + consoleLogLevel: 'unset', + debugScriptlets: false, + debugScriptletInjector: false, + disableWebAssembly: false, + extensionUpdateForceReload: false, + filterAuthorMode: false, + filterOnHeaders: false, + loggerPopupType: 'popup', + manualUpdateAssetFetchPeriod: 500, + popupFontSize: 'unset', + popupPanelDisabledSections: 0, + popupPanelLockedSections: 0, + popupPanelHeightMode: 0, + requestJournalProcessPeriod: 1000, + selfieAfter: 3, + strictBlockingBypassDuration: 120, + suspendTabsUntilReady: 'unset', + uiPopupConfig: 'undocumented', + uiFlavor: 'unset', + uiStyles: 'unset', + uiTheme: 'unset', + updateAssetBypassBrowserCache: false, + userResourcesLocation: 'unset', +}; + +const userSettingsDefault = { + advancedUserEnabled: false, + alwaysDetachLogger: true, + autoUpdate: true, + cloudStorageEnabled: false, + cnameUncloakEnabled: true, + collapseBlocked: true, + colorBlindFriendly: false, + contextMenuEnabled: true, + dynamicFilteringEnabled: false, + externalLists: '', + firewallPaneMinimized: true, + hyperlinkAuditingDisabled: true, + ignoreGenericCosmeticFilters: vAPI.webextFlavor.soup.has('mobile'), + importedLists: [], + largeMediaSize: 50, + parseAllABPHideFilters: true, + popupPanelSections: 0b111, + prefetchingDisabled: true, + requestLogMaxEntries: 1000, + showIconBadge: true, + tooltipsDisabled: false, + webrtcIPAddressHidden: false, +}; + +const µBlock = { // jshint ignore:line + userSettingsDefault: userSettingsDefault, + userSettings: Object.assign({}, userSettingsDefault), + + hiddenSettingsDefault: hiddenSettingsDefault, + hiddenSettingsAdmin: {}, + hiddenSettings: Object.assign({}, hiddenSettingsDefault), + + noDashboard: false, + + // Features detection. + privacySettingsSupported: vAPI.browserSettings instanceof Object, + cloudStorageSupported: vAPI.cloud instanceof Object, + canFilterResponseData: typeof browser.webRequest.filterResponseData === 'function', + canInjectScriptletsNow: vAPI.webextFlavor.soup.has('chromium'), + + // https://github.com/chrisaljoudi/uBlock/issues/180 + // Whitelist directives need to be loaded once the PSL is available + netWhitelist: new Map(), + netWhitelistModifyTime: 0, + netWhitelistDefault: [ + 'about-scheme', + 'chrome-extension-scheme', + 'chrome-scheme', + 'edge-scheme', + 'moz-extension-scheme', + 'opera-scheme', + 'vivaldi-scheme', + 'wyciwyg-scheme', // Firefox's "What-You-Cache-Is-What-You-Get" + ], + + localSettings: { + blockedRequestCount: 0, + allowedRequestCount: 0, + }, + localSettingsLastModified: 0, + localSettingsLastSaved: 0, + + // Read-only + systemSettings: { + compiledMagic: 37, // Increase when compiled format changes + selfieMagic: 37, // Increase when selfie format changes + }, + + // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501 + // The assumption is that cache storage state reflects whether + // compiled or selfie assets are available or not. The properties + // below is to no longer rely on this assumption -- though it's still + // not clear how the assumption could be wrong, and it's still not + // clear whether relying on those properties will really solve the + // issue. It's just an attempt at hardening. + compiledFormatChanged: false, + selfieIsInvalid: false, + + compiledCosmeticSection: 200, + compiledScriptletSection: 300, + compiledHTMLSection: 400, + compiledHTTPHeaderSection: 500, + compiledSentinelSection: 1000, + compiledBadSubsection: 1, + + restoreBackupSettings: { + lastRestoreFile: '', + lastRestoreTime: 0, + lastBackupFile: '', + lastBackupTime: 0, + }, + + commandShortcuts: new Map(), + + // Allows to fully customize uBO's assets, typically set through admin + // settings. The content of 'assets.json' will also tell which filter + // lists to enable by default when uBO is first installed. + assetsBootstrapLocation: undefined, + + userFiltersPath: 'user-filters', + pslAssetKey: 'public_suffix_list.dat', + + selectedFilterLists: [], + availableFilterLists: {}, + badLists: new Map(), + + // https://github.com/uBlockOrigin/uBlock-issues/issues/974 + // This can be used to defer filtering decision-making. + readyToFilter: false, + + pageStores: new Map(), + pageStoresToken: 0, + + storageQuota: vAPI.storage.QUOTA_BYTES, + storageUsed: 0, + + noopFunc: function(){}, + + apiErrorCount: 0, + + maybeGoodPopup: { + tabId: 0, + url: '', + }, + + epickerArgs: { + eprom: null, + mouse: false, + target: '', + zap: false, + }, + + scriptlets: {}, + + cspNoInlineScript: "script-src 'unsafe-eval' * blob: data:", + cspNoScripting: 'script-src http: https:', + cspNoInlineFont: 'font-src *', + + liveBlockingProfiles: [], + blockingProfileColorCache: new Map(), +}; + +µBlock.domainFromHostname = domainFromHostname; +µBlock.hostnameFromURI = hostnameFromURI; + +µBlock.FilteringContext = class extends FilteringContext { + duplicate() { + return (new µBlock.FilteringContext(this)); + } + + fromTabId(tabId) { + const tabContext = µBlock.tabContextManager.mustLookup(tabId); + this.tabOrigin = tabContext.origin; + this.tabHostname = tabContext.rootHostname; + this.tabDomain = tabContext.rootDomain; + this.tabId = tabContext.tabId; + return this; + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/459 + // In case of a request for frame and if ever no context is specified, + // assume the origin of the context is the same as the request itself. + fromWebrequestDetails(details) { + const tabId = details.tabId; + this.type = details.type; + if ( this.itype === this.MAIN_FRAME && tabId > 0 ) { + µBlock.tabContextManager.push(tabId, details.url); + } + this.fromTabId(tabId); // Must be called AFTER tab context management + this.realm = ''; + this.id = details.requestId; + this.setURL(details.url); + this.aliasURL = details.aliasURL || undefined; + if ( this.itype !== this.SUB_FRAME ) { + this.docId = details.frameId; + this.frameId = -1; + } else { + this.docId = details.parentFrameId; + this.frameId = details.frameId; + } + if ( this.tabId > 0 ) { + if ( this.docId === 0 ) { + this.docOrigin = this.tabOrigin; + this.docHostname = this.tabHostname; + this.docDomain = this.tabDomain; + } else if ( details.documentUrl !== undefined ) { + this.setDocOriginFromURL(details.documentUrl); + } else { + const pageStore = µBlock.pageStoreFromTabId(this.tabId); + const docStore = pageStore && pageStore.getFrameStore(this.docId); + if ( docStore ) { + this.setDocOriginFromURL(docStore.rawURL); + } else { + this.setDocOrigin(this.tabOrigin); + } + } + } else if ( details.documentUrl !== undefined ) { + const origin = originFromURI( + µBlock.normalizeTabURL(0, details.documentUrl) + ); + this.setDocOrigin(origin).setTabOrigin(origin); + } else if ( this.docId === -1 || (this.itype & this.FRAME_ANY) !== 0 ) { + const origin = originFromURI(this.url); + this.setDocOrigin(origin).setTabOrigin(origin); + } else { + this.setDocOrigin(this.tabOrigin); + } + this.redirectURL = undefined; + this.filter = undefined; + return this; + } + + getTabOrigin() { + if ( this.tabOrigin === undefined ) { + const tabContext = µBlock.tabContextManager.mustLookup(this.tabId); + this.tabOrigin = tabContext.origin; + this.tabHostname = tabContext.rootHostname; + this.tabDomain = tabContext.rootDomain; + } + return super.getTabOrigin(); + } + + getTabHostname() { + if ( this.tabHostname === undefined ) { + this.tabHostname = hostnameFromURI(this.getTabOrigin()); + } + return super.getTabHostname(); + } + + toLogger() { + this.tstamp = Date.now(); + if ( this.domain === undefined ) { + void this.getDomain(); + } + if ( this.docDomain === undefined ) { + void this.getDocDomain(); + } + if ( this.tabDomain === undefined ) { + void this.getTabDomain(); + } + const logger = µBlock.logger; + const filters = this.filter; + // Many filters may have been applied to the current context + if ( Array.isArray(filters) === false ) { + return logger.writeOne(this); + } + for ( const filter of filters ) { + this.filter = filter; + logger.writeOne(this); + } + } +}; + +µBlock.filteringContext = new µBlock.FilteringContext(); +µBlock.CompiledListWriter = CompiledListWriter; +µBlock.StaticFilteringParser = StaticFilteringParser; +µBlock.staticNetFilteringEngine = staticNetFilteringEngine; + +globals.µBlock = µBlock; /******************************************************************************/ -const µBlock = (( ) => { // jshint ignore:line - - const hiddenSettingsDefault = { - allowGenericProceduralFilters: false, - assetFetchTimeout: 30, - autoCommentFilterTemplate: '{{date}} {{origin}}', - autoUpdateAssetFetchPeriod: 120, - autoUpdateDelayAfterLaunch: 180, - autoUpdatePeriod: 4, - benchmarkDatasetURL: 'unset', - blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001', - cacheStorageAPI: 'unset', - cacheStorageCompression: true, - cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate', - cloudStorageCompression: true, - cnameIgnoreList: 'unset', - cnameIgnore1stParty: true, - cnameIgnoreExceptions: true, - cnameIgnoreRootDocument: true, - cnameMaxTTL: 120, - cnameReplayFullURL: false, - cnameUncloak: true, - cnameUncloakProxied: false, - consoleLogLevel: 'unset', - debugScriptlets: false, - debugScriptletInjector: false, - disableWebAssembly: false, - extensionUpdateForceReload: false, - filterAuthorMode: false, - filterOnHeaders: false, - loggerPopupType: 'popup', - manualUpdateAssetFetchPeriod: 500, - popupFontSize: 'unset', - popupPanelDisabledSections: 0, - popupPanelLockedSections: 0, - popupPanelHeightMode: 0, - requestJournalProcessPeriod: 1000, - selfieAfter: 3, - strictBlockingBypassDuration: 120, - suspendTabsUntilReady: 'unset', - uiPopupConfig: 'undocumented', - uiFlavor: 'unset', - uiStyles: 'unset', - uiTheme: 'unset', - updateAssetBypassBrowserCache: false, - userResourcesLocation: 'unset', - }; - - const userSettingsDefault = { - advancedUserEnabled: false, - alwaysDetachLogger: true, - autoUpdate: true, - cloudStorageEnabled: false, - cnameUncloakEnabled: true, - collapseBlocked: true, - colorBlindFriendly: false, - contextMenuEnabled: true, - dynamicFilteringEnabled: false, - externalLists: '', - firewallPaneMinimized: true, - hyperlinkAuditingDisabled: true, - ignoreGenericCosmeticFilters: vAPI.webextFlavor.soup.has('mobile'), - importedLists: [], - largeMediaSize: 50, - parseAllABPHideFilters: true, - popupPanelSections: 0b111, - prefetchingDisabled: true, - requestLogMaxEntries: 1000, - showIconBadge: true, - tooltipsDisabled: false, - webrtcIPAddressHidden: false, - }; - - return { - userSettingsDefault: userSettingsDefault, - userSettings: Object.assign({}, userSettingsDefault), - - hiddenSettingsDefault: hiddenSettingsDefault, - hiddenSettingsAdmin: {}, - hiddenSettings: Object.assign({}, hiddenSettingsDefault), - - noDashboard: false, - - // Features detection. - privacySettingsSupported: vAPI.browserSettings instanceof Object, - cloudStorageSupported: vAPI.cloud instanceof Object, - canFilterResponseData: typeof browser.webRequest.filterResponseData === 'function', - canInjectScriptletsNow: vAPI.webextFlavor.soup.has('chromium'), - - // https://github.com/chrisaljoudi/uBlock/issues/180 - // Whitelist directives need to be loaded once the PSL is available - netWhitelist: new Map(), - netWhitelistModifyTime: 0, - netWhitelistDefault: [ - 'about-scheme', - 'chrome-extension-scheme', - 'chrome-scheme', - 'edge-scheme', - 'moz-extension-scheme', - 'opera-scheme', - 'vivaldi-scheme', - 'wyciwyg-scheme', // Firefox's "What-You-Cache-Is-What-You-Get" - ], - - localSettings: { - blockedRequestCount: 0, - allowedRequestCount: 0, - }, - localSettingsLastModified: 0, - localSettingsLastSaved: 0, - - // Read-only - systemSettings: { - compiledMagic: 37, // Increase when compiled format changes - selfieMagic: 37, // Increase when selfie format changes - }, - - // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501 - // The assumption is that cache storage state reflects whether - // compiled or selfie assets are available or not. The properties - // below is to no longer rely on this assumption -- though it's still - // not clear how the assumption could be wrong, and it's still not - // clear whether relying on those properties will really solve the - // issue. It's just an attempt at hardening. - compiledFormatChanged: false, - selfieIsInvalid: false, - - compiledNetworkSection: 100, - compiledCosmeticSection: 200, - compiledScriptletSection: 300, - compiledHTMLSection: 400, - compiledHTTPHeaderSection: 500, - compiledSentinelSection: 1000, - compiledBadSubsection: 1, - - restoreBackupSettings: { - lastRestoreFile: '', - lastRestoreTime: 0, - lastBackupFile: '', - lastBackupTime: 0, - }, - - commandShortcuts: new Map(), - - // Allows to fully customize uBO's assets, typically set through admin - // settings. The content of 'assets.json' will also tell which filter - // lists to enable by default when uBO is first installed. - assetsBootstrapLocation: undefined, - - userFiltersPath: 'user-filters', - pslAssetKey: 'public_suffix_list.dat', - - selectedFilterLists: [], - availableFilterLists: {}, - badLists: new Map(), - - // https://github.com/uBlockOrigin/uBlock-issues/issues/974 - // This can be used to defer filtering decision-making. - readyToFilter: false, - - pageStores: new Map(), - pageStoresToken: 0, - - storageQuota: vAPI.storage.QUOTA_BYTES, - storageUsed: 0, - - noopFunc: function(){}, - - apiErrorCount: 0, - - maybeGoodPopup: { - tabId: 0, - url: '', - }, - - epickerArgs: { - eprom: null, - mouse: false, - target: '', - zap: false, - }, - - scriptlets: {}, - - cspNoInlineScript: "script-src 'unsafe-eval' * blob: data:", - cspNoScripting: 'script-src http: https:', - cspNoInlineFont: 'font-src *', - - liveBlockingProfiles: [], - blockingProfileColorCache: new Map(), - }; - -})(); - -/******************************************************************************/ +export default µBlock; diff --git a/src/js/base64-custom.js b/src/js/base64-custom.js new file mode 100644 index 000000000..fab6c08c6 --- /dev/null +++ b/src/js/base64-custom.js @@ -0,0 +1,246 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// Custom base64 codecs. These codecs are meant to encode/decode typed arrays +// to/from strings. + +// https://github.com/uBlockOrigin/uBlock-issues/issues/461 +// Provide a fallback encoding for Chromium 59 and less by issuing a plain +// JSON string. The fallback can be removed once min supported version is +// above 59. + +// TODO: rename µBlock.base64 to µBlock.SparseBase64, now that +// µBlock.DenseBase64 has been introduced. +// TODO: Should no longer need to test presence of TextEncoder/TextDecoder. + +const valToDigit = new Uint8Array(64); +const digitToVal = new Uint8Array(128); +{ + const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@%'; + for ( let i = 0, n = chars.length; i < n; i++ ) { + const c = chars.charCodeAt(i); + valToDigit[i] = c; + digitToVal[c] = i; + } +} + +// The sparse base64 codec is best for buffers which contains a lot of +// small u32 integer values. Those small u32 integer values are better +// represented with stringified integers, because small values can be +// represented with fewer bits than the usual base64 codec. For example, +// 0 become '0 ', i.e. 16 bits instead of 48 bits with official base64 +// codec. + +const sparseBase64 = { + magic: 'Base64_1', + + encode: function(arrbuf, arrlen) { + const inputLength = (arrlen + 3) >>> 2; + const inbuf = new Uint32Array(arrbuf, 0, inputLength); + const outputLength = this.magic.length + 7 + inputLength * 7; + const outbuf = new Uint8Array(outputLength); + // magic bytes + let j = 0; + for ( let i = 0; i < this.magic.length; i++ ) { + outbuf[j++] = this.magic.charCodeAt(i); + } + // array size + let v = inputLength; + do { + outbuf[j++] = valToDigit[v & 0b111111]; + v >>>= 6; + } while ( v !== 0 ); + outbuf[j++] = 0x20 /* ' ' */; + // array content + for ( let i = 0; i < inputLength; i++ ) { + v = inbuf[i]; + do { + outbuf[j++] = valToDigit[v & 0b111111]; + v >>>= 6; + } while ( v !== 0 ); + outbuf[j++] = 0x20 /* ' ' */; + } + if ( typeof TextDecoder === 'undefined' ) { + return JSON.stringify( + Array.from(new Uint32Array(outbuf.buffer, 0, j >>> 2)) + ); + } + const textDecoder = new TextDecoder(); + return textDecoder.decode(new Uint8Array(outbuf.buffer, 0, j)); + }, + + decode: function(instr, arrbuf) { + if ( instr.charCodeAt(0) === 0x5B /* '[' */ ) { + const inbuf = JSON.parse(instr); + if ( arrbuf instanceof ArrayBuffer === false ) { + return new Uint32Array(inbuf); + } + const outbuf = new Uint32Array(arrbuf); + outbuf.set(inbuf); + return outbuf; + } + if ( instr.startsWith(this.magic) === false ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + const inputLength = instr.length; + const outputLength = this.decodeSize(instr) >> 2; + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint32Array(outputLength) + : new Uint32Array(arrbuf); + let i = instr.indexOf(' ', this.magic.length) + 1; + if ( i === -1 ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + // array content + let j = 0; + for (;;) { + if ( j === outputLength || i >= inputLength ) { break; } + let v = 0, l = 0; + for (;;) { + const c = instr.charCodeAt(i++); + if ( c === 0x20 /* ' ' */ ) { break; } + v += digitToVal[c] << l; + l += 6; + } + outbuf[j++] = v; + } + if ( i < inputLength || j < outputLength ) { + throw new Error('Invalid µBlock.base64 encoding'); + } + return outbuf; + }, + + decodeSize: function(instr) { + if ( instr.startsWith(this.magic) === false ) { return 0; } + let v = 0, l = 0, i = this.magic.length; + for (;;) { + const c = instr.charCodeAt(i++); + if ( c === 0x20 /* ' ' */ ) { break; } + v += digitToVal[c] << l; + l += 6; + } + return v << 2; + }, +}; + +// The dense base64 codec is best for typed buffers which values are +// more random. For example, buffer contents as a result of compression +// contain less repetitive values and thus the content is more +// random-looking. + +// TODO: Investigate that in Firefox, creating a new Uint8Array from the +// ArrayBuffer fails, the content of the resulting Uint8Array is +// non-sensical. WASM-related? + +const denseBase64 = { + magic: 'DenseBase64_1', + + encode: function(input) { + const m = input.length % 3; + const n = input.length - m; + let outputLength = n / 3 * 4; + if ( m !== 0 ) { + outputLength += m + 1; + } + const output = new Uint8Array(outputLength); + let j = 0; + for ( let i = 0; i < n; i += 3) { + const i1 = input[i+0]; + const i2 = input[i+1]; + const i3 = input[i+2]; + output[j+0] = valToDigit[ i1 >>> 2]; + output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; + output[j+2] = valToDigit[i2 << 2 & 0b111100 | i3 >>> 6]; + output[j+3] = valToDigit[i3 & 0b111111 ]; + j += 4; + } + if ( m !== 0 ) { + const i1 = input[n]; + output[j+0] = valToDigit[i1 >>> 2]; + if ( m === 1 ) { // 1 value + output[j+1] = valToDigit[i1 << 4 & 0b110000]; + } else { // 2 values + const i2 = input[n+1]; + output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; + output[j+2] = valToDigit[i2 << 2 & 0b111100 ]; + } + } + const textDecoder = new TextDecoder(); + const b64str = textDecoder.decode(output); + return this.magic + b64str; + }, + + decode: function(instr, arrbuf) { + if ( instr.startsWith(this.magic) === false ) { + throw new Error('Invalid µBlock.denseBase64 encoding'); + } + const outputLength = this.decodeSize(instr); + const outbuf = arrbuf instanceof ArrayBuffer === false + ? new Uint8Array(outputLength) + : new Uint8Array(arrbuf); + const inputLength = instr.length - this.magic.length; + let i = this.magic.length; + let j = 0; + const m = inputLength & 3; + const n = i + inputLength - m; + while ( i < n ) { + const i1 = digitToVal[instr.charCodeAt(i+0)]; + const i2 = digitToVal[instr.charCodeAt(i+1)]; + const i3 = digitToVal[instr.charCodeAt(i+2)]; + const i4 = digitToVal[instr.charCodeAt(i+3)]; + i += 4; + outbuf[j+0] = i1 << 2 | i2 >>> 4; + outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; + outbuf[j+2] = i3 << 6 & 0b11000000 | i4; + j += 3; + } + if ( m !== 0 ) { + const i1 = digitToVal[instr.charCodeAt(i+0)]; + const i2 = digitToVal[instr.charCodeAt(i+1)]; + outbuf[j+0] = i1 << 2 | i2 >>> 4; + if ( m === 3 ) { + const i3 = digitToVal[instr.charCodeAt(i+2)]; + outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; + } + } + return outbuf; + }, + + decodeSize: function(instr) { + if ( instr.startsWith(this.magic) === false ) { return 0; } + const inputLength = instr.length - this.magic.length; + const m = inputLength & 3; + const n = inputLength - m; + let outputLength = (n >>> 2) * 3; + if ( m !== 0 ) { + outputLength += m - 1; + } + return outputLength; + }, +}; + +/******************************************************************************/ + +export { denseBase64, sparseBase64 }; diff --git a/src/js/strie.js b/src/js/biditrie.js similarity index 97% rename from src/js/strie.js rename to src/js/biditrie.js index 9c26af224..78dc7754c 100644 --- a/src/js/strie.js +++ b/src/js/biditrie.js @@ -23,11 +23,6 @@ 'use strict'; -// ***************************************************************************** -// start of local namespace - -{ - /******************************************************************************* A BidiTrieContainer is mostly a large buffer in which distinct but related @@ -130,7 +125,7 @@ const toSegmentInfo = (aL, l, r) => ((r - l) << 24) | (aL + l); const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); -µBlock.BidiTrieContainer = class { +const BidiTrieContainer = class { constructor(extraHandler) { const len = PAGE_SIZE * 4; @@ -697,10 +692,10 @@ const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); return -1; } - async enableWASM() { + async enableWASM(modulePath) { if ( typeof WebAssembly !== 'object' ) { return false; } if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; } - const module = await getWasmModule(); + const module = await getWasmModule(modulePath); if ( module instanceof WebAssembly.Module === false ) { return false; } const memory = new WebAssembly.Memory({ initial: roundToPageSize(this.buf8.length) >>> 16 @@ -828,7 +823,7 @@ const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); */ -µBlock.BidiTrieContainer.prototype.STrieRef = class { +BidiTrieContainer.prototype.STrieRef = class { constructor(container, iroot, size) { this.container = container; this.iroot = iroot; @@ -930,20 +925,7 @@ const roundToPageSize = v => (v + PAGE_SIZE-1) & ~(PAGE_SIZE-1); const getWasmModule = (( ) => { let wasmModulePromise; - // The directory from which the current script was fetched should also - // contain the related WASM file. The script is fetched from a trusted - // location, and consequently so will be the related WASM file. - let workingDir; - { - const url = new URL(document.currentScript.src); - const match = /[^\/]+$/.exec(url.pathname); - if ( match !== null ) { - url.pathname = url.pathname.slice(0, match.index); - } - workingDir = url.href; - } - - return async function() { + return async function(modulePath) { if ( wasmModulePromise instanceof Promise ) { return wasmModulePromise; } @@ -967,19 +949,18 @@ const getWasmModule = (( ) => { if ( uint8s[0] !== 1 ) { return; } wasmModulePromise = fetch( - workingDir + 'wasm/biditrie.wasm', + `${modulePath}/wasm/biditrie.wasm`, { mode: 'same-origin' } ).then( WebAssembly.compileStreaming ).catch(reason => { - log.info(reason); + console.info(reason); }); return wasmModulePromise; }; })(); -// end of local namespace -// ***************************************************************************** +/******************************************************************************/ -} +export { BidiTrieContainer }; diff --git a/src/js/cachestorage.js b/src/js/cachestorage.js index 08e948e32..85cd9c0ac 100644 --- a/src/js/cachestorage.js +++ b/src/js/cachestorage.js @@ -25,6 +25,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + // The code below has been originally manually imported from: // Commit: https://github.com/nikrolls/uBlock-Edge/commit/d1538ea9bea89d507219d3219592382eee306134 // Commit date: 29 October 2016 diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js index eebc81125..70be0dc7f 100644 --- a/src/js/codemirror/ubo-static-filtering.js +++ b/src/js/codemirror/ubo-static-filtering.js @@ -25,8 +25,7 @@ /******************************************************************************/ -{ -// >>>>> start of local scope +import { StaticFilteringParser } from '../static-filtering-parser.js'; /******************************************************************************/ @@ -40,9 +39,6 @@ let hintHelperRegistered = false; /******************************************************************************/ CodeMirror.defineMode('ubo-static-filtering', function() { - const StaticFilteringParser = typeof vAPI === 'object' - ? vAPI.StaticFilteringParser - : self.StaticFilteringParser; if ( StaticFilteringParser instanceof Object === false ) { return; } const parser = new StaticFilteringParser({ interactive: true }); @@ -417,9 +413,6 @@ CodeMirror.defineMode('ubo-static-filtering', function() { // https://codemirror.net/demo/complete.html const initHints = function() { - const StaticFilteringParser = typeof vAPI === 'object' - ? vAPI.StaticFilteringParser - : self.StaticFilteringParser; if ( StaticFilteringParser instanceof Object === false ) { return; } const parser = new StaticFilteringParser(); @@ -802,8 +795,3 @@ CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => { } /******************************************************************************/ - -// <<<<< end of local scope -} - -/******************************************************************************/ diff --git a/src/js/commands.js b/src/js/commands.js index a695427fb..463e63c08 100644 --- a/src/js/commands.js +++ b/src/js/commands.js @@ -19,9 +19,12 @@ Home: https://github.com/gorhill/uBlock */ +'use strict'; + /******************************************************************************/ -'use strict'; +import { hostnameFromURI } from './uri-utils.js'; +import µBlock from './background.js'; /******************************************************************************/ @@ -71,11 +74,11 @@ const relaxBlockingMode = (( ) => { if ( tab instanceof Object === false || tab.id <= 0 ) { return; } const µb = µBlock; - const normalURL = µb.normalizePageURL(tab.id, tab.url); + const normalURL = µb.normalizeTabURL(tab.id, tab.url); if ( µb.getNetFilteringSwitch(normalURL) === false ) { return; } - const hn = µb.URI.hostnameFromURI(normalURL); + const hn = hostnameFromURI(normalURL); const curProfileBits = µb.blockingModeFromHostname(hn); let newProfileBits; for ( const profile of µb.liveBlockingProfiles ) { diff --git a/src/js/contextmenu.js b/src/js/contextmenu.js index 12190a430..e664c3364 100644 --- a/src/js/contextmenu.js +++ b/src/js/contextmenu.js @@ -23,6 +23,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + µBlock.contextMenu = (( ) => { /******************************************************************************/ diff --git a/src/js/cosmetic-filtering.js b/src/js/cosmetic-filtering.js index f0d045475..3601a7229 100644 --- a/src/js/cosmetic-filtering.js +++ b/src/js/cosmetic-filtering.js @@ -23,7 +23,13 @@ /******************************************************************************/ -µBlock.cosmeticFilteringEngine = (( ) => { +import { + domainFromHostname, + entityFromDomain, + hostnameFromURI, +} from './uri-utils.js'; + +import µBlock from './background.js'; /******************************************************************************/ @@ -253,7 +259,6 @@ const FilterContainer = function() { // Reset all, thus reducing to a minimum memory footprint of the context. FilterContainer.prototype.reset = function() { - this.µburi = µb.URI; this.frozen = false; this.acceptedCount = 0; this.discardedCount = 0; @@ -369,7 +374,6 @@ FilterContainer.prototype.compile = function(parser, writer) { /******************************************************************************/ FilterContainer.prototype.compileGenericSelector = function(parser, writer) { - writer.select(µb.compiledCosmeticSection + COMPILED_GENERIC_SECTION); if ( parser.isException() ) { this.compileGenericUnhideSelector(parser, writer); } else { @@ -385,14 +389,17 @@ FilterContainer.prototype.compileGenericHideSelector = function( ) { const { raw, compiled, pseudoclass } = parser.result; if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', text: `Invalid generic cosmetic filter in ${who}: ${raw}` }); + return; } + writer.select(µb.compiledCosmeticSection + COMPILED_GENERIC_SECTION); + const type = compiled.charCodeAt(0); let key; @@ -429,7 +436,7 @@ FilterContainer.prototype.compileGenericHideSelector = function( if ( µb.hiddenSettings.allowGenericProceduralFilters === true ) { return this.compileSpecificSelector(parser, '', false, writer); } - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', @@ -485,7 +492,7 @@ FilterContainer.prototype.compileGenericUnhideSelector = function( // Procedural cosmetic filters are acceptable as generic exception filters. const { raw, compiled } = parser.result; if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', @@ -494,6 +501,8 @@ FilterContainer.prototype.compileGenericUnhideSelector = function( return; } + writer.select(µb.compiledCosmeticSection + COMPILED_SPECIFIC_SECTION); + // https://github.com/chrisaljoudi/uBlock/issues/497 // All generic exception filters are stored as hostname-based filter // whereas the hostname is the empty string (which matches all @@ -511,10 +520,9 @@ FilterContainer.prototype.compileSpecificSelector = function( not, writer ) { - writer.select(µb.compiledCosmeticSection + COMPILED_SPECIFIC_SECTION); const { raw, compiled, exception } = parser.result; if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; + const who = writer.properties.get('name') || '?'; µb.logger.writeOne({ realm: 'message', type: 'error', @@ -523,6 +531,8 @@ FilterContainer.prototype.compileSpecificSelector = function( return; } + writer.select(µb.compiledCosmeticSection + COMPILED_SPECIFIC_SECTION); + // https://github.com/chrisaljoudi/uBlock/issues/145 let unhide = exception ? 1 : 0; if ( not ) { unhide ^= 1; } @@ -1146,9 +1156,9 @@ FilterContainer.prototype.benchmark = async function() { const request = requests[i]; if ( request.cpt !== 'main_frame' ) { continue; } count += 1; - details.hostname = µb.URI.hostnameFromURI(request.url); - details.domain = µb.URI.domainFromHostname(details.hostname); - details.entity = µb.URI.entityFromDomain(details.domain); + details.hostname = hostnameFromURI(request.url); + details.domain = domainFromHostname(details.hostname); + details.entity = entityFromDomain(details.domain); void this.retrieveSpecificSelectors(details, options); } const t1 = self.performance.now(); @@ -1159,10 +1169,8 @@ FilterContainer.prototype.benchmark = async function() { /******************************************************************************/ -return new FilterContainer(); - -/******************************************************************************/ - -})(); +// Export + +µBlock.cosmeticFilteringEngine = new FilterContainer(); /******************************************************************************/ diff --git a/src/js/dyna-rules.js b/src/js/dyna-rules.js index 0e489bfab..1abd8e841 100644 --- a/src/js/dyna-rules.js +++ b/src/js/dyna-rules.js @@ -19,17 +19,23 @@ Home: https://github.com/gorhill/uMatrix */ -/* global diff_match_patch, CodeMirror, uDom, uBlockDashboard */ +/* global CodeMirror, diff_match_patch, uDom, uBlockDashboard */ 'use strict'; /******************************************************************************/ -(( ) => { +import '../lib/publicsuffixlist/publicsuffixlist.js'; + +import globals from './globals.js'; +import { hostnameFromURI } from './uri-utils.js'; + +import './codemirror/ubo-dynamic-filtering.js'; /******************************************************************************/ -const psl = self.publicSuffixList; +const publicSuffixList = globals.publicSuffixList; + const hostnameToDomainMap = new Map(); const mergeView = new CodeMirror.MergeView( @@ -376,8 +382,8 @@ const onFilterChanged = (( ) => { }; return function() { - if ( timer !== undefined ) { self.cancelIdleCallback(timer); } - timer = self.requestIdleCallback(process, { timeout: 773 }); + if ( timer !== undefined ) { globals.cancelIdleCallback(timer); } + timer = globals.requestIdleCallback(process, { timeout: 773 }); }; })(); @@ -393,7 +399,9 @@ const onPresentationChanged = (( ) => { const sortNormalizeHn = function(hn) { let domain = hostnameToDomainMap.get(hn); if ( domain === undefined ) { - domain = /(\d|\])$/.test(hn) ? hn : psl.getDomain(hn); + domain = /(\d|\])$/.test(hn) + ? hn + : publicSuffixList.getDomain(hn); hostnameToDomainMap.set(hn, domain); } let normalized = domain || hn; @@ -424,7 +432,7 @@ const onPresentationChanged = (( ) => { } else if ( (match = reUrlRule.exec(rule)) !== null ) { type = '\x10FFFF'; srcHn = sortNormalizeHn(match[1]); - desHn = sortNormalizeHn(vAPI.hostnameFromURI(match[2])); + desHn = sortNormalizeHn(hostnameFromURI(match[2])); extra = match[3]; } if ( sortType === 0 ) { @@ -499,13 +507,13 @@ const onPresentationChanged = (( ) => { const mode = origPane.doc.getMode(); mode.sortType = sortType; mode.setHostnameToDomainMap(hostnameToDomainMap); - mode.setPSL(psl); + mode.setPSL(publicSuffixList); } { const mode = editPane.doc.getMode(); mode.sortType = sortType; mode.setHostnameToDomainMap(hostnameToDomainMap); - mode.setPSL(psl); + mode.setPSL(publicSuffixList); } sort(origPane.modified); sort(editPane.modified); @@ -547,8 +555,8 @@ const onTextChanged = (( ) => { }; return function(now) { - if ( timer !== undefined ) { self.cancelIdleCallback(timer); } - timer = now ? process() : self.requestIdleCallback(process, { timeout: 57 }); + if ( timer !== undefined ) { globals.cancelIdleCallback(timer); } + timer = now ? process() : globals.requestIdleCallback(process, { timeout: 57 }); }; })(); @@ -617,11 +625,11 @@ const editSaveHandler = function() { /******************************************************************************/ -self.cloud.onPush = function() { +globals.cloud.onPush = function() { return thePanes.orig.original.join('\n'); }; -self.cloud.onPull = function(data, append) { +globals.cloud.onPull = function(data, append) { if ( typeof data !== 'string' ) { return; } applyDiff( false, @@ -632,7 +640,7 @@ self.cloud.onPull = function(data, append) { /******************************************************************************/ -self.hasUnsavedData = function() { +globals.hasUnsavedData = function() { return mergeView.editor().isClean(cleanEditToken) === false; }; @@ -643,7 +651,7 @@ vAPI.messaging.send('dashboard', { }).then(details => { thePanes.orig.original = details.permanentRules; thePanes.edit.original = details.sessionRules; - psl.fromSelfie(details.pslSelfie); + publicSuffixList.fromSelfie(details.pslSelfie); onPresentationChanged(true); }); @@ -668,5 +676,3 @@ mergeView.editor().on('updateDiff', ( ) => { onTextChanged(); }); /******************************************************************************/ -})(); - diff --git a/src/js/dynamic-net-filtering.js b/src/js/dynamic-net-filtering.js index c0f322e03..6f2833027 100644 --- a/src/js/dynamic-net-filtering.js +++ b/src/js/dynamic-net-filtering.js @@ -19,18 +19,23 @@ Home: https://github.com/gorhill/uBlock */ -/* global punycode */ /* jshint bitwise: false */ 'use strict'; /******************************************************************************/ -{ -// >>>>> start of local scope +import '../lib/punycode.js'; + +import globals from './globals.js'; +import { domainFromHostname } from './uri-utils.js'; +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; /******************************************************************************/ +const punycode = globals.punycode; + const supportedDynamicTypes = { '3p': true, 'image': true, @@ -89,8 +94,6 @@ const is3rdParty = function(srcHostname, desHostname) { desHostname.charAt(desHostname.length - srcDomain.length - 1) !== '.'; }; -const domainFromHostname = µBlock.URI.domainFromHostname; - /******************************************************************************/ const Matrix = class { @@ -429,7 +432,7 @@ const Matrix = class { fromString(text, append) { - const lineIter = new µBlock.LineIterator(text); + const lineIter = new LineIterator(text); if ( append !== true ) { this.reset(); } while ( lineIter.eot() === false ) { this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); @@ -541,13 +544,10 @@ Matrix.prototype.magicId = 1; /******************************************************************************/ +// Export + µBlock.Firewall = Matrix; -// <<<<< end of local scope -} - -/******************************************************************************/ - µBlock.sessionFirewall = new µBlock.Firewall(); µBlock.permanentFirewall = new µBlock.Firewall(); diff --git a/src/js/epicker-ui.js b/src/js/epicker-ui.js index e45c0035f..5cbbc1d1f 100644 --- a/src/js/epicker-ui.js +++ b/src/js/epicker-ui.js @@ -23,6 +23,13 @@ 'use strict'; +/******************************************************************************/ + +import './codemirror/ubo-static-filtering.js'; + +import { hostnameFromURI } from './uri-utils.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; + /******************************************************************************/ /******************************************************************************/ @@ -144,7 +151,7 @@ const renderRange = function(id, value, invert = false) { const userFilterFromCandidate = function(filter) { if ( filter === '' || filter === '!' ) { return; } - const hn = vAPI.hostnameFromURI(docURL.href); + const hn = hostnameFromURI(docURL.href); // Cosmetic filter? if ( reCosmeticAnchor.test(filter) ) { @@ -828,7 +835,7 @@ const startPicker = function() { $id('candidateFilters').addEventListener('click', onCandidateClicked); $stor('#resultsetDepth input').addEventListener('input', onDepthChanged); $stor('#resultsetSpecificity input').addEventListener('input', onSpecificityChanged); - staticFilteringParser = new vAPI.StaticFilteringParser({ interactive: true }); + staticFilteringParser = new StaticFilteringParser({ interactive: true }); }; /******************************************************************************/ diff --git a/src/js/filtering-context.js b/src/js/filtering-context.js index af68ca19e..53d99e80d 100644 --- a/src/js/filtering-context.js +++ b/src/js/filtering-context.js @@ -23,14 +23,11 @@ /******************************************************************************/ -{ -// >>>>> start of local scope - -/******************************************************************************/ - -const originFromURI = µBlock.URI.originFromURI; -const hostnameFromURI = vAPI.hostnameFromURI; -const domainFromHostname = vAPI.domainFromHostname; +import { + hostnameFromURI, + domainFromHostname, + originFromURI, +} from './uri-utils.js'; /******************************************************************************/ @@ -133,68 +130,6 @@ const FilteringContext = class { return (this.itype & FONT_ANY) !== 0; } - fromTabId(tabId) { - const tabContext = µBlock.tabContextManager.mustLookup(tabId); - this.tabOrigin = tabContext.origin; - this.tabHostname = tabContext.rootHostname; - this.tabDomain = tabContext.rootDomain; - this.tabId = tabContext.tabId; - return this; - } - - // https://github.com/uBlockOrigin/uBlock-issues/issues/459 - // In case of a request for frame and if ever no context is specified, - // assume the origin of the context is the same as the request itself. - fromWebrequestDetails(details) { - const tabId = details.tabId; - this.type = details.type; - if ( this.itype === MAIN_FRAME && tabId > 0 ) { - µBlock.tabContextManager.push(tabId, details.url); - } - this.fromTabId(tabId); // Must be called AFTER tab context management - this.realm = ''; - this.id = details.requestId; - this.setURL(details.url); - this.aliasURL = details.aliasURL || undefined; - if ( this.itype !== SUB_FRAME ) { - this.docId = details.frameId; - this.frameId = -1; - } else { - this.docId = details.parentFrameId; - this.frameId = details.frameId; - } - if ( this.tabId > 0 ) { - if ( this.docId === 0 ) { - this.docOrigin = this.tabOrigin; - this.docHostname = this.tabHostname; - this.docDomain = this.tabDomain; - } else if ( details.documentUrl !== undefined ) { - this.setDocOriginFromURL(details.documentUrl); - } else { - const pageStore = µBlock.pageStoreFromTabId(this.tabId); - const docStore = pageStore && pageStore.getFrameStore(this.docId); - if ( docStore ) { - this.setDocOriginFromURL(docStore.rawURL); - } else { - this.setDocOrigin(this.tabOrigin); - } - } - } else if ( details.documentUrl !== undefined ) { - const origin = originFromURI( - µBlock.normalizePageURL(0, details.documentUrl) - ); - this.setDocOrigin(origin).setTabOrigin(origin); - } else if ( this.docId === -1 || (this.itype & FRAME_ANY) !== 0 ) { - const origin = originFromURI(this.url); - this.setDocOrigin(origin).setTabOrigin(origin); - } else { - this.setDocOrigin(this.tabOrigin); - } - this.redirectURL = undefined; - this.filter = undefined; - return this; - } - fromFilteringContext(other) { this.realm = other.realm; this.type = other.type; @@ -331,12 +266,6 @@ const FilteringContext = class { } getTabOrigin() { - if ( this.tabOrigin === undefined ) { - const tabContext = µBlock.tabContextManager.mustLookup(this.tabId); - this.tabOrigin = tabContext.origin; - this.tabHostname = tabContext.rootHostname; - this.tabDomain = tabContext.rootDomain; - } return this.tabOrigin; } @@ -353,9 +282,6 @@ const FilteringContext = class { } getTabHostname() { - if ( this.tabHostname === undefined ) { - this.tabHostname = hostnameFromURI(this.getTabOrigin()); - } return this.tabHostname; } @@ -422,29 +348,6 @@ const FilteringContext = class { } return this; } - - toLogger() { - this.tstamp = Date.now(); - if ( this.domain === undefined ) { - void this.getDomain(); - } - if ( this.docDomain === undefined ) { - void this.getDocDomain(); - } - if ( this.tabDomain === undefined ) { - void this.getTabDomain(); - } - const logger = µBlock.logger; - const filters = this.filter; - // Many filters may have been applied to the current context - if ( Array.isArray(filters) === false ) { - return logger.writeOne(this); - } - for ( const filter of filters ) { - this.filter = filter; - logger.writeOne(this); - } - } }; /******************************************************************************/ @@ -475,8 +378,4 @@ FilteringContext.prototype.SCRIPT_ANY = FilteringContext.SCRIPT_ANY = SCRIPT_ANY /******************************************************************************/ -µBlock.FilteringContext = FilteringContext; -µBlock.filteringContext = new FilteringContext(); - -// <<<<< end of local scope -} +export { FilteringContext }; diff --git a/src/js/globals.js b/src/js/globals.js new file mode 100644 index 000000000..3401bd8c2 --- /dev/null +++ b/src/js/globals.js @@ -0,0 +1,53 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/globalThis + +const globals = (( ) => { + // jshint ignore:start + if ( typeof globalThis !== 'undefined' ) { return globalThis; } + if ( typeof self !== 'undefined' ) { return self; } + if ( typeof global !== 'undefined' ) { return global; } + // jshint ignore:end +})(); + +// https://en.wikipedia.org/wiki/.invalid +if ( globals.location === undefined ) { + globals.location = new URL('https://ublock0.invalid/'); +} + +// https://developer.mozilla.org/en-US/docs/Web/API/Window/requestIdleCallback +if ( globals.requestIdleCallback === undefined ) { + globals.requestIdleCallback = function(callback) { + return globals.setTimeout(callback, 1); + }; + globals.cancelIdleCallback = function(handle) { + return globals.clearTimeout(handle); + }; +} + +/******************************************************************************/ + +export default globals; diff --git a/src/js/hnswitches.js b/src/js/hnswitches.js index 4f00517de..9df8a7494 100644 --- a/src/js/hnswitches.js +++ b/src/js/hnswitches.js @@ -19,24 +19,31 @@ Home: https://github.com/gorhill/uBlock */ -/* global punycode */ /* jshint bitwise: false */ 'use strict'; /******************************************************************************/ -µBlock.HnSwitches = (( ) => { +import '../lib/punycode.js'; + +import globals from './globals.js'; +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; /******************************************************************************/ -var HnSwitches = function() { +const punycode = globals.punycode; + +/******************************************************************************/ + +const HnSwitches = function() { this.reset(); }; /******************************************************************************/ -var switchBitOffsets = { +const switchBitOffsets = { 'no-strict-blocking': 0, 'no-popups': 2, 'no-cosmetic-filtering': 4, @@ -46,12 +53,12 @@ var switchBitOffsets = { 'no-scripting': 12, }; -var switchStateToNameMap = { +const switchStateToNameMap = { '1': 'true', '2': 'false' }; -var nameToSwitchStateMap = { +const nameToSwitchStateMap = { 'true': 1, 'false': 2, 'on': 1, @@ -61,7 +68,7 @@ var nameToSwitchStateMap = { /******************************************************************************/ // For performance purpose, as simple tests as possible -var reNotASCII = /[^\x20-\x7F]/; +const reNotASCII = /[^\x20-\x7F]/; // http://tools.ietf.org/html/rfc5952 // 4.3: "MUST be represented in lowercase" @@ -82,14 +89,14 @@ HnSwitches.prototype.reset = function() { HnSwitches.prototype.assign = function(from) { // Remove rules not in other - for ( let hn of this.switches.keys() ) { + for ( const hn of this.switches.keys() ) { if ( from.switches.has(hn) === false ) { this.switches.delete(hn); this.changed = true; } } // Add/change rules in other - for ( let [hn, bits] of from.switches ) { + for ( const [hn, bits] of from.switches ) { if ( this.switches.get(hn) !== bits ) { this.switches.set(hn, bits); this.changed = true; @@ -100,8 +107,8 @@ HnSwitches.prototype.assign = function(from) { /******************************************************************************/ HnSwitches.prototype.copyRules = function(from, srcHostname) { - let thisBits = this.switches.get(srcHostname); - let fromBits = from.switches.get(srcHostname); + const thisBits = this.switches.get(srcHostname); + const fromBits = from.switches.get(srcHostname); if ( fromBits !== thisBits ) { if ( fromBits !== undefined ) { this.switches.set(srcHostname, fromBits); @@ -124,7 +131,7 @@ HnSwitches.prototype.hasSameRules = function(other, srcHostname) { // If value is undefined, the switch is removed HnSwitches.prototype.toggle = function(switchName, hostname, newVal) { - let bitOffset = switchBitOffsets[switchName]; + const bitOffset = switchBitOffsets[switchName]; if ( bitOffset === undefined ) { return false; } if ( newVal === this.evaluate(switchName, hostname) ) { return false; } let bits = this.switches.get(hostname) || 0; @@ -142,7 +149,7 @@ HnSwitches.prototype.toggle = function(switchName, hostname, newVal) { /******************************************************************************/ HnSwitches.prototype.toggleOneZ = function(switchName, hostname, newState) { - let bitOffset = switchBitOffsets[switchName]; + const bitOffset = switchBitOffsets[switchName]; if ( bitOffset === undefined ) { return false; } let state = this.evaluateZ(switchName, hostname); if ( newState === state ) { return false; } @@ -171,8 +178,8 @@ HnSwitches.prototype.toggleBranchZ = function(switchName, targetHostname, newSta // Turn off all descendant switches, they will inherit the state of the // branch's origin. - let targetLen = targetHostname.length; - for ( let hostname of this.switches.keys() ) { + const targetLen = targetHostname.length; + for ( const hostname of this.switches.keys() ) { if ( hostname === targetHostname ) { continue; } if ( hostname.length <= targetLen ) { continue; } if ( hostname.endsWith(targetHostname) === false ) { continue; } @@ -201,7 +208,7 @@ HnSwitches.prototype.toggleZ = function(switchName, hostname, deep, newState) { // 2 = forced default state (to override a broader non-default state) HnSwitches.prototype.evaluate = function(switchName, hostname) { - let bits = this.switches.get(hostname); + const bits = this.switches.get(hostname); if ( bits === undefined ) { return 0; } @@ -215,14 +222,14 @@ HnSwitches.prototype.evaluate = function(switchName, hostname) { /******************************************************************************/ HnSwitches.prototype.evaluateZ = function(switchName, hostname) { - let bitOffset = switchBitOffsets[switchName]; + const bitOffset = switchBitOffsets[switchName]; if ( bitOffset === undefined ) { this.r = 0; return false; } this.n = switchName; µBlock.decomposeHostname(hostname, this.decomposedSource); - for ( let shn of this.decomposedSource ) { + for ( const shn of this.decomposedSource ) { let bits = this.switches.get(shn); if ( bits !== undefined ) { bits = bits >>> bitOffset & 3; @@ -250,16 +257,16 @@ HnSwitches.prototype.toLogData = function() { /******************************************************************************/ HnSwitches.prototype.toArray = function() { - let out = [], - toUnicode = punycode.toUnicode; + const out = []; + const toUnicode = punycode.toUnicode; for ( let hostname of this.switches.keys() ) { - for ( var switchName in switchBitOffsets ) { + for ( const switchName in switchBitOffsets ) { if ( switchBitOffsets.hasOwnProperty(switchName) === false ) { continue; } - let val = this.evaluate(switchName, hostname); + const val = this.evaluate(switchName, hostname); if ( val === 0 ) { continue; } - if ( hostname.indexOf('xn--') !== -1 ) { + if ( hostname.includes('xn--') ) { hostname = toUnicode(hostname); } out.push(switchName + ': ' + hostname + ' ' + switchStateToNameMap[val]); @@ -275,7 +282,7 @@ HnSwitches.prototype.toString = function() { /******************************************************************************/ HnSwitches.prototype.fromString = function(text, append) { - let lineIter = new µBlock.LineIterator(text); + const lineIter = new LineIterator(text); if ( append !== true ) { this.reset(); } while ( lineIter.eot() === false ) { this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); @@ -297,7 +304,7 @@ HnSwitches.prototype.validateRuleParts = function(parts) { HnSwitches.prototype.addFromRuleParts = function(parts) { if ( this.validateRuleParts(parts) !== undefined ) { - let switchName = parts[0].slice(0, -1); + const switchName = parts[0].slice(0, -1); if ( switchBitOffsets.hasOwnProperty(switchName) ) { this.toggle(switchName, parts[1], nameToSwitchStateMap[parts[2]]); return true; @@ -316,15 +323,11 @@ HnSwitches.prototype.removeFromRuleParts = function(parts) { /******************************************************************************/ -return HnSwitches; - -/******************************************************************************/ - -})(); - -/******************************************************************************/ - -µBlock.sessionSwitches = new µBlock.HnSwitches(); -µBlock.permanentSwitches = new µBlock.HnSwitches(); +// Export + +µBlock.HnSwitches = HnSwitches; + +µBlock.sessionSwitches = new HnSwitches(); +µBlock.permanentSwitches = new HnSwitches(); /******************************************************************************/ diff --git a/src/js/hntrie.js b/src/js/hntrie.js index dd9c82e67..f18c13a90 100644 --- a/src/js/hntrie.js +++ b/src/js/hntrie.js @@ -23,11 +23,6 @@ 'use strict'; -// ***************************************************************************** -// start of local namespace - -{ - /******************************************************************************* The original prototype was to develop an idea I had about using jump indices @@ -461,10 +456,10 @@ const HNTrieContainer = class { return n === hr || hn.charCodeAt(hl-1) === 0x2E /* '.' */; } - async enableWASM() { + async enableWASM(modulePath) { if ( typeof WebAssembly !== 'object' ) { return false; } if ( this.wasmMemory instanceof WebAssembly.Memory ) { return true; } - const module = await getWasmModule(); + const module = await getWasmModule(modulePath); if ( module instanceof WebAssembly.Module === false ) { return false; } const memory = new WebAssembly.Memory({ initial: 2 }); const instance = await WebAssembly.instantiate(module, { @@ -772,20 +767,7 @@ HNTrieContainer.prototype.HNTrieRef.prototype.needle = ''; const getWasmModule = (( ) => { let wasmModulePromise; - // The directory from which the current script was fetched should also - // contain the related WASM file. The script is fetched from a trusted - // location, and consequently so will be the related WASM file. - let workingDir; - { - const url = new URL(document.currentScript.src); - const match = /[^\/]+$/.exec(url.pathname); - if ( match !== null ) { - url.pathname = url.pathname.slice(0, match.index); - } - workingDir = url.href; - } - - return async function() { + return async function(modulePath) { if ( wasmModulePromise instanceof Promise ) { return wasmModulePromise; } @@ -809,7 +791,7 @@ const getWasmModule = (( ) => { if ( uint8s[0] !== 1 ) { return; } wasmModulePromise = fetch( - workingDir + 'wasm/hntrie.wasm', + `${modulePath}/wasm/hntrie.wasm`, { mode: 'same-origin' } ).then( WebAssembly.compileStreaming @@ -823,9 +805,4 @@ const getWasmModule = (( ) => { /******************************************************************************/ -µBlock.HNTrieContainer = HNTrieContainer; - -// end of local namespace -// ***************************************************************************** - -} +export { HNTrieContainer }; diff --git a/src/js/html-filtering.js b/src/js/html-filtering.js index 8431d670c..52e9bb962 100644 --- a/src/js/html-filtering.js +++ b/src/js/html-filtering.js @@ -23,421 +23,426 @@ /******************************************************************************/ -µBlock.htmlFilteringEngine = (function() { - const µb = µBlock; - const pselectors = new Map(); - const duplicates = new Set(); - - const filterDB = new µb.staticExtFilteringEngine.HostnameBasedDB(2); - const sessionFilterDB = new µb.staticExtFilteringEngine.SessionDB(); - - let acceptedCount = 0; - let discardedCount = 0; - let docRegister; - - const api = { - get acceptedCount() { - return acceptedCount; - }, - get discardedCount() { - return discardedCount; - } - }; - - const PSelectorHasTextTask = class { - constructor(task) { - let arg0 = task[1], arg1; - if ( Array.isArray(task[1]) ) { - arg1 = arg0[1]; arg0 = arg0[0]; - } - this.needle = new RegExp(arg0, arg1); - } - transpose(node, output) { - if ( this.needle.test(node.textContent) ) { - output.push(node); - } - } - }; - - const PSelectorIfTask = class { - constructor(task) { - this.pselector = new PSelector(task[1]); - } - transpose(node, output) { - if ( this.pselector.test(node) === this.target ) { - output.push(node); - } - } - get invalid() { - return this.pselector.invalid; - } - }; - PSelectorIfTask.prototype.target = true; - - const PSelectorIfNotTask = class extends PSelectorIfTask { - }; - PSelectorIfNotTask.prototype.target = false; - - const PSelectorMinTextLengthTask = class { - constructor(task) { - this.min = task[1]; - } - transpose(node, output) { - if ( node.textContent.length >= this.min ) { - output.push(node); - } - } - }; - - const PSelectorSpathTask = class { - constructor(task) { - this.spath = task[1]; - } - transpose(node, output) { - const parent = node.parentElement; - if ( parent === null ) { return; } - let pos = 1; - for (;;) { - node = node.previousElementSibling; - if ( node === null ) { break; } - pos += 1; - } - const nodes = parent.querySelectorAll( - `:scope > :nth-child(${pos})${this.spath}` - ); - for ( const node of nodes ) { - output.push(node); - } - } - }; - - const PSelectorUpwardTask = class { - constructor(task) { - const arg = task[1]; - if ( typeof arg === 'number' ) { - this.i = arg; - } else { - this.s = arg; - } - } - transpose(node, output) { - if ( this.s !== '' ) { - const parent = node.parentElement; - if ( parent === null ) { return; } - node = parent.closest(this.s); - if ( node === null ) { return; } - } else { - let nth = this.i; - for (;;) { - node = node.parentElement; - if ( node === null ) { return; } - nth -= 1; - if ( nth === 0 ) { break; } - } - } - output.push(node); - } - }; - PSelectorUpwardTask.prototype.i = 0; - PSelectorUpwardTask.prototype.s = ''; - - const PSelectorXpathTask = class { - constructor(task) { - this.xpe = task[1]; - } - transpose(node, output) { - const xpr = docRegister.evaluate( - this.xpe, - node, - null, - XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, - null - ); - let j = xpr.snapshotLength; - while ( j-- ) { - const node = xpr.snapshotItem(j); - if ( node.nodeType === 1 ) { - output.push(node); - } - } - } - }; - - const PSelector = class { - constructor(o) { - this.raw = o.raw; - this.selector = o.selector; - this.tasks = []; - if ( !o.tasks ) { return; } - for ( const task of o.tasks ) { - const ctor = this.operatorToTaskMap.get(task[0]); - if ( ctor === undefined ) { - this.invalid = true; - break; - } - const pselector = new ctor(task); - if ( pselector instanceof PSelectorIfTask && pselector.invalid ) { - this.invalid = true; - break; - } - this.tasks.push(pselector); - } - } - prime(input) { - const root = input || docRegister; - if ( this.selector === '' ) { return [ root ]; } - return Array.from(root.querySelectorAll(this.selector)); - } - exec(input) { - if ( this.invalid ) { return []; } - let nodes = this.prime(input); - for ( const task of this.tasks ) { - if ( nodes.length === 0 ) { break; } - const transposed = []; - for ( const node of nodes ) { - task.transpose(node, transposed); - } - nodes = transposed; - } - return nodes; - } - test(input) { - if ( this.invalid ) { return false; } - const nodes = this.prime(input); - for ( const node of nodes ) { - let output = [ node ]; - for ( const task of this.tasks ) { - const transposed = []; - for ( const node of output ) { - task.transpose(node, transposed); - } - output = transposed; - if ( output.length === 0 ) { break; } - } - if ( output.length !== 0 ) { return true; } - } - return false; - } - }; - PSelector.prototype.operatorToTaskMap = new Map([ - [ ':has', PSelectorIfTask ], - [ ':has-text', PSelectorHasTextTask ], - [ ':if', PSelectorIfTask ], - [ ':if-not', PSelectorIfNotTask ], - [ ':min-text-length', PSelectorMinTextLengthTask ], - [ ':not', PSelectorIfNotTask ], - [ ':nth-ancestor', PSelectorUpwardTask ], - [ ':spath', PSelectorSpathTask ], - [ ':upward', PSelectorUpwardTask ], - [ ':xpath', PSelectorXpathTask ], - ]); - PSelector.prototype.invalid = false; - - const logOne = function(details, exception, selector) { - µBlock.filteringContext - .duplicate() - .fromTabId(details.tabId) - .setRealm('extended') - .setType('dom') - .setURL(details.url) - .setDocOriginFromURL(details.url) - .setFilter({ - source: 'extended', - raw: `${exception === 0 ? '##' : '#@#'}^${selector}` - }) - .toLogger(); - }; - - const applyProceduralSelector = function(details, selector) { - let pselector = pselectors.get(selector); - if ( pselector === undefined ) { - pselector = new PSelector(JSON.parse(selector)); - pselectors.set(selector, pselector); - } - const nodes = pselector.exec(); - let modified = false; - for ( const node of nodes ) { - node.remove(); - modified = true; - } - if ( modified && µb.logger.enabled ) { - logOne(details, 0, pselector.raw); - } - return modified; - }; - - const applyCSSSelector = function(details, selector) { - const nodes = docRegister.querySelectorAll(selector); - let modified = false; - for ( const node of nodes ) { - node.remove(); - modified = true; - } - if ( modified && µb.logger.enabled ) { - logOne(details, 0, selector); - } - return modified; - }; - - api.reset = function() { - filterDB.clear(); - pselectors.clear(); - duplicates.clear(); - acceptedCount = 0; - discardedCount = 0; - }; - - api.freeze = function() { - duplicates.clear(); - filterDB.collectGarbage(); - }; - - api.compile = function(parser, writer) { - const { raw, compiled, exception } = parser.result; - if ( compiled === undefined ) { - const who = writer.properties.get('assetKey') || '?'; - µb.logger.writeOne({ - realm: 'message', - type: 'error', - text: `Invalid HTML filter in ${who}: ##${raw}` - }); - return; - } - - writer.select(µb.compiledHTMLSection); - - // TODO: Mind negated hostnames, they are currently discarded. - - for ( const { hn, not, bad } of parser.extOptions() ) { - if ( bad ) { continue; } - let kind = 0; - if ( exception ) { - if ( not ) { continue; } - kind |= 0b01; - } - if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { - kind |= 0b10; - } - writer.push([ 64, hn, kind, compiled ]); - } - }; - - api.compileTemporary = function(parser) { - return { - session: sessionFilterDB, - selector: parser.result.compiled, - }; - }; - - api.fromCompiledContent = function(reader) { - // Don't bother loading filters if stream filtering is not supported. - if ( µb.canFilterResponseData === false ) { return; } - - reader.select(µb.compiledHTMLSection); - - while ( reader.next() ) { - acceptedCount += 1; - const fingerprint = reader.fingerprint(); - if ( duplicates.has(fingerprint) ) { - discardedCount += 1; - continue; - } - duplicates.add(fingerprint); - const args = reader.args(); - filterDB.store(args[1], args[2], args[3]); - } - }; - - api.getSession = function() { - return sessionFilterDB; - }; - - api.retrieve = function(details) { - const hostname = details.hostname; - - const plains = new Set(); - const procedurals = new Set(); - const exceptions = new Set(); - - if ( sessionFilterDB.isNotEmpty ) { - sessionFilterDB.retrieve([ null, exceptions ]); - } - filterDB.retrieve( - hostname, - [ plains, exceptions, procedurals, exceptions ] - ); - const entity = details.entity !== '' - ? `${hostname.slice(0, -details.domain.length)}${details.entity}` - : '*'; - filterDB.retrieve( - entity, - [ plains, exceptions, procedurals, exceptions ], - 1 - ); - - if ( plains.size === 0 && procedurals.size === 0 ) { return; } - - // https://github.com/gorhill/uBlock/issues/2835 - // Do not filter if the site is under an `allow` rule. - if ( - µb.userSettings.advancedUserEnabled && - µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 - ) { - return; - } - - const out = { plains, procedurals }; - - if ( exceptions.size === 0 ) { - return out; - } - - for ( const selector of exceptions ) { - if ( plains.has(selector) ) { - plains.delete(selector); - logOne(details, 1, selector); - continue; - } - if ( procedurals.has(selector) ) { - procedurals.delete(selector); - logOne(details, 1, JSON.parse(selector).raw); - continue; - } - } - - if ( plains.size !== 0 || procedurals.size !== 0 ) { - return out; - } - }; - - api.apply = function(doc, details) { - docRegister = doc; - let modified = false; - for ( const selector of details.selectors.plains ) { - if ( applyCSSSelector(details, selector) ) { - modified = true; - } - } - for ( const selector of details.selectors.procedurals ) { - if ( applyProceduralSelector(details, selector) ) { - modified = true; - } - } - - docRegister = undefined; - return modified; - }; - - api.toSelfie = function() { - return filterDB.toSelfie(); - }; - - api.fromSelfie = function(selfie) { - filterDB.fromSelfie(selfie); - pselectors.clear(); - }; - - return api; -})(); +import µBlock from './background.js'; + +/******************************************************************************/ + +const µb = µBlock; +const pselectors = new Map(); +const duplicates = new Set(); + +const filterDB = new µb.staticExtFilteringEngine.HostnameBasedDB(2); +const sessionFilterDB = new µb.staticExtFilteringEngine.SessionDB(); + +let acceptedCount = 0; +let discardedCount = 0; +let docRegister; + +const api = { + get acceptedCount() { + return acceptedCount; + }, + get discardedCount() { + return discardedCount; + } +}; + +const PSelectorHasTextTask = class { + constructor(task) { + let arg0 = task[1], arg1; + if ( Array.isArray(task[1]) ) { + arg1 = arg0[1]; arg0 = arg0[0]; + } + this.needle = new RegExp(arg0, arg1); + } + transpose(node, output) { + if ( this.needle.test(node.textContent) ) { + output.push(node); + } + } +}; + +const PSelectorIfTask = class { + constructor(task) { + this.pselector = new PSelector(task[1]); + } + transpose(node, output) { + if ( this.pselector.test(node) === this.target ) { + output.push(node); + } + } + get invalid() { + return this.pselector.invalid; + } +}; +PSelectorIfTask.prototype.target = true; + +const PSelectorIfNotTask = class extends PSelectorIfTask { +}; +PSelectorIfNotTask.prototype.target = false; + +const PSelectorMinTextLengthTask = class { + constructor(task) { + this.min = task[1]; + } + transpose(node, output) { + if ( node.textContent.length >= this.min ) { + output.push(node); + } + } +}; + +const PSelectorSpathTask = class { + constructor(task) { + this.spath = task[1]; + } + transpose(node, output) { + const parent = node.parentElement; + if ( parent === null ) { return; } + let pos = 1; + for (;;) { + node = node.previousElementSibling; + if ( node === null ) { break; } + pos += 1; + } + const nodes = parent.querySelectorAll( + `:scope > :nth-child(${pos})${this.spath}` + ); + for ( const node of nodes ) { + output.push(node); + } + } +}; + +const PSelectorUpwardTask = class { + constructor(task) { + const arg = task[1]; + if ( typeof arg === 'number' ) { + this.i = arg; + } else { + this.s = arg; + } + } + transpose(node, output) { + if ( this.s !== '' ) { + const parent = node.parentElement; + if ( parent === null ) { return; } + node = parent.closest(this.s); + if ( node === null ) { return; } + } else { + let nth = this.i; + for (;;) { + node = node.parentElement; + if ( node === null ) { return; } + nth -= 1; + if ( nth === 0 ) { break; } + } + } + output.push(node); + } +}; +PSelectorUpwardTask.prototype.i = 0; +PSelectorUpwardTask.prototype.s = ''; + +const PSelectorXpathTask = class { + constructor(task) { + this.xpe = task[1]; + } + transpose(node, output) { + const xpr = docRegister.evaluate( + this.xpe, + node, + null, + XPathResult.UNORDERED_NODE_SNAPSHOT_TYPE, + null + ); + let j = xpr.snapshotLength; + while ( j-- ) { + const node = xpr.snapshotItem(j); + if ( node.nodeType === 1 ) { + output.push(node); + } + } + } +}; + +const PSelector = class { + constructor(o) { + this.raw = o.raw; + this.selector = o.selector; + this.tasks = []; + if ( !o.tasks ) { return; } + for ( const task of o.tasks ) { + const ctor = this.operatorToTaskMap.get(task[0]); + if ( ctor === undefined ) { + this.invalid = true; + break; + } + const pselector = new ctor(task); + if ( pselector instanceof PSelectorIfTask && pselector.invalid ) { + this.invalid = true; + break; + } + this.tasks.push(pselector); + } + } + prime(input) { + const root = input || docRegister; + if ( this.selector === '' ) { return [ root ]; } + return Array.from(root.querySelectorAll(this.selector)); + } + exec(input) { + if ( this.invalid ) { return []; } + let nodes = this.prime(input); + for ( const task of this.tasks ) { + if ( nodes.length === 0 ) { break; } + const transposed = []; + for ( const node of nodes ) { + task.transpose(node, transposed); + } + nodes = transposed; + } + return nodes; + } + test(input) { + if ( this.invalid ) { return false; } + const nodes = this.prime(input); + for ( const node of nodes ) { + let output = [ node ]; + for ( const task of this.tasks ) { + const transposed = []; + for ( const node of output ) { + task.transpose(node, transposed); + } + output = transposed; + if ( output.length === 0 ) { break; } + } + if ( output.length !== 0 ) { return true; } + } + return false; + } +}; +PSelector.prototype.operatorToTaskMap = new Map([ + [ ':has', PSelectorIfTask ], + [ ':has-text', PSelectorHasTextTask ], + [ ':if', PSelectorIfTask ], + [ ':if-not', PSelectorIfNotTask ], + [ ':min-text-length', PSelectorMinTextLengthTask ], + [ ':not', PSelectorIfNotTask ], + [ ':nth-ancestor', PSelectorUpwardTask ], + [ ':spath', PSelectorSpathTask ], + [ ':upward', PSelectorUpwardTask ], + [ ':xpath', PSelectorXpathTask ], +]); +PSelector.prototype.invalid = false; + +const logOne = function(details, exception, selector) { + µBlock.filteringContext + .duplicate() + .fromTabId(details.tabId) + .setRealm('extended') + .setType('dom') + .setURL(details.url) + .setDocOriginFromURL(details.url) + .setFilter({ + source: 'extended', + raw: `${exception === 0 ? '##' : '#@#'}^${selector}` + }) + .toLogger(); +}; + +const applyProceduralSelector = function(details, selector) { + let pselector = pselectors.get(selector); + if ( pselector === undefined ) { + pselector = new PSelector(JSON.parse(selector)); + pselectors.set(selector, pselector); + } + const nodes = pselector.exec(); + let modified = false; + for ( const node of nodes ) { + node.remove(); + modified = true; + } + if ( modified && µb.logger.enabled ) { + logOne(details, 0, pselector.raw); + } + return modified; +}; + +const applyCSSSelector = function(details, selector) { + const nodes = docRegister.querySelectorAll(selector); + let modified = false; + for ( const node of nodes ) { + node.remove(); + modified = true; + } + if ( modified && µb.logger.enabled ) { + logOne(details, 0, selector); + } + return modified; +}; + +api.reset = function() { + filterDB.clear(); + pselectors.clear(); + duplicates.clear(); + acceptedCount = 0; + discardedCount = 0; +}; + +api.freeze = function() { + duplicates.clear(); + filterDB.collectGarbage(); +}; + +api.compile = function(parser, writer) { + const { raw, compiled, exception } = parser.result; + if ( compiled === undefined ) { + const who = writer.properties.get('name') || '?'; + µb.logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid HTML filter in ${who}: ##${raw}` + }); + return; + } + + writer.select(µb.compiledHTMLSection); + + // TODO: Mind negated hostnames, they are currently discarded. + + for ( const { hn, not, bad } of parser.extOptions() ) { + if ( bad ) { continue; } + let kind = 0; + if ( exception ) { + if ( not ) { continue; } + kind |= 0b01; + } + if ( compiled.charCodeAt(0) === 0x7B /* '{' */ ) { + kind |= 0b10; + } + writer.push([ 64, hn, kind, compiled ]); + } +}; + +api.compileTemporary = function(parser) { + return { + session: sessionFilterDB, + selector: parser.result.compiled, + }; +}; + +api.fromCompiledContent = function(reader) { + // Don't bother loading filters if stream filtering is not supported. + if ( µb.canFilterResponseData === false ) { return; } + + reader.select(µb.compiledHTMLSection); + + while ( reader.next() ) { + acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( duplicates.has(fingerprint) ) { + discardedCount += 1; + continue; + } + duplicates.add(fingerprint); + const args = reader.args(); + filterDB.store(args[1], args[2], args[3]); + } +}; + +api.getSession = function() { + return sessionFilterDB; +}; + +api.retrieve = function(details) { + const hostname = details.hostname; + + const plains = new Set(); + const procedurals = new Set(); + const exceptions = new Set(); + + if ( sessionFilterDB.isNotEmpty ) { + sessionFilterDB.retrieve([ null, exceptions ]); + } + filterDB.retrieve( + hostname, + [ plains, exceptions, procedurals, exceptions ] + ); + const entity = details.entity !== '' + ? `${hostname.slice(0, -details.domain.length)}${details.entity}` + : '*'; + filterDB.retrieve( + entity, + [ plains, exceptions, procedurals, exceptions ], + 1 + ); + + if ( plains.size === 0 && procedurals.size === 0 ) { return; } + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not filter if the site is under an `allow` rule. + if ( + µb.userSettings.advancedUserEnabled && + µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 + ) { + return; + } + + const out = { plains, procedurals }; + + if ( exceptions.size === 0 ) { + return out; + } + + for ( const selector of exceptions ) { + if ( plains.has(selector) ) { + plains.delete(selector); + logOne(details, 1, selector); + continue; + } + if ( procedurals.has(selector) ) { + procedurals.delete(selector); + logOne(details, 1, JSON.parse(selector).raw); + continue; + } + } + + if ( plains.size !== 0 || procedurals.size !== 0 ) { + return out; + } +}; + +api.apply = function(doc, details) { + docRegister = doc; + let modified = false; + for ( const selector of details.selectors.plains ) { + if ( applyCSSSelector(details, selector) ) { + modified = true; + } + } + for ( const selector of details.selectors.procedurals ) { + if ( applyProceduralSelector(details, selector) ) { + modified = true; + } + } + docRegister = undefined; + return modified; +}; + +api.toSelfie = function() { + return filterDB.toSelfie(); +}; + +api.fromSelfie = function(selfie) { + filterDB.fromSelfie(selfie); + pselectors.clear(); +}; + +/******************************************************************************/ + +// Export + +µBlock.htmlFilteringEngine = api; /******************************************************************************/ diff --git a/src/js/httpheader-filtering.js b/src/js/httpheader-filtering.js index 52e78721a..19c6b40e7 100644 --- a/src/js/httpheader-filtering.js +++ b/src/js/httpheader-filtering.js @@ -23,8 +23,8 @@ /******************************************************************************/ -{ -// >>>>> start of local scope +import { entityFromDomain } from './uri-utils.js'; +import µBlock from './background.js'; /******************************************************************************/ @@ -157,7 +157,7 @@ api.apply = function(fctxt, headers) { if ( hostname === '' ) { return; } const domain = fctxt.getDomain(); - let entity = µb.URI.entityFromDomain(domain); + let entity = entityFromDomain(domain); if ( entity !== '' ) { entity = `${hostname.slice(0, -domain.length)}${entity}`; } else { @@ -218,11 +218,10 @@ api.fromSelfie = function(selfie) { filterDB.fromSelfie(selfie); }; +/******************************************************************************/ + +// Export + µb.httpheaderFilteringEngine = api; /******************************************************************************/ - -// <<<<< end of local scope -} - -/******************************************************************************/ diff --git a/src/js/logger-ui-inspector.js b/src/js/logger-ui-inspector.js index 17fdb5cad..26b892198 100644 --- a/src/js/logger-ui-inspector.js +++ b/src/js/logger-ui-inspector.js @@ -25,6 +25,10 @@ /******************************************************************************/ +import globals from './globals.js'; + +/******************************************************************************/ + (( ) => { /******************************************************************************/ @@ -43,7 +47,7 @@ if ( /******************************************************************************/ -var logger = self.logger; +const logger = globals.logger; var inspectorConnectionId; var inspectedTabId = 0; var inspectedURL = ''; diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index 2dd738d2f..88175b931 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -25,7 +25,8 @@ /******************************************************************************/ -(( ) => { +import globals from './globals.js'; +import { hostnameFromURI } from './uri-utils.js'; /******************************************************************************/ @@ -33,7 +34,7 @@ // accumulated over time. const messaging = vAPI.messaging; -const logger = self.logger = { ownerId: Date.now() }; +const logger = globals.logger = { ownerId: Date.now() }; const logDate = new Date(); const logDateTimezoneOffset = logDate.getTimezoneOffset() * 60000; const loggerEntries = []; @@ -1677,8 +1678,8 @@ const reloadTab = function(ev) { const aliasURL = text ? aliasURLFromID(text) : ''; if ( aliasURL !== '' ) { rows[8].children[1].textContent = - vAPI.hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' + - vAPI.hostnameFromURI(canonicalURL); + hostnameFromURI(aliasURL) + ' \u21d2\n\u2003' + + hostnameFromURI(canonicalURL); rows[9].children[1].textContent = aliasURL; } else { rows[8].style.display = 'none'; @@ -2891,5 +2892,3 @@ if ( self.location.search.includes('popup=1') ) { } /******************************************************************************/ - -})(); diff --git a/src/js/logger.js b/src/js/logger.js index 6e3741402..eb417d306 100644 --- a/src/js/logger.js +++ b/src/js/logger.js @@ -22,70 +22,73 @@ 'use strict'; /******************************************************************************/ + +import µBlock from './background.js'; + /******************************************************************************/ -µBlock.logger = (function() { +let buffer = null; +let lastReadTime = 0; +let writePtr = 0; - let buffer = null; - let lastReadTime = 0; - let writePtr = 0; +// After 60 seconds without being read, a buffer will be considered +// unused, and thus removed from memory. +const logBufferObsoleteAfter = 30 * 1000; - // After 60 seconds without being read, a buffer will be considered - // unused, and thus removed from memory. - const logBufferObsoleteAfter = 30 * 1000; +const janitor = ( ) => { + if ( + buffer !== null && + lastReadTime < (Date.now() - logBufferObsoleteAfter) + ) { + api.enabled = false; + buffer = null; + writePtr = 0; + api.ownerId = undefined; + vAPI.messaging.broadcast({ what: 'loggerDisabled' }); + } + if ( buffer !== null ) { + vAPI.setTimeout(janitor, logBufferObsoleteAfter); + } +}; - const janitor = ( ) => { - if ( - buffer !== null && - lastReadTime < (Date.now() - logBufferObsoleteAfter) - ) { - api.enabled = false; - buffer = null; - writePtr = 0; - api.ownerId = undefined; - vAPI.messaging.broadcast({ what: 'loggerDisabled' }); +const boxEntry = function(details) { + if ( details.tstamp === undefined ) { + details.tstamp = Date.now(); + } + return JSON.stringify(details); +}; + +const api = { + enabled: false, + ownerId: undefined, + writeOne: function(details) { + if ( buffer === null ) { return; } + const box = boxEntry(details); + if ( writePtr === buffer.length ) { + buffer.push(box); + } else { + buffer[writePtr] = box; } - if ( buffer !== null ) { + writePtr += 1; + }, + readAll: function(ownerId) { + this.ownerId = ownerId; + if ( buffer === null ) { + this.enabled = true; + buffer = []; vAPI.setTimeout(janitor, logBufferObsoleteAfter); } - }; - - const boxEntry = function(details) { - if ( details.tstamp === undefined ) { - details.tstamp = Date.now(); - } - return JSON.stringify(details); - }; - - const api = { - enabled: false, - ownerId: undefined, - writeOne: function(details) { - if ( buffer === null ) { return; } - const box = boxEntry(details); - if ( writePtr === buffer.length ) { - buffer.push(box); - } else { - buffer[writePtr] = box; - } - writePtr += 1; - }, - readAll: function(ownerId) { - this.ownerId = ownerId; - if ( buffer === null ) { - this.enabled = true; - buffer = []; - vAPI.setTimeout(janitor, logBufferObsoleteAfter); - } - const out = buffer.slice(0, writePtr); - writePtr = 0; - lastReadTime = Date.now(); - return out; - }, - }; - - return api; - -})(); + const out = buffer.slice(0, writePtr); + writePtr = 0; + lastReadTime = Date.now(); + return out; + }, +}; + +/******************************************************************************/ + +// Export + +µBlock.logger = api; /******************************************************************************/ diff --git a/src/js/lz4.js b/src/js/lz4.js index efb8e7d6c..539fc8a94 100644 --- a/src/js/lz4.js +++ b/src/js/lz4.js @@ -23,6 +23,10 @@ 'use strict'; +/******************************************************************************/ + +import µBlock from './background.js'; + /******************************************************************************* Experimental support for storage compression. @@ -32,9 +36,6 @@ **/ -{ -// >>>> Start of private namespace - /******************************************************************************/ let lz4CodecInstance; @@ -198,6 +199,3 @@ const decodeValue = function(inputArray) { }; /******************************************************************************/ - -// <<<< End of private namespace -} diff --git a/src/js/messaging.js b/src/js/messaging.js index bd311a15d..ab161940f 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -19,10 +19,27 @@ Home: https://github.com/gorhill/uBlock */ -/******************************************************************************/ +'use strict'; + /******************************************************************************/ -'use strict'; +import '../lib/publicsuffixlist/publicsuffixlist.js'; +import '../lib/punycode.js'; + +import globals from './globals.js'; + +import { + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; + +import { StaticFilteringParser } from './static-filtering-parser.js'; +import µBlock from './background.js'; + +/******************************************************************************/ // https://github.com/uBlockOrigin/uBlock-issues/issues/710 // Listeners have a name and a "privileged" status. @@ -51,12 +68,11 @@ const clickToLoad = function(request, sender) { }; const getDomainNames = function(targets) { - const µburi = µb.URI; return targets.map(target => { if ( typeof target !== 'string' ) { return ''; } return target.indexOf('/') !== -1 - ? µburi.domainFromURI(target) || '' - : µburi.domainFromHostname(target) || target; + ? domainFromURI(target) || '' + : domainFromHostname(target) || target; }); }; @@ -98,8 +114,13 @@ const onMessage = function(request, sender, callback) { return; case 'sfneBenchmark': - µb.staticNetFilteringEngine.benchmark().then(result => { - callback(result); + µb.loadBenchmarkDataset().then(requests => { + µb.staticNetFilteringEngine.benchmark( + requests, + { redirectEngine: µb.redirectEngine } + ).then(result => { + callback(result); + }); }); return; @@ -244,7 +265,7 @@ const getHostnameDict = function(hostnameDetailsMap, out) { for ( const hnDetails of hostnameDetailsMap.values() ) { const hostname = hnDetails.hostname; if ( hnDict[hostname] !== undefined ) { continue; } - const domain = vAPI.domainFromHostname(hostname) || hostname; + const domain = domainFromHostname(hostname) || hostname; const dnDetails = hostnameDetailsMap.get(domain) || { counts: createCounts() }; if ( hnDict[domain] === undefined ) { @@ -336,7 +357,7 @@ const popupDataFromTabId = function(tabId, tabTitle) { getHostnameDict(pageStore.getAllHostnameDetails(), r); r.contentLastModified = pageStore.contentLastModified; getFirewallRules(rootHostname, r); - r.canElementPicker = µb.URI.isNetworkURI(r.rawURL); + r.canElementPicker = isNetworkURI(r.rawURL); r.noPopups = µb.sessionSwitches.evaluateZ( 'no-popups', rootHostname @@ -568,9 +589,9 @@ const retrieveContentScriptParameters = async function(sender, request) { request.tabId = tabId; request.frameId = frameId; - request.hostname = µb.URI.hostnameFromURI(request.url); - request.domain = µb.URI.domainFromHostname(request.hostname); - request.entity = µb.URI.entityFromDomain(request.domain); + request.hostname = hostnameFromURI(request.url); + request.domain = domainFromHostname(request.hostname); + request.entity = entityFromDomain(request.domain); response.specificCosmeticFilters = µb.cosmeticFilteringEngine.retrieveSpecificSelectors(request, response); @@ -597,7 +618,7 @@ const retrieveContentScriptParameters = async function(sender, request) { // effective URL is available here in `request.url`. if ( µb.canInjectScriptletsNow === false || - µb.URI.isNetworkURI(sender.frameURL) === false + isNetworkURI(sender.frameURL) === false ) { response.scriptlets = µb.scriptletFilteringEngine.retrieve(request); } @@ -1017,16 +1038,15 @@ const resetUserData = async function() { // Filter lists const prepListEntries = function(entries) { - const µburi = µb.URI; for ( const k in entries ) { if ( entries.hasOwnProperty(k) === false ) { continue; } const entry = entries[k]; if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { - entry.supportName = µburi.hostnameFromURI(entry.supportURL); + entry.supportName = hostnameFromURI(entry.supportURL); } else if ( typeof entry.homeURL === 'string' && entry.homeURL !== '' ) { - const hn = µburi.hostnameFromURI(entry.homeURL); + const hn = hostnameFromURI(entry.homeURL); entry.supportURL = `http://${hn}/`; - entry.supportName = µburi.domainFromHostname(hn); + entry.supportName = domainFromHostname(hn); } } }; @@ -1059,7 +1079,7 @@ const getLists = async function(callback) { // TODO: also return origin of embedded frames? const getOriginHints = function() { - const punycode = self.punycode; + const punycode = globals.punycode; const out = new Set(); for ( const tabId of µb.pageStores.keys() ) { if ( tabId === -1 ) { continue; } @@ -1088,7 +1108,7 @@ const getRules = function() { µb.sessionSwitches.toArray(), µb.sessionURLFiltering.toArray() ), - pslSelfie: self.publicSuffixList.toSelfie(), + pslSelfie: globals.publicSuffixList.toSelfie(), }; }; @@ -1393,7 +1413,7 @@ const getURLFilteringData = function(details) { }; const compileTemporaryException = function(filter) { - const parser = new vAPI.StaticFilteringParser(); + const parser = new StaticFilteringParser(); parser.analyze(filter); if ( parser.shouldDiscard() ) { return; } return µb.staticExtFilteringEngine.compileTemporary(parser); diff --git a/src/js/pagestore.js b/src/js/pagestore.js index 676632118..64d14c2b4 100644 --- a/src/js/pagestore.js +++ b/src/js/pagestore.js @@ -21,6 +21,16 @@ 'use strict'; +/******************************************************************************/ + +import { + domainFromHostname, + hostnameFromURI, + isNetworkURI, +} from './uri-utils.js'; + +import µBlock from './background.js'; + /******************************************************************************* A PageRequestStore object is used to store net requests in two ways: @@ -30,11 +40,6 @@ To create a log of net requests **/ -{ - -// start of private namespace -// >>>>> - /******************************************************************************/ const µb = µBlock; @@ -188,9 +193,8 @@ const FrameStore = class { this.clickToLoad = false; this.rawURL = frameURL; if ( frameURL !== undefined ) { - this.hostname = vAPI.hostnameFromURI(frameURL); - this.domain = - vAPI.domainFromHostname(this.hostname) || this.hostname; + this.hostname = hostnameFromURI(frameURL); + this.domain = domainFromHostname(this.hostname) || this.hostname; } // Evaluated on-demand // - 0b01: specific cosmetic filtering @@ -213,7 +217,7 @@ const FrameStore = class { } this._cosmeticFilteringBits = 0b11; { - const result = µb.staticNetFilteringEngine.matchStringReverse( + const result = µb.staticNetFilteringEngine.matchRequestReverse( 'specifichide', this.rawURL ); @@ -232,7 +236,7 @@ const FrameStore = class { } } { - const result = µb.staticNetFilteringEngine.matchStringReverse( + const result = µb.staticNetFilteringEngine.matchRequestReverse( 'generichide', this.rawURL ); @@ -600,7 +604,7 @@ const PageStore = class { getAllHostnameDetails() { if ( this.hostnameDetailsMap.has(this.tabHostname) === false && - µb.URI.isNetworkURI(this.rawURL) + isNetworkURI(this.rawURL) ) { this.hostnameDetailsMap.set( this.tabHostname, @@ -651,12 +655,12 @@ const PageStore = class { if ( this.journalLastUncommitted !== -1 && this.journalLastUncommitted < this.journalLastCommitted && - this.journalLastUncommittedOrigin === vAPI.hostnameFromURI(url) + this.journalLastUncommittedOrigin === hostnameFromURI(url) ) { this.journalLastCommitted = this.journalLastUncommitted; } } else if ( type === 'uncommitted' ) { - const newOrigin = vAPI.hostnameFromURI(url); + const newOrigin = hostnameFromURI(url); if ( this.journalLastUncommitted === -1 || this.journalLastUncommittedOrigin !== newOrigin @@ -807,7 +811,7 @@ const PageStore = class { // Static filtering has lowest precedence. const snfe = µb.staticNetFilteringEngine; if ( result === 0 || result === 3 ) { - result = snfe.matchString(fctxt); + result = snfe.matchRequest(fctxt); if ( result !== 0 ) { if ( loggerEnabled ) { fctxt.setFilter(snfe.toLogData()); @@ -912,7 +916,10 @@ const PageStore = class { } redirectBlockedRequest(fctxt) { - const directives = µb.staticNetFilteringEngine.redirectRequest(fctxt); + const directives = µb.staticNetFilteringEngine.redirectRequest( + µb.redirectEngine, + fctxt + ); if ( directives === undefined ) { return; } if ( µb.logger.enabled !== true ) { return; } fctxt.pushFilters(directives.map(a => a.logData())); @@ -1062,7 +1069,7 @@ const PageStore = class { } } if ( exceptCname === undefined ) { - const result = µb.staticNetFilteringEngine.matchStringReverse( + const result = µb.staticNetFilteringEngine.matchRequestReverse( 'cname', frameStore instanceof Object ? frameStore.rawURL @@ -1083,7 +1090,7 @@ const PageStore = class { } getBlockedResources(request, response) { - const normalURL = µb.normalizePageURL(this.tabId, request.frameURL); + const normalURL = µb.normalizeTabURL(this.tabId, request.frameURL); const resources = request.resources; const fctxt = µb.filteringContext; fctxt.fromTabId(this.tabId) @@ -1124,8 +1131,3 @@ PageStore.junkyardMax = 10; µb.PageStore = PageStore; /******************************************************************************/ - -// <<<<< -// end of private namespace - -} diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 59dae48c0..a07fddbe6 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -23,9 +23,9 @@ /******************************************************************************/ -µBlock.redirectEngine = (( ) => { +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; -/******************************************************************************/ /******************************************************************************/ // The resources referenced below are found in ./web_accessible_resources/ @@ -356,7 +356,7 @@ RedirectEngine.prototype.resourceContentFromName = function(name, mime) { // Append newlines to raw text to ensure processing of trailing resource. RedirectEngine.prototype.resourcesFromString = function(text) { - const lineIter = new µBlock.LineIterator( + const lineIter = new LineIterator( removeTopCommentBlock(text) + '\n\n' ); const reNonEmptyLine = /\S/; @@ -583,11 +583,10 @@ RedirectEngine.prototype.invalidateResourcesSelfie = function() { µBlock.assets.remove('compiled/redirectEngine/resources'); }; -/******************************************************************************/ /******************************************************************************/ -return new RedirectEngine(); +// Export + +µBlock.redirectEngine = new RedirectEngine(); /******************************************************************************/ - -})(); diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 52d79da8c..8549287a5 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -400,8 +400,8 @@ if ( if ( typeof rawFilter !== 'string' || rawFilter === '' ) { return; } const µb = µBlock; - const writer = new µb.CompiledLineIO.Writer(); - const parser = new vAPI.StaticFilteringParser(); + const writer = new µb.CompiledListWriter(); + const parser = new µb.StaticFilteringParser(); parser.setMaxTokenLength(µb.staticNetFilteringEngine.MAX_TOKEN_LENGTH); parser.analyze(rawFilter); @@ -435,20 +435,20 @@ if ( await initWorker(); const id = messageId++; - const hostname = µBlock.URI.hostnameFromURI(details.url); + const hostname = µBlock.hostnameFromURI(details.url); worker.postMessage({ what: 'fromCosmeticFilter', id: id, - domain: µBlock.URI.domainFromHostname(hostname), + domain: µBlock.domainFromHostname(hostname), hostname: hostname, ignoreGeneric: - µBlock.staticNetFilteringEngine.matchStringReverse( + µBlock.staticNetFilteringEngine.matchRequestReverse( 'generichide', details.url ) === 2, ignoreSpecific: - µBlock.staticNetFilteringEngine.matchStringReverse( + µBlock.staticNetFilteringEngine.matchRequestReverse( 'specifichide', details.url ) === 2, diff --git a/src/js/scriptlet-filtering.js b/src/js/scriptlet-filtering.js index ad41c508f..5da7a0b16 100644 --- a/src/js/scriptlet-filtering.js +++ b/src/js/scriptlet-filtering.js @@ -23,435 +23,447 @@ /******************************************************************************/ -µBlock.scriptletFilteringEngine = (function() { - const µb = µBlock; - const duplicates = new Set(); - const scriptletCache = new µb.MRUCache(32); - const reEscapeScriptArg = /[\\'"]/g; +import { + domainFromHostname, + entityFromDomain, + hostnameFromURI, +} from './uri-utils.js'; - const scriptletDB = new µb.staticExtFilteringEngine.HostnameBasedDB(1); - const sessionScriptletDB = new µb.staticExtFilteringEngine.SessionDB(); - - let acceptedCount = 0; - let discardedCount = 0; - - const api = { - get acceptedCount() { - return acceptedCount; - }, - get discardedCount() { - return discardedCount; - } - }; - - // Purpose of `contentscriptCode` below is too programmatically inject - // content script code which only purpose is to inject scriptlets. This - // essentially does the same as what uBO's declarative content script does, - // except that this allows to inject the scriptlets earlier than it is - // possible through the declarative content script. - // - // Declaratively: - // 1. Browser injects generic content script => - // 2. Content script queries scriptlets => - // 3. Main process sends scriptlets => - // 4. Content script injects scriptlets - // - // Programmatically: - // 1. uBO injects specific scriptlets-aware content script => - // 2. Content script injects scriptlets - // - // However currently this programmatic injection works well only on - // Chromium-based browsers, it does not work properly with Firefox. More - // investigations is needed to find out why this fails with Firefox. - // Consequently, the programmatic-injection code path is taken only with - // Chromium-based browsers. - - const contentscriptCode = (( ) => { - const parts = [ - '(', - function(hostname, scriptlets) { - if ( - document.location === null || - hostname !== document.location.hostname - ) { - return; - } - const injectScriptlets = function(d) { - let script; - try { - script = d.createElement('script'); - script.appendChild(d.createTextNode( - decodeURIComponent(scriptlets)) - ); - (d.head || d.documentElement).appendChild(script); - } catch (ex) { - } - if ( script ) { - if ( script.parentNode ) { - script.parentNode.removeChild(script); - } - script.textContent = ''; - } - }; - injectScriptlets(document); - }.toString(), - ')(', - '"', 'hostname-slot', '", ', - '"', 'scriptlets-slot', '"', - '); void 0;', - ]; - return { - parts: parts, - hostnameSlot: parts.indexOf('hostname-slot'), - scriptletsSlot: parts.indexOf('scriptlets-slot'), - assemble: function(hostname, scriptlets) { - this.parts[this.hostnameSlot] = hostname; - this.parts[this.scriptletsSlot] = - encodeURIComponent(scriptlets); - return this.parts.join(''); - } - }; - })(); - - // TODO: Probably should move this into StaticFilteringParser - // https://github.com/uBlockOrigin/uBlock-issues/issues/1031 - // Normalize scriptlet name to its canonical, unaliased name. - const normalizeRawFilter = function(rawFilter) { - const rawToken = rawFilter.slice(4, -1); - const rawEnd = rawToken.length; - let end = rawToken.indexOf(','); - if ( end === -1 ) { end = rawEnd; } - const token = rawToken.slice(0, end).trim(); - const alias = token.endsWith('.js') ? token.slice(0, -3) : token; - let normalized = µb.redirectEngine.aliases.get(`${alias}.js`); - normalized = normalized === undefined - ? alias - : normalized.slice(0, -3); - let beg = end + 1; - while ( beg < rawEnd ) { - end = rawToken.indexOf(',', beg); - if ( end === -1 ) { end = rawEnd; } - normalized += ', ' + rawToken.slice(beg, end).trim(); - beg = end + 1; - } - return `+js(${normalized})`; - }; - - const lookupScriptlet = function(rawToken, reng, toInject) { - if ( toInject.has(rawToken) ) { return; } - if ( scriptletCache.resetTime < reng.modifyTime ) { - scriptletCache.reset(); - } - let content = scriptletCache.lookup(rawToken); - if ( content === undefined ) { - const pos = rawToken.indexOf(','); - let token, args; - if ( pos === -1 ) { - token = rawToken; - } else { - token = rawToken.slice(0, pos).trim(); - args = rawToken.slice(pos + 1).trim(); - } - // TODO: The alias lookup can be removed once scriptlet resources - // with obsolete name are converted to their new name. - if ( reng.aliases.has(token) ) { - token = reng.aliases.get(token); - } else { - token = `${token}.js`; - } - content = reng.resourceContentFromName( - token, - 'application/javascript' - ); - if ( !content ) { return; } - if ( args ) { - content = patchScriptlet(content, args); - if ( !content ) { return; } - } - content = - 'try {\n' + - content + '\n' + - '} catch ( e ) { }'; - scriptletCache.add(rawToken, content); - } - toInject.set(rawToken, content); - }; - - // Fill-in scriptlet argument placeholders. - const patchScriptlet = function(content, args) { - let s = args; - let len = s.length; - let beg = 0, pos = 0; - let i = 1; - while ( beg < len ) { - pos = s.indexOf(',', pos); - // Escaped comma? If so, skip. - if ( pos > 0 && s.charCodeAt(pos - 1) === 0x5C /* '\\' */ ) { - s = s.slice(0, pos - 1) + s.slice(pos); - len -= 1; - continue; - } - if ( pos === -1 ) { pos = len; } - content = content.replace( - `{{${i}}}`, - s.slice(beg, pos).trim().replace(reEscapeScriptArg, '\\$&') - ); - beg = pos = pos + 1; - i++; - } - return content; - }; - - const logOne = function(isException, token, details) { - µBlock.filteringContext - .duplicate() - .fromTabId(details.tabId) - .setRealm('extended') - .setType('dom') - .setURL(details.url) - .setDocOriginFromURL(details.url) - .setFilter({ - source: 'extended', - raw: (isException ? '#@#' : '##') + `+js(${token})` - }) - .toLogger(); - }; - - api.reset = function() { - scriptletDB.clear(); - duplicates.clear(); - acceptedCount = 0; - discardedCount = 0; - }; - - api.freeze = function() { - duplicates.clear(); - scriptletDB.collectGarbage(); - }; - - api.compile = function(parser, writer) { - writer.select(µb.compiledScriptletSection); - - // Only exception filters are allowed to be global. - const { raw, exception } = parser.result; - const normalized = normalizeRawFilter(raw); - - // Tokenless is meaningful only for exception filters. - if ( normalized === '+js()' && exception === false ) { return; } - - if ( parser.hasOptions() === false ) { - if ( exception ) { - writer.push([ 32, '', 1, normalized ]); - } - return; - } - - // https://github.com/gorhill/uBlock/issues/3375 - // Ignore instances of exception filter with negated hostnames, - // because there is no way to create an exception to an exception. - - for ( const { hn, not, bad } of parser.extOptions() ) { - if ( bad ) { continue; } - let kind = 0; - if ( exception ) { - if ( not ) { continue; } - kind |= 1; - } else if ( not ) { - kind |= 1; - } - writer.push([ 32, hn, kind, normalized ]); - } - }; - - api.compileTemporary = function(parser) { - return { - session: sessionScriptletDB, - selector: parser.result.compiled, - }; - }; - - // 01234567890123456789 - // +js(token[, arg[, ...]]) - // ^ ^ - // 4 -1 - - api.fromCompiledContent = function(reader) { - reader.select(µb.compiledScriptletSection); - - while ( reader.next() ) { - acceptedCount += 1; - const fingerprint = reader.fingerprint(); - if ( duplicates.has(fingerprint) ) { - discardedCount += 1; - continue; - } - duplicates.add(fingerprint); - const args = reader.args(); - if ( args.length < 4 ) { continue; } - scriptletDB.store(args[1], args[2], args[3].slice(4, -1)); - } - }; - - api.getSession = function() { - return sessionScriptletDB; - }; - - const $scriptlets = new Set(); - const $exceptions = new Set(); - const $scriptletToCodeMap = new Map(); - - api.retrieve = function(request) { - if ( scriptletDB.size === 0 ) { return; } - - const reng = µb.redirectEngine; - if ( !reng ) { return; } - - const hostname = request.hostname; - - $scriptlets.clear(); - $exceptions.clear(); - - if ( sessionScriptletDB.isNotEmpty ) { - sessionScriptletDB.retrieve([ null, $exceptions ]); - } - scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]); - const entity = request.entity !== '' - ? `${hostname.slice(0, -request.domain.length)}${request.entity}` - : '*'; - scriptletDB.retrieve(entity, [ $scriptlets, $exceptions ], 1); - if ( $scriptlets.size === 0 ) { return; } - - // https://github.com/gorhill/uBlock/issues/2835 - // Do not inject scriptlets if the site is under an `allow` rule. - if ( - µb.userSettings.advancedUserEnabled && - µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 - ) { - return; - } - - const loggerEnabled = µb.logger.enabled; - - // Wholly disable scriptlet injection? - if ( $exceptions.has('') ) { - if ( loggerEnabled ) { - logOne(true, '', request); - } - return; - } - - $scriptletToCodeMap.clear(); - for ( const rawToken of $scriptlets ) { - lookupScriptlet(rawToken, reng, $scriptletToCodeMap); - } - if ( $scriptletToCodeMap.size === 0 ) { return; } - - // Return an array of scriptlets, and log results if needed. - const out = []; - for ( const [ rawToken, code ] of $scriptletToCodeMap ) { - const isException = $exceptions.has(rawToken); - if ( isException === false ) { - out.push(code); - } - if ( loggerEnabled ) { - logOne(isException, rawToken, request); - } - } - - if ( out.length === 0 ) { return; } - - if ( µb.hiddenSettings.debugScriptlets ) { - out.unshift('debugger;'); - } - - // https://github.com/uBlockOrigin/uBlock-issues/issues/156 - // Provide a private Map() object available for use by all - // scriptlets. - out.unshift( - '(function() {', - '// >>>> start of private namespace', - '' - ); - out.push( - '', - '// <<<< end of private namespace', - '})();' - ); - - return out.join('\n'); - }; - - api.hasScriptlet = function(hostname, exceptionBit, scriptlet) { - return scriptletDB.hasStr(hostname, exceptionBit, scriptlet); - }; - - api.injectNow = function(details) { - if ( typeof details.frameId !== 'number' ) { return; } - const request = { - tabId: details.tabId, - frameId: details.frameId, - url: details.url, - hostname: µb.URI.hostnameFromURI(details.url), - domain: undefined, - entity: undefined - }; - request.domain = µb.URI.domainFromHostname(request.hostname); - request.entity = µb.URI.entityFromDomain(request.domain); - const scriptlets = µb.scriptletFilteringEngine.retrieve(request); - if ( scriptlets === undefined ) { return; } - let code = contentscriptCode.assemble(request.hostname, scriptlets); - if ( µb.hiddenSettings.debugScriptletInjector ) { - code = 'debugger;\n' + code; - } - vAPI.tabs.executeScript(details.tabId, { - code, - frameId: details.frameId, - matchAboutBlank: true, - runAt: 'document_start', - }); - }; - - api.toSelfie = function() { - return scriptletDB.toSelfie(); - }; - - api.fromSelfie = function(selfie) { - scriptletDB.fromSelfie(selfie); - }; - - api.benchmark = async function() { - const requests = await µb.loadBenchmarkDataset(); - if ( Array.isArray(requests) === false || requests.length === 0 ) { - log.print('No requests found to benchmark'); - return; - } - log.print('Benchmarking scriptletFilteringEngine.retrieve()...'); - const details = { - domain: '', - entity: '', - hostname: '', - tabId: 0, - url: '', - }; - let count = 0; - const t0 = self.performance.now(); - for ( let i = 0; i < requests.length; i++ ) { - const request = requests[i]; - if ( request.cpt !== 'main_frame' ) { continue; } - count += 1; - details.url = request.url; - details.hostname = µb.URI.hostnameFromURI(request.url); - details.domain = µb.URI.domainFromHostname(details.hostname); - details.entity = µb.URI.entityFromDomain(details.domain); - void this.retrieve(details); - } - const t1 = self.performance.now(); - const dur = t1 - t0; - log.print(`Evaluated ${count} requests in ${dur.toFixed(0)} ms`); - log.print(`\tAverage: ${(dur / count).toFixed(3)} ms per request`); - }; - - return api; -})(); +import µBlock from './background.js'; + +/******************************************************************************/ + +const µb = µBlock; +const duplicates = new Set(); +const scriptletCache = new µb.MRUCache(32); +const reEscapeScriptArg = /[\\'"]/g; + +const scriptletDB = new µb.staticExtFilteringEngine.HostnameBasedDB(1); +const sessionScriptletDB = new µb.staticExtFilteringEngine.SessionDB(); + +let acceptedCount = 0; +let discardedCount = 0; + +const api = { + get acceptedCount() { + return acceptedCount; + }, + get discardedCount() { + return discardedCount; + } +}; + +// Purpose of `contentscriptCode` below is too programmatically inject +// content script code which only purpose is to inject scriptlets. This +// essentially does the same as what uBO's declarative content script does, +// except that this allows to inject the scriptlets earlier than it is +// possible through the declarative content script. +// +// Declaratively: +// 1. Browser injects generic content script => +// 2. Content script queries scriptlets => +// 3. Main process sends scriptlets => +// 4. Content script injects scriptlets +// +// Programmatically: +// 1. uBO injects specific scriptlets-aware content script => +// 2. Content script injects scriptlets +// +// However currently this programmatic injection works well only on +// Chromium-based browsers, it does not work properly with Firefox. More +// investigations is needed to find out why this fails with Firefox. +// Consequently, the programmatic-injection code path is taken only with +// Chromium-based browsers. + +const contentscriptCode = (( ) => { + const parts = [ + '(', + function(hostname, scriptlets) { + if ( + document.location === null || + hostname !== document.location.hostname + ) { + return; + } + const injectScriptlets = function(d) { + let script; + try { + script = d.createElement('script'); + script.appendChild(d.createTextNode( + decodeURIComponent(scriptlets)) + ); + (d.head || d.documentElement).appendChild(script); + } catch (ex) { + } + if ( script ) { + if ( script.parentNode ) { + script.parentNode.removeChild(script); + } + script.textContent = ''; + } + }; + injectScriptlets(document); + }.toString(), + ')(', + '"', 'hostname-slot', '", ', + '"', 'scriptlets-slot', '"', + '); void 0;', + ]; + return { + parts: parts, + hostnameSlot: parts.indexOf('hostname-slot'), + scriptletsSlot: parts.indexOf('scriptlets-slot'), + assemble: function(hostname, scriptlets) { + this.parts[this.hostnameSlot] = hostname; + this.parts[this.scriptletsSlot] = + encodeURIComponent(scriptlets); + return this.parts.join(''); + } + }; +})(); + +// TODO: Probably should move this into StaticFilteringParser +// https://github.com/uBlockOrigin/uBlock-issues/issues/1031 +// Normalize scriptlet name to its canonical, unaliased name. +const normalizeRawFilter = function(rawFilter) { + const rawToken = rawFilter.slice(4, -1); + const rawEnd = rawToken.length; + let end = rawToken.indexOf(','); + if ( end === -1 ) { end = rawEnd; } + const token = rawToken.slice(0, end).trim(); + const alias = token.endsWith('.js') ? token.slice(0, -3) : token; + let normalized = µb.redirectEngine.aliases.get(`${alias}.js`); + normalized = normalized === undefined + ? alias + : normalized.slice(0, -3); + let beg = end + 1; + while ( beg < rawEnd ) { + end = rawToken.indexOf(',', beg); + if ( end === -1 ) { end = rawEnd; } + normalized += ', ' + rawToken.slice(beg, end).trim(); + beg = end + 1; + } + return `+js(${normalized})`; +}; + +const lookupScriptlet = function(rawToken, reng, toInject) { + if ( toInject.has(rawToken) ) { return; } + if ( scriptletCache.resetTime < reng.modifyTime ) { + scriptletCache.reset(); + } + let content = scriptletCache.lookup(rawToken); + if ( content === undefined ) { + const pos = rawToken.indexOf(','); + let token, args; + if ( pos === -1 ) { + token = rawToken; + } else { + token = rawToken.slice(0, pos).trim(); + args = rawToken.slice(pos + 1).trim(); + } + // TODO: The alias lookup can be removed once scriptlet resources + // with obsolete name are converted to their new name. + if ( reng.aliases.has(token) ) { + token = reng.aliases.get(token); + } else { + token = `${token}.js`; + } + content = reng.resourceContentFromName( + token, + 'application/javascript' + ); + if ( !content ) { return; } + if ( args ) { + content = patchScriptlet(content, args); + if ( !content ) { return; } + } + content = + 'try {\n' + + content + '\n' + + '} catch ( e ) { }'; + scriptletCache.add(rawToken, content); + } + toInject.set(rawToken, content); +}; + +// Fill-in scriptlet argument placeholders. +const patchScriptlet = function(content, args) { + let s = args; + let len = s.length; + let beg = 0, pos = 0; + let i = 1; + while ( beg < len ) { + pos = s.indexOf(',', pos); + // Escaped comma? If so, skip. + if ( pos > 0 && s.charCodeAt(pos - 1) === 0x5C /* '\\' */ ) { + s = s.slice(0, pos - 1) + s.slice(pos); + len -= 1; + continue; + } + if ( pos === -1 ) { pos = len; } + content = content.replace( + `{{${i}}}`, + s.slice(beg, pos).trim().replace(reEscapeScriptArg, '\\$&') + ); + beg = pos = pos + 1; + i++; + } + return content; +}; + +const logOne = function(isException, token, details) { + µBlock.filteringContext + .duplicate() + .fromTabId(details.tabId) + .setRealm('extended') + .setType('dom') + .setURL(details.url) + .setDocOriginFromURL(details.url) + .setFilter({ + source: 'extended', + raw: (isException ? '#@#' : '##') + `+js(${token})` + }) + .toLogger(); +}; + +api.reset = function() { + scriptletDB.clear(); + duplicates.clear(); + acceptedCount = 0; + discardedCount = 0; +}; + +api.freeze = function() { + duplicates.clear(); + scriptletDB.collectGarbage(); +}; + +api.compile = function(parser, writer) { + writer.select(µb.compiledScriptletSection); + + // Only exception filters are allowed to be global. + const { raw, exception } = parser.result; + const normalized = normalizeRawFilter(raw); + + // Tokenless is meaningful only for exception filters. + if ( normalized === '+js()' && exception === false ) { return; } + + if ( parser.hasOptions() === false ) { + if ( exception ) { + writer.push([ 32, '', 1, normalized ]); + } + return; + } + + // https://github.com/gorhill/uBlock/issues/3375 + // Ignore instances of exception filter with negated hostnames, + // because there is no way to create an exception to an exception. + + for ( const { hn, not, bad } of parser.extOptions() ) { + if ( bad ) { continue; } + let kind = 0; + if ( exception ) { + if ( not ) { continue; } + kind |= 1; + } else if ( not ) { + kind |= 1; + } + writer.push([ 32, hn, kind, normalized ]); + } +}; + +api.compileTemporary = function(parser) { + return { + session: sessionScriptletDB, + selector: parser.result.compiled, + }; +}; + +// 01234567890123456789 +// +js(token[, arg[, ...]]) +// ^ ^ +// 4 -1 + +api.fromCompiledContent = function(reader) { + reader.select(µb.compiledScriptletSection); + + while ( reader.next() ) { + acceptedCount += 1; + const fingerprint = reader.fingerprint(); + if ( duplicates.has(fingerprint) ) { + discardedCount += 1; + continue; + } + duplicates.add(fingerprint); + const args = reader.args(); + if ( args.length < 4 ) { continue; } + scriptletDB.store(args[1], args[2], args[3].slice(4, -1)); + } +}; + +api.getSession = function() { + return sessionScriptletDB; +}; + +const $scriptlets = new Set(); +const $exceptions = new Set(); +const $scriptletToCodeMap = new Map(); + +api.retrieve = function(request) { + if ( scriptletDB.size === 0 ) { return; } + + const reng = µb.redirectEngine; + if ( !reng ) { return; } + + const hostname = request.hostname; + + $scriptlets.clear(); + $exceptions.clear(); + + if ( sessionScriptletDB.isNotEmpty ) { + sessionScriptletDB.retrieve([ null, $exceptions ]); + } + scriptletDB.retrieve(hostname, [ $scriptlets, $exceptions ]); + const entity = request.entity !== '' + ? `${hostname.slice(0, -request.domain.length)}${request.entity}` + : '*'; + scriptletDB.retrieve(entity, [ $scriptlets, $exceptions ], 1); + if ( $scriptlets.size === 0 ) { return; } + + // https://github.com/gorhill/uBlock/issues/2835 + // Do not inject scriptlets if the site is under an `allow` rule. + if ( + µb.userSettings.advancedUserEnabled && + µb.sessionFirewall.evaluateCellZY(hostname, hostname, '*') === 2 + ) { + return; + } + + const loggerEnabled = µb.logger.enabled; + + // Wholly disable scriptlet injection? + if ( $exceptions.has('') ) { + if ( loggerEnabled ) { + logOne(true, '', request); + } + return; + } + + $scriptletToCodeMap.clear(); + for ( const rawToken of $scriptlets ) { + lookupScriptlet(rawToken, reng, $scriptletToCodeMap); + } + if ( $scriptletToCodeMap.size === 0 ) { return; } + + // Return an array of scriptlets, and log results if needed. + const out = []; + for ( const [ rawToken, code ] of $scriptletToCodeMap ) { + const isException = $exceptions.has(rawToken); + if ( isException === false ) { + out.push(code); + } + if ( loggerEnabled ) { + logOne(isException, rawToken, request); + } + } + + if ( out.length === 0 ) { return; } + + if ( µb.hiddenSettings.debugScriptlets ) { + out.unshift('debugger;'); + } + + // https://github.com/uBlockOrigin/uBlock-issues/issues/156 + // Provide a private Map() object available for use by all + // scriptlets. + out.unshift( + '(function() {', + '// >>>> start of private namespace', + '' + ); + out.push( + '', + '// <<<< end of private namespace', + '})();' + ); + + return out.join('\n'); +}; + +api.hasScriptlet = function(hostname, exceptionBit, scriptlet) { + return scriptletDB.hasStr(hostname, exceptionBit, scriptlet); +}; + +api.injectNow = function(details) { + if ( typeof details.frameId !== 'number' ) { return; } + const request = { + tabId: details.tabId, + frameId: details.frameId, + url: details.url, + hostname: hostnameFromURI(details.url), + domain: undefined, + entity: undefined + }; + request.domain = domainFromHostname(request.hostname); + request.entity = entityFromDomain(request.domain); + const scriptlets = µb.scriptletFilteringEngine.retrieve(request); + if ( scriptlets === undefined ) { return; } + let code = contentscriptCode.assemble(request.hostname, scriptlets); + if ( µb.hiddenSettings.debugScriptletInjector ) { + code = 'debugger;\n' + code; + } + vAPI.tabs.executeScript(details.tabId, { + code, + frameId: details.frameId, + matchAboutBlank: true, + runAt: 'document_start', + }); +}; + +api.toSelfie = function() { + return scriptletDB.toSelfie(); +}; + +api.fromSelfie = function(selfie) { + scriptletDB.fromSelfie(selfie); +}; + +api.benchmark = async function() { + const requests = await µb.loadBenchmarkDataset(); + if ( Array.isArray(requests) === false || requests.length === 0 ) { + log.print('No requests found to benchmark'); + return; + } + log.print('Benchmarking scriptletFilteringEngine.retrieve()...'); + const details = { + domain: '', + entity: '', + hostname: '', + tabId: 0, + url: '', + }; + let count = 0; + const t0 = self.performance.now(); + for ( let i = 0; i < requests.length; i++ ) { + const request = requests[i]; + if ( request.cpt !== 'main_frame' ) { continue; } + count += 1; + details.url = request.url; + details.hostname = hostnameFromURI(request.url); + details.domain = domainFromHostname(details.hostname); + details.entity = entityFromDomain(details.domain); + void this.retrieve(details); + } + const t1 = self.performance.now(); + const dur = t1 - t0; + log.print(`Evaluated ${count} requests in ${dur.toFixed(0)} ms`); + log.print(`\tAverage: ${(dur / count).toFixed(3)} ms per request`); +}; + +/******************************************************************************/ + +// Export + +µBlock.scriptletFilteringEngine = api; /******************************************************************************/ diff --git a/src/js/start.js b/src/js/start.js index 3d48e903a..45b185435 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -23,6 +23,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + // Load all: executed once. (async ( ) => { @@ -33,7 +37,6 @@ const µb = µBlock; /******************************************************************************/ vAPI.app.onShutdown = function() { - const µb = µBlock; µb.staticFilteringReverseLookup.shutdown(); µb.assets.updateStop(); µb.staticNetFilteringEngine.reset(); @@ -317,7 +320,7 @@ try { } if ( µb.hiddenSettings.disableWebAssembly !== true ) { - µb.staticNetFilteringEngine.enableWASM().then(( ) => { + µb.staticNetFilteringEngine.enableWASM('/js').then(( ) => { log.info(`WASM modules ready ${Date.now()-vAPI.T0} ms after launch`); }); } @@ -445,7 +448,7 @@ if ( browser.runtime.onUpdateAvailable.addListener(details => { const toInt = vAPI.app.intFromVersion; if ( - µBlock.hiddenSettings.extensionUpdateForceReload === true || + µb.hiddenSettings.extensionUpdateForceReload === true || toInt(details.version) <= toInt(vAPI.app.version) ) { vAPI.app.restart(); diff --git a/src/js/static-ext-filtering.js b/src/js/static-ext-filtering.js index b9b9f2fd2..04b86795c 100644 --- a/src/js/static-ext-filtering.js +++ b/src/js/static-ext-filtering.js @@ -21,6 +21,10 @@ 'use strict'; +/******************************************************************************/ + +import µBlock from './background.js'; + /******************************************************************************* All static extended filters are of the form: @@ -48,326 +52,328 @@ **/ -µBlock.staticExtFilteringEngine = (( ) => { - const µb = µBlock; +const µb = µBlock; - //-------------------------------------------------------------------------- - // Public API - //-------------------------------------------------------------------------- +//-------------------------------------------------------------------------- +// Public API +//-------------------------------------------------------------------------- - const api = { - get acceptedCount() { - return µb.cosmeticFilteringEngine.acceptedCount + - µb.scriptletFilteringEngine.acceptedCount + - µb.httpheaderFilteringEngine.acceptedCount + - µb.htmlFilteringEngine.acceptedCount; - }, - get discardedCount() { - return µb.cosmeticFilteringEngine.discardedCount + - µb.scriptletFilteringEngine.discardedCount + - µb.httpheaderFilteringEngine.discardedCount + - µb.htmlFilteringEngine.discardedCount; - }, - }; +const api = { + get acceptedCount() { + return µb.cosmeticFilteringEngine.acceptedCount + + µb.scriptletFilteringEngine.acceptedCount + + µb.httpheaderFilteringEngine.acceptedCount + + µb.htmlFilteringEngine.acceptedCount; + }, + get discardedCount() { + return µb.cosmeticFilteringEngine.discardedCount + + µb.scriptletFilteringEngine.discardedCount + + µb.httpheaderFilteringEngine.discardedCount + + µb.htmlFilteringEngine.discardedCount; + }, +}; - //-------------------------------------------------------------------------- - // Public classes - //-------------------------------------------------------------------------- +//-------------------------------------------------------------------------- +// Public classes +//-------------------------------------------------------------------------- - api.HostnameBasedDB = class { - constructor(nBits, selfie = undefined) { - this.nBits = nBits; - this.timer = undefined; - this.strToIdMap = new Map(); - this.hostnameToSlotIdMap = new Map(); - // Array of integer pairs - this.hostnameSlots = []; - // Array of strings (selectors and pseudo-selectors) - this.strSlots = []; - this.size = 0; - if ( selfie !== undefined ) { - this.fromSelfie(selfie); +api.HostnameBasedDB = class { + constructor(nBits, selfie = undefined) { + this.nBits = nBits; + this.timer = undefined; + this.strToIdMap = new Map(); + this.hostnameToSlotIdMap = new Map(); + // Array of integer pairs + this.hostnameSlots = []; + // Array of strings (selectors and pseudo-selectors) + this.strSlots = []; + this.size = 0; + if ( selfie !== undefined ) { + this.fromSelfie(selfie); + } + } + + store(hn, bits, s) { + this.size += 1; + let iStr = this.strToIdMap.get(s); + if ( iStr === undefined ) { + iStr = this.strSlots.length; + this.strSlots.push(s); + this.strToIdMap.set(s, iStr); + if ( this.timer === undefined ) { + this.collectGarbage(true); } } - - store(hn, bits, s) { - this.size += 1; - let iStr = this.strToIdMap.get(s); - if ( iStr === undefined ) { - iStr = this.strSlots.length; - this.strSlots.push(s); - this.strToIdMap.set(s, iStr); - if ( this.timer === undefined ) { - this.collectGarbage(true); - } - } - const strId = iStr << this.nBits | bits; - let iHn = this.hostnameToSlotIdMap.get(hn); - if ( iHn === undefined ) { - this.hostnameToSlotIdMap.set(hn, this.hostnameSlots.length); - this.hostnameSlots.push(strId, 0); - return; - } - // Add as last item. - while ( this.hostnameSlots[iHn+1] !== 0 ) { - iHn = this.hostnameSlots[iHn+1]; - } - this.hostnameSlots[iHn+1] = this.hostnameSlots.length; + const strId = iStr << this.nBits | bits; + let iHn = this.hostnameToSlotIdMap.get(hn); + if ( iHn === undefined ) { + this.hostnameToSlotIdMap.set(hn, this.hostnameSlots.length); this.hostnameSlots.push(strId, 0); + return; } - - clear() { - this.hostnameToSlotIdMap.clear(); - this.hostnameSlots.length = 0; - this.strSlots.length = 0; - this.strToIdMap.clear(); - this.size = 0; + // Add as last item. + while ( this.hostnameSlots[iHn+1] !== 0 ) { + iHn = this.hostnameSlots[iHn+1]; } + this.hostnameSlots[iHn+1] = this.hostnameSlots.length; + this.hostnameSlots.push(strId, 0); + } - collectGarbage(later = false) { - if ( later === false ) { - if ( this.timer !== undefined ) { - self.cancelIdleCallback(this.timer); - this.timer = undefined; - } - this.strToIdMap.clear(); - return; + clear() { + this.hostnameToSlotIdMap.clear(); + this.hostnameSlots.length = 0; + this.strSlots.length = 0; + this.strToIdMap.clear(); + this.size = 0; + } + + collectGarbage(later = false) { + if ( later === false ) { + if ( this.timer !== undefined ) { + self.cancelIdleCallback(this.timer); + this.timer = undefined; } - if ( this.timer !== undefined ) { return; } - this.timer = self.requestIdleCallback( - ( ) => { - this.timer = undefined; - this.strToIdMap.clear(); - }, - { timeout: 5000 } - ); + this.strToIdMap.clear(); + return; } + if ( this.timer !== undefined ) { return; } + this.timer = self.requestIdleCallback( + ( ) => { + this.timer = undefined; + this.strToIdMap.clear(); + }, + { timeout: 5000 } + ); + } - // modifiers = 1: return only specific items - // modifiers = 2: return only generic items - // - retrieve(hostname, out, modifiers = 0) { - if ( modifiers === 2 ) { + // modifiers = 1: return only specific items + // modifiers = 2: return only generic items + // + retrieve(hostname, out, modifiers = 0) { + if ( modifiers === 2 ) { + hostname = ''; + } + const mask = out.length - 1; // out.length must be power of two + for (;;) { + let iHn = this.hostnameToSlotIdMap.get(hostname); + if ( iHn !== undefined ) { + do { + const strId = this.hostnameSlots[iHn+0]; + out[strId & mask].add( + this.strSlots[strId >>> this.nBits] + ); + iHn = this.hostnameSlots[iHn+1]; + } while ( iHn !== 0 ); + } + if ( hostname === '' ) { break; } + const pos = hostname.indexOf('.'); + if ( pos === -1 ) { + if ( modifiers === 1 ) { break; } + hostname = ''; + } else { + hostname = hostname.slice(pos + 1); + } + } + } + + hasStr(hostname, exceptionBit, value) { + let found = false; + for (;;) { + let iHn = this.hostnameToSlotIdMap.get(hostname); + if ( iHn !== undefined ) { + do { + const strId = this.hostnameSlots[iHn+0]; + const str = this.strSlots[strId >>> this.nBits]; + if ( (strId & exceptionBit) !== 0 ) { + if ( str === value || str === '' ) { return false; } + } + if ( str === value ) { found = true; } + iHn = this.hostnameSlots[iHn+1]; + } while ( iHn !== 0 ); + } + if ( hostname === '' ) { break; } + const pos = hostname.indexOf('.'); + if ( pos !== -1 ) { + hostname = hostname.slice(pos + 1); + } else if ( hostname !== '*' ) { + hostname = '*'; + } else { hostname = ''; } - const mask = out.length - 1; // out.length must be power of two - for (;;) { - let iHn = this.hostnameToSlotIdMap.get(hostname); - if ( iHn !== undefined ) { - do { - const strId = this.hostnameSlots[iHn+0]; - out[strId & mask].add( - this.strSlots[strId >>> this.nBits] - ); - iHn = this.hostnameSlots[iHn+1]; - } while ( iHn !== 0 ); - } - if ( hostname === '' ) { break; } - const pos = hostname.indexOf('.'); - if ( pos === -1 ) { - if ( modifiers === 1 ) { break; } - hostname = ''; - } else { - hostname = hostname.slice(pos + 1); - } + } + return found; + } + + toSelfie() { + return { + hostnameToSlotIdMap: Array.from(this.hostnameToSlotIdMap), + hostnameSlots: this.hostnameSlots, + strSlots: this.strSlots, + size: this.size + }; + } + + fromSelfie(selfie) { + if ( selfie === undefined ) { return; } + this.hostnameToSlotIdMap = new Map(selfie.hostnameToSlotIdMap); + this.hostnameSlots = selfie.hostnameSlots; + this.strSlots = selfie.strSlots; + this.size = selfie.size; + } +}; + +api.SessionDB = class { + constructor() { + this.db = new Map(); + } + compile(s) { + return s; + } + add(bits, s) { + const bucket = this.db.get(bits); + if ( bucket === undefined ) { + this.db.set(bits, new Set([ s ])); + } else { + bucket.add(s); + } + } + remove(bits, s) { + const bucket = this.db.get(bits); + if ( bucket === undefined ) { return; } + bucket.delete(s); + if ( bucket.size !== 0 ) { return; } + this.db.delete(bits); + } + retrieve(out) { + const mask = out.length - 1; + for ( const [ bits, bucket ] of this.db ) { + const i = bits & mask; + if ( out[i] instanceof Object === false ) { continue; } + for ( const s of bucket ) { + out[i].add(s); } } + } + has(bits, s) { + const selectors = this.db.get(bits); + return selectors !== undefined && selectors.has(s); + } + clear() { + this.db.clear(); + } + get isNotEmpty() { + return this.db.size !== 0; + } +}; - hasStr(hostname, exceptionBit, value) { - let found = false; - for (;;) { - let iHn = this.hostnameToSlotIdMap.get(hostname); - if ( iHn !== undefined ) { - do { - const strId = this.hostnameSlots[iHn+0]; - const str = this.strSlots[strId >>> this.nBits]; - if ( (strId & exceptionBit) !== 0 ) { - if ( str === value || str === '' ) { return false; } - } - if ( str === value ) { found = true; } - iHn = this.hostnameSlots[iHn+1]; - } while ( iHn !== 0 ); - } - if ( hostname === '' ) { break; } - const pos = hostname.indexOf('.'); - if ( pos !== -1 ) { - hostname = hostname.slice(pos + 1); - } else if ( hostname !== '*' ) { - hostname = '*'; - } else { - hostname = ''; - } - } - return found; - } +//-------------------------------------------------------------------------- +// Public methods +//-------------------------------------------------------------------------- - toSelfie() { - return { - hostnameToSlotIdMap: Array.from(this.hostnameToSlotIdMap), - hostnameSlots: this.hostnameSlots, - strSlots: this.strSlots, - size: this.size - }; - } +api.reset = function() { + µb.cosmeticFilteringEngine.reset(); + µb.scriptletFilteringEngine.reset(); + µb.httpheaderFilteringEngine.reset(); + µb.htmlFilteringEngine.reset(); +}; - fromSelfie(selfie) { - if ( selfie === undefined ) { return; } - this.hostnameToSlotIdMap = new Map(selfie.hostnameToSlotIdMap); - this.hostnameSlots = selfie.hostnameSlots; - this.strSlots = selfie.strSlots; - this.size = selfie.size; - } - }; +api.freeze = function() { + µb.cosmeticFilteringEngine.freeze(); + µb.scriptletFilteringEngine.freeze(); + µb.httpheaderFilteringEngine.freeze(); + µb.htmlFilteringEngine.freeze(); +}; - api.SessionDB = class { - constructor() { - this.db = new Map(); - } - compile(s) { - return s; - } - add(bits, s) { - const bucket = this.db.get(bits); - if ( bucket === undefined ) { - this.db.set(bits, new Set([ s ])); - } else { - bucket.add(s); - } - } - remove(bits, s) { - const bucket = this.db.get(bits); - if ( bucket === undefined ) { return; } - bucket.delete(s); - if ( bucket.size !== 0 ) { return; } - this.db.delete(bits); - } - retrieve(out) { - const mask = out.length - 1; - for ( const [ bits, bucket ] of this.db ) { - const i = bits & mask; - if ( out[i] instanceof Object === false ) { continue; } - for ( const s of bucket ) { - out[i].add(s); - } - } - } - has(bits, s) { - const selectors = this.db.get(bits); - return selectors !== undefined && selectors.has(s); - } - clear() { - this.db.clear(); - } - get isNotEmpty() { - return this.db.size !== 0; - } - }; +api.compile = function(parser, writer) { + if ( parser.category !== parser.CATStaticExtFilter ) { return false; } - //-------------------------------------------------------------------------- - // Public methods - //-------------------------------------------------------------------------- - - api.reset = function() { - µb.cosmeticFilteringEngine.reset(); - µb.scriptletFilteringEngine.reset(); - µb.httpheaderFilteringEngine.reset(); - µb.htmlFilteringEngine.reset(); - }; - - api.freeze = function() { - µb.cosmeticFilteringEngine.freeze(); - µb.scriptletFilteringEngine.freeze(); - µb.httpheaderFilteringEngine.freeze(); - µb.htmlFilteringEngine.freeze(); - }; - - api.compile = function(parser, writer) { - if ( parser.category !== parser.CATStaticExtFilter ) { return false; } - - if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) { - const who = writer.properties.get('assetKey') || '?'; - µb.logger.writeOne({ - realm: 'message', - type: 'error', - text: `Invalid extended filter in ${who}: ${parser.raw}` - }); - return true; - } - - // Scriptlet injection - if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { - µb.scriptletFilteringEngine.compile(parser, writer); - return true; - } - - // Response header filtering - if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { - µb.httpheaderFilteringEngine.compile(parser, writer); - return true; - } - - // HTML filtering - // TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML - // filtering syntax. - if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { - µb.htmlFilteringEngine.compile(parser, writer); - return true; - } - - // Cosmetic filtering - µb.cosmeticFilteringEngine.compile(parser, writer); - return true; - }; - - api.compileTemporary = function(parser) { - if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { - return µb.scriptletFilteringEngine.compileTemporary(parser); - } - if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { - return µb.httpheaderFilteringEngine.compileTemporary(parser); - } - if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { - return µb.htmlFilteringEngine.compileTemporary(parser); - } - return µb.cosmeticFilteringEngine.compileTemporary(parser); - }; - - api.fromCompiledContent = function(reader, options) { - µb.cosmeticFilteringEngine.fromCompiledContent(reader, options); - µb.scriptletFilteringEngine.fromCompiledContent(reader, options); - µb.httpheaderFilteringEngine.fromCompiledContent(reader, options); - µb.htmlFilteringEngine.fromCompiledContent(reader, options); - }; - - api.toSelfie = function(path) { - return µBlock.assets.put( - `${path}/main`, - JSON.stringify({ - cosmetic: µb.cosmeticFilteringEngine.toSelfie(), - scriptlets: µb.scriptletFilteringEngine.toSelfie(), - httpHeaders: µb.httpheaderFilteringEngine.toSelfie(), - html: µb.htmlFilteringEngine.toSelfie(), - }) - ); - }; - - api.fromSelfie = function(path) { - return µBlock.assets.get(`${path}/main`).then(details => { - let selfie; - try { - selfie = JSON.parse(details.content); - } catch (ex) { - } - if ( selfie instanceof Object === false ) { return false; } - µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); - µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); - µb.httpheaderFilteringEngine.fromSelfie(selfie.httpHeaders); - µb.htmlFilteringEngine.fromSelfie(selfie.html); - return true; + if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) { + const who = writer.properties.get('name') || '?'; + µb.logger.writeOne({ + realm: 'message', + type: 'error', + text: `Invalid extended filter in ${who}: ${parser.raw}` }); - }; + return true; + } - return api; -})(); + // Scriptlet injection + if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { + µb.scriptletFilteringEngine.compile(parser, writer); + return true; + } + + // Response header filtering + if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { + µb.httpheaderFilteringEngine.compile(parser, writer); + return true; + } + + // HTML filtering + // TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML + // filtering syntax. + if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { + µb.htmlFilteringEngine.compile(parser, writer); + return true; + } + + // Cosmetic filtering + µb.cosmeticFilteringEngine.compile(parser, writer); + return true; +}; + +api.compileTemporary = function(parser) { + if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) { + return µb.scriptletFilteringEngine.compileTemporary(parser); + } + if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) { + return µb.httpheaderFilteringEngine.compileTemporary(parser); + } + if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) { + return µb.htmlFilteringEngine.compileTemporary(parser); + } + return µb.cosmeticFilteringEngine.compileTemporary(parser); +}; + +api.fromCompiledContent = function(reader, options) { + µb.cosmeticFilteringEngine.fromCompiledContent(reader, options); + µb.scriptletFilteringEngine.fromCompiledContent(reader, options); + µb.httpheaderFilteringEngine.fromCompiledContent(reader, options); + µb.htmlFilteringEngine.fromCompiledContent(reader, options); +}; + +api.toSelfie = function(path) { + return µBlock.assets.put( + `${path}/main`, + JSON.stringify({ + cosmetic: µb.cosmeticFilteringEngine.toSelfie(), + scriptlets: µb.scriptletFilteringEngine.toSelfie(), + httpHeaders: µb.httpheaderFilteringEngine.toSelfie(), + html: µb.htmlFilteringEngine.toSelfie(), + }) + ); +}; + +api.fromSelfie = function(path) { + return µBlock.assets.get(`${path}/main`).then(details => { + let selfie; + try { + selfie = JSON.parse(details.content); + } catch (ex) { + } + if ( selfie instanceof Object === false ) { return false; } + µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmetic); + µb.scriptletFilteringEngine.fromSelfie(selfie.scriptlets); + µb.httpheaderFilteringEngine.fromSelfie(selfie.httpHeaders); + µb.htmlFilteringEngine.fromSelfie(selfie.html); + return true; + }); +}; + +/******************************************************************************/ + +// Export + +µBlock.staticExtFilteringEngine = api; /******************************************************************************/ diff --git a/src/js/static-filtering-io.js b/src/js/static-filtering-io.js new file mode 100644 index 000000000..5240c3bc2 --- /dev/null +++ b/src/js/static-filtering-io.js @@ -0,0 +1,147 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +// https://www.reddit.com/r/uBlockOrigin/comments/oq6kt5/ubo_loads_generic_filter_instead_of_specific/ +// Ensure blocks of content are sorted in ascending id order, such that the +// specific cosmetic filters will be found (and thus reported) before the +// generic ones. + +const serialize = JSON.stringify; +const unserialize = JSON.parse; + +const blockStartPrefix = '#block-start-'; // ensure no special regex characters +const blockEndPrefix = '#block-end-'; // ensure no special regex characters + +class CompiledListWriter { + constructor() { + this.blockId = undefined; + this.block = undefined; + this.blocks = new Map(); + this.properties = new Map(); + } + push(args) { + this.block.push(serialize(args)); + } + last() { + if ( Array.isArray(this.block) && this.block.length !== 0 ) { + return this.block[this.block.length - 1]; + } + } + select(blockId) { + if ( blockId === this.blockId ) { return; } + this.blockId = blockId; + this.block = this.blocks.get(blockId); + if ( this.block === undefined ) { + this.blocks.set(blockId, (this.block = [])); + } + return this; + } + toString() { + const result = []; + const sortedBlocks = + Array.from(this.blocks).sort((a, b) => a[0] - b[0]); + for ( const [ id, lines ] of sortedBlocks ) { + if ( lines.length === 0 ) { continue; } + result.push( + blockStartPrefix + id, + lines.join('\n'), + blockEndPrefix + id + ); + } + return result.join('\n'); + } + static serialize(arg) { + return serialize(arg); + } +} + +class CompiledListReader { + constructor(raw, blockId) { + this.block = ''; + this.len = 0; + this.offset = 0; + this.line = ''; + this.blocks = new Map(); + this.properties = new Map(); + const reBlockStart = new RegExp(`^${blockStartPrefix}(\\d+)\\n`, 'gm'); + let match = reBlockStart.exec(raw); + while ( match !== null ) { + let beg = match.index + match[0].length; + let end = raw.indexOf(blockEndPrefix + match[1], beg); + this.blocks.set(parseInt(match[1], 10), raw.slice(beg, end)); + reBlockStart.lastIndex = end; + match = reBlockStart.exec(raw); + } + if ( blockId !== undefined ) { + this.select(blockId); + } + } + next() { + if ( this.offset === this.len ) { + this.line = ''; + return false; + } + let pos = this.block.indexOf('\n', this.offset); + if ( pos !== -1 ) { + this.line = this.block.slice(this.offset, pos); + this.offset = pos + 1; + } else { + this.line = this.block.slice(this.offset); + this.offset = this.len; + } + return true; + } + select(blockId) { + this.block = this.blocks.get(blockId) || ''; + this.len = this.block.length; + this.offset = 0; + return this; + } + fingerprint() { + return this.line; + } + args() { + return unserialize(this.line); + } + static unserialize(arg) { + return unserialize(arg); + } +} + +CompiledListWriter.prototype.NETWORK_SECTION = +CompiledListReader.prototype.NETWORK_SECTION = 100; + +CompiledListWriter.blockStartPrefix = +CompiledListReader.blockStartPrefix = blockStartPrefix; + +CompiledListWriter.blockEndPrefix = +CompiledListReader.blockEndPrefix = blockEndPrefix; + +/******************************************************************************/ + +export { + CompiledListReader, + CompiledListWriter, +}; diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index c4112e92f..a07d0bd73 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -21,6 +21,12 @@ 'use strict'; +/******************************************************************************/ + +import '../lib/regexanalyzer/regex.js'; + +import globals from './globals.js'; + /******************************************************************************* The goal is for the static filtering parser to avoid external @@ -66,9 +72,6 @@ **/ -{ -// >>>>> start of local scope - /******************************************************************************/ const Parser = class { @@ -116,7 +119,7 @@ const Parser = class { // https://github.com/uBlockOrigin/uBlock-issues/issues/1146 // From https://codemirror.net/doc/manual.html#option_specialChars this.reInvalidCharacters = /[\x00-\x1F\x7F-\x9F\xAD\u061C\u200B-\u200F\u2028\u2029\uFEFF\uFFF9-\uFFFC]/; - this.punycoder = new URL(self.location); + this.punycoder = new URL(globals.location); // TODO: mind maxTokenLength this.reGoodRegexToken = /[^\x01%0-9A-Za-z][%0-9A-Za-z]{7,}|[^\x01%0-9A-Za-z][%0-9A-Za-z]{1,6}[^\x01%0-9A-Za-z]/; @@ -1299,7 +1302,11 @@ Parser.prototype.SelectorCompiler = class { [ 'matches-css-before', ':matches-css-before' ], ]); this.reSimpleSelector = /^[#.][A-Za-z_][\w-]*$/; - this.div = document.createElement('div'); + this.div = (( ) => { + if ( typeof document !== 'object' ) { return null; } + if ( document instanceof Object === false ) { return null; } + return document.createElement('div'); + })(); this.rePseudoElement = /:(?::?after|:?before|:-?[a-z][a-z-]*[a-z])$/; this.reProceduralOperator = new RegExp([ '^(?:', @@ -1424,6 +1431,7 @@ Parser.prototype.SelectorCompiler = class { if ( pos !== -1 ) { return this.cssSelectorType(s.slice(0, pos)) === 1 ? 3 : 0; } + if ( this.div === null ) { return 1; } try { this.div.matches(`${s}, ${s}:not(#foo)`); } catch (ex) { @@ -1533,6 +1541,7 @@ Parser.prototype.SelectorCompiler = class { // https://github.com/uBlockOrigin/uBlock-issues/issues/668 compileStyleProperties(s) { if ( /url\(|\\/i.test(s) ) { return; } + if ( this.div === null ) { return s; } this.div.style.cssText = s; if ( this.div.style.cssText === '' ) { return; } this.div.style.cssText = ''; @@ -2874,7 +2883,7 @@ Parser.regexUtils = Parser.prototype.regexUtils = (( ) => { return '\x01'; }; - const Regex = self.Regex; + const Regex = globals.Regex; if ( Regex instanceof Object === false || Regex.Analyzer instanceof Object === false @@ -2914,13 +2923,6 @@ Parser.regexUtils = Parser.prototype.regexUtils = (( ) => { /******************************************************************************/ -if ( typeof vAPI === 'object' && vAPI !== null ) { - vAPI.StaticFilteringParser = Parser; -} else { - self.StaticFilteringParser = Parser; -} +const StaticFilteringParser = Parser; -/******************************************************************************/ - -// <<<<< end of local scope -} +export { StaticFilteringParser }; diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 81d282f5b..f71844ed6 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -23,11 +23,38 @@ /******************************************************************************/ -µBlock.staticNetFilteringEngine = (( ) => { +import globals from './globals.js'; +import { sparseBase64 } from './base64-custom.js'; +import { BidiTrieContainer } from './biditrie.js'; +import { HNTrieContainer } from './hntrie.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; +import { CompiledListReader } from './static-filtering-io.js'; + +import { + domainFromHostname, + hostnameFromNetworkURL, +} from './uri-utils.js'; + + +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility +// +// This import would be best done dynamically, but since dynamic imports are +// not supported by older browsers, for now a static import is necessary. +import { FilteringContext } from './filtering-context.js'; /******************************************************************************/ -const µb = µBlock; +// Access to a key-val store is optional and useful only for optimal +// initialization at module load time. Probably could re-arrange code +// to export an init() function with optimization parameters which would +// need to be called by module clients. For now, I want modularizing with +// minimal amount of changes. + +const keyvalStore = typeof vAPI !== 'undefined' + ? vAPI.localStorage + : { getItem() { return null; }, setItem() {} }; + +/******************************************************************************/ // fedcba9876543210 // || | || | @@ -41,7 +68,7 @@ const µb = µBlock; // |+-------------- bit 10: headers-based filters // +--------------- bit 11-15: unused -const CategoryCount = 1 << 0xb; // shift left to first unused bit +const CategoryCount = 1 << 0xb; // shift left to first unused bit const RealmBitsMask = 0b00000000111; const ActionBitsMask = 0b00000000011; @@ -57,7 +84,7 @@ const AnyParty = 0b00000000000; const FirstParty = 0b00000001000; const ThirdParty = 0b00000010000; const AllParties = 0b00000011000; -const Headers = 0b10000000000; +const HEADERS = 0b10000000000; const typeNameToTypeValue = { 'no_type': 0 << TypeBitsOffset, @@ -141,6 +168,8 @@ const typeValueToTypeName = [ const MAX_TOKEN_LENGTH = 7; +const COMPILED_BAD_SECTION = 1; + /******************************************************************************/ // See the following as short-lived registers, used during evaluation. They are @@ -365,14 +394,14 @@ const bidiTrieMatchExtra = function(l, r, ix) { return 0; }; -const bidiTrie = new µb.BidiTrieContainer(bidiTrieMatchExtra); +const bidiTrie = new BidiTrieContainer(bidiTrieMatchExtra); const bidiTriePrime = function() { - bidiTrie.reset(vAPI.localStorage.getItem('SNFE.bidiTrie')); + bidiTrie.reset(keyvalStore.getItem('SNFE.bidiTrie')); }; const bidiTrieOptimize = function(shrink = false) { - vAPI.localStorage.setItem('SNFE.bidiTrie', bidiTrie.optimize(shrink)); + keyvalStore.setItem('SNFE.bidiTrie', bidiTrie.optimize(shrink)); }; /******************************************************************************* @@ -1223,7 +1252,7 @@ const domainOptIterator = new DomainOptIterator(''); const filterOrigin = (( ) => { const FilterOrigin = class { constructor() { - this.trieContainer = new µb.HNTrieContainer(); + this.trieContainer = new HNTrieContainer(); } compile(domainOptList, prepend, units) { @@ -1298,7 +1327,7 @@ const filterOrigin = (( ) => { prime() { this.trieContainer.reset( - vAPI.localStorage.getItem('SNFE.filterOrigin.trieDetails') + keyvalStore.getItem('SNFE.filterOrigin.trieDetails') ); } @@ -1307,7 +1336,7 @@ const filterOrigin = (( ) => { } optimize() { - vAPI.localStorage.setItem( + keyvalStore.setItem( 'SNFE.filterOrigin.trieDetails', this.trieContainer.optimize() ); @@ -1631,7 +1660,7 @@ const FilterModifier = class { } logData(details) { - let opt = vAPI.StaticFilteringParser.netOptionTokenNames.get(this.type); + let opt = StaticFilteringParser.netOptionTokenNames.get(this.type); if ( this.value !== '' ) { opt += `=${this.value}`; } @@ -1933,7 +1962,7 @@ const FilterHostnameDict = class { static prime() { return FilterHostnameDict.trieContainer.reset( - vAPI.localStorage.getItem('SNFE.FilterHostnameDict.trieDetails') + keyvalStore.getItem('SNFE.FilterHostnameDict.trieDetails') ); } @@ -1942,7 +1971,7 @@ const FilterHostnameDict = class { } static optimize() { - vAPI.localStorage.setItem( + keyvalStore.setItem( 'SNFE.FilterHostnameDict.trieDetails', FilterHostnameDict.trieContainer.optimize() ); @@ -1953,7 +1982,7 @@ const FilterHostnameDict = class { } }; -FilterHostnameDict.trieContainer = new µb.HNTrieContainer(); +FilterHostnameDict.trieContainer = new HNTrieContainer(); registerFilterClass(FilterHostnameDict); @@ -2402,7 +2431,7 @@ const FilterOnHeaders = class { match() { if ( this.parsed === undefined ) { this.parsed = - vAPI.StaticFilteringParser.parseHeaderValue(this.headerOpt); + StaticFilteringParser.parseHeaderValue(this.headerOpt); } const { bad, name, not, re, value } = this.parsed; if ( bad ) { return false; } @@ -2556,14 +2585,14 @@ const urlTokenizer = new (class { } toSelfie() { - return µBlock.base64.encode( + return sparseBase64.encode( this.knownTokens.buffer, this.knownTokens.byteLength ); } fromSelfie(selfie) { - return µBlock.base64.decode(selfie, this.knownTokens.buffer); + return sparseBase64.decode(selfie, this.knownTokens.buffer); } // https://github.com/chrisaljoudi/uBlock/issues/1118 @@ -3183,7 +3212,7 @@ const FilterParser = class { // Mind `\b` directives: `/\bads\b/` should result in token being `ads`, // not `bads`. extractTokenFromRegex(pattern) { - pattern = vAPI.StaticFilteringParser.regexUtils.toTokenizableStr(pattern); + pattern = StaticFilteringParser.regexUtils.toTokenizableStr(pattern); this.reToken.lastIndex = 0; let bestToken; let bestBadness = 0x7FFFFFFF; @@ -3278,7 +3307,7 @@ FilterParser.parse = (( ) => { parser = undefined; return; } - ttlTimer = vAPI.setTimeout(ttlProcess, 10007); + ttlTimer = globals.setTimeout(ttlProcess, 10007); }; return p => { @@ -3287,7 +3316,7 @@ FilterParser.parse = (( ) => { } last = Date.now(); if ( ttlTimer === undefined ) { - ttlTimer = vAPI.setTimeout(ttlProcess, 10007); + ttlTimer = globals.setTimeout(ttlProcess, 10007); } return parser.parse(p); }; @@ -3351,7 +3380,7 @@ FilterContainer.prototype.reset = function() { // Cancel potentially pending optimization run. if ( this.optimizeTimerId !== undefined ) { - self.cancelIdleCallback(this.optimizeTimerId); + globals.cancelIdleCallback(this.optimizeTimerId); this.optimizeTimerId = undefined; } @@ -3365,7 +3394,7 @@ FilterContainer.prototype.reset = function() { FilterContainer.prototype.freeze = function() { const filterBucketId = FilterBucket.fid; - const unserialize = µb.CompiledLineIO.unserialize; + const unserialize = CompiledListReader.unserialize; const t0 = Date.now(); @@ -3452,17 +3481,24 @@ FilterContainer.prototype.freeze = function() { // Optimizing is not critical for the static network filtering engine to // work properly, so defer this until later to allow for reduced delay to // readiness when no valid selfie is available. - this.optimizeTimerId = self.requestIdleCallback(( ) => { - this.optimizeTimerId = undefined; - this.optimize(); - }, { timeout: 5000 }); + if ( this.optimizeTimerId === undefined ) { + this.optimizeTimerId = globals.requestIdleCallback(( ) => { + this.optimizeTimerId = undefined; + this.optimize(); + }, { timeout: 5000 }); + } - log.info(`staticNetFilteringEngine.freeze() took ${Date.now()-t0} ms`); + console.info(`staticNetFilteringEngine.freeze() took ${Date.now()-t0} ms`); }; /******************************************************************************/ FilterContainer.prototype.optimize = function() { + if ( this.optimizeTimerId !== undefined ) { + globals.cancelIdleCallback(this.optimizeTimerId); + this.optimizeTimerId = undefined; + } + const t0 = Date.now(); for ( let bits = 0, n = this.categories.length; bits < n; bits++ ) { @@ -3488,12 +3524,19 @@ FilterContainer.prototype.optimize = function() { filterUnits[i] = null; } - log.info(`staticNetFilteringEngine.optimize() took ${Date.now()-t0} ms`); + console.info(`staticNetFilteringEngine.optimize() took ${Date.now()-t0} ms`); }; /******************************************************************************/ -FilterContainer.prototype.toSelfie = function(path) { +FilterContainer.prototype.toSelfie = function(storage, path) { + if ( + storage instanceof Object === false || + storage.put instanceof Function === false + ) { + return Promise.resolve(); + } + const categoriesToSelfie = ( ) => { const selfie = []; for ( let bits = 0, n = this.categories.length; bits < n; bits++ ) { @@ -3508,26 +3551,26 @@ FilterContainer.prototype.toSelfie = function(path) { filterOrigin.optimize(); return Promise.all([ - µb.assets.put( + storage.put( `${path}/FilterHostnameDict.trieContainer`, - FilterHostnameDict.trieContainer.serialize(µb.base64) + FilterHostnameDict.trieContainer.serialize(sparseBase64) ), - µb.assets.put( + storage.put( `${path}/FilterOrigin.trieContainer`, - filterOrigin.trieContainer.serialize(µb.base64) + filterOrigin.trieContainer.serialize(sparseBase64) ), - µb.assets.put( + storage.put( `${path}/bidiTrie`, - bidiTrie.serialize(µb.base64) + bidiTrie.serialize(sparseBase64) ), - µb.assets.put( + storage.put( `${path}/filterSequences`, - µb.base64.encode( + sparseBase64.encode( Uint32Array.from(filterSequences).buffer, filterSequenceWritePtr << 2 ) ), - µb.assets.put( + storage.put( `${path}/main`, JSON.stringify({ processedFilterCount: this.processedFilterCount, @@ -3548,38 +3591,45 @@ FilterContainer.prototype.toSelfie = function(path) { /******************************************************************************/ -FilterContainer.prototype.fromSelfie = function(path) { +FilterContainer.prototype.fromSelfie = function(storage, path) { + if ( + storage instanceof Object === false || + storage.get instanceof Function === false + ) { + return Promise.resolve(); + } + return Promise.all([ - µb.assets.get(`${path}/FilterHostnameDict.trieContainer`).then(details => + storage.get(`${path}/FilterHostnameDict.trieContainer`).then(details => FilterHostnameDict.trieContainer.unserialize( details.content, - µb.base64 + sparseBase64 ) ), - µb.assets.get(`${path}/FilterOrigin.trieContainer`).then(details => + storage.get(`${path}/FilterOrigin.trieContainer`).then(details => filterOrigin.trieContainer.unserialize( details.content, - µb.base64 + sparseBase64 ) ), - µb.assets.get(`${path}/bidiTrie`).then(details => + storage.get(`${path}/bidiTrie`).then(details => bidiTrie.unserialize( details.content, - µb.base64 + sparseBase64 ) ), - µb.assets.get(`${path}/filterSequences`).then(details => { - const size = µb.base64.decodeSize(details.content) >> 2; + storage.get(`${path}/filterSequences`).then(details => { + const size = sparseBase64.decodeSize(details.content) >> 2; if ( size === 0 ) { return false; } filterSequenceBufferResize(size); filterSequenceWritePtr = size; - const buf32 = µb.base64.decode(details.content); + const buf32 = sparseBase64.decode(details.content); for ( let i = 0; i < size; i++ ) { filterSequences[i] = buf32[i]; } return true; }), - µb.assets.get(`${path}/main`).then(details => { + storage.get(`${path}/main`).then(details => { let selfie; try { selfie = JSON.parse(details.content); @@ -3615,7 +3665,7 @@ FilterContainer.prototype.fromSelfie = function(path) { /******************************************************************************/ FilterContainer.prototype.compile = function(parser, writer) { - // ORDER OF TESTS IS IMPORTANT! + this.error = undefined; const parsed = FilterParser.parse(parser); @@ -3624,19 +3674,15 @@ FilterContainer.prototype.compile = function(parser, writer) { // Ignore filters with unsupported options if ( parsed.unsupported ) { - const who = writer.properties.get('assetKey') || '?'; - µb.logger.writeOne({ - realm: 'message', - type: 'error', - text: `Invalid network filter in ${who}: ${parser.raw}` - }); + const who = writer.properties.get('name') || '?'; + this.error = `Invalid network filter in ${who}: ${parser.raw}`; return false; } writer.select( parsed.badFilter - ? µb.compiledNetworkSection + µb.compiledBadSubsection - : µb.compiledNetworkSection + ? writer.NETWORK_SECTION + COMPILED_BAD_SECTION + : writer.NETWORK_SECTION ); // Reminder: @@ -3747,7 +3793,7 @@ FilterContainer.prototype.compileParsed = function(parsed, writer) { // Header if ( parsed.headerOpt !== undefined ) { units.push(FilterOnHeaders.compile(parsed)); - parsed.action |= Headers; + parsed.action |= HEADERS; } // Modifier @@ -3805,8 +3851,8 @@ FilterContainer.prototype.compileToAtomicFilter = function( /******************************************************************************/ -FilterContainer.prototype.fromCompiledContent = function(reader) { - reader.select(µb.compiledNetworkSection); +FilterContainer.prototype.fromCompiled = function(reader) { + reader.select(reader.NETWORK_SECTION); while ( reader.next() ) { this.acceptedCount += 1; if ( this.goodFilters.has(reader.line) ) { @@ -3816,7 +3862,7 @@ FilterContainer.prototype.fromCompiledContent = function(reader) { } } - reader.select(µb.compiledNetworkSection + µb.compiledBadSubsection); + reader.select(reader.NETWORK_SECTION + COMPILED_BAD_SECTION); while ( reader.next() ) { this.badFilters.add(reader.line); } @@ -3863,7 +3909,7 @@ FilterContainer.prototype.matchAndFetchModifiers = function( const results = []; const env = { - modifier: vAPI.StaticFilteringParser.netOptionTokenIds.get(modifierType) || 0, + modifier: StaticFilteringParser.netOptionTokenIds.get(modifierType) || 0, bits: 0, th: 0, iunit: 0, @@ -4118,7 +4164,7 @@ FilterContainer.prototype.realmMatchString = function( // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/ // Add support for `specifichide`. -FilterContainer.prototype.matchStringReverse = function(type, url) { +FilterContainer.prototype.matchRequestReverse = function(type, url) { const typeBits = typeNameToTypeValue[type] | 0x80000000; // Prime tokenizer: we get a normalized URL in return. @@ -4127,8 +4173,8 @@ FilterContainer.prototype.matchStringReverse = function(type, url) { this.$filterUnit = 0; // These registers will be used by various filters - $docHostname = $requestHostname = vAPI.hostnameFromNetworkURL(url); - $docDomain = vAPI.domainFromHostname($docHostname); + $docHostname = $requestHostname = hostnameFromNetworkURL(url); + $docDomain = domainFromHostname($docHostname); $docEntity.reset(); // Exception filters @@ -4165,7 +4211,7 @@ FilterContainer.prototype.matchStringReverse = function(type, url) { * * @returns {integer} 0=no match, 1=block, 2=allow (exeption) */ -FilterContainer.prototype.matchString = function(fctxt, modifiers = 0) { +FilterContainer.prototype.matchRequest = function(fctxt, modifiers = 0) { let typeValue = typeNameToTypeValue[fctxt.type]; if ( modifiers === 0 ) { if ( typeValue === undefined ) { @@ -4227,10 +4273,10 @@ FilterContainer.prototype.matchHeaders = function(fctxt, headers) { $httpHeaders.init(headers); let r = 0; - if ( this.realmMatchString(Headers | BlockImportant, typeValue, partyBits) ) { + if ( this.realmMatchString(HEADERS | BlockImportant, typeValue, partyBits) ) { r = 1; - } else if ( this.realmMatchString(Headers | BlockAction, typeValue, partyBits) ) { - r = this.realmMatchString(Headers | AllowAction, typeValue, partyBits) + } else if ( this.realmMatchString(HEADERS | BlockAction, typeValue, partyBits) ) { + r = this.realmMatchString(HEADERS | AllowAction, typeValue, partyBits) ? 2 : 1; } @@ -4242,21 +4288,23 @@ FilterContainer.prototype.matchHeaders = function(fctxt, headers) { /******************************************************************************/ -FilterContainer.prototype.redirectRequest = function(fctxt) { +FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) { const directives = this.matchAndFetchModifiers(fctxt, 'redirect-rule'); // No directive is the most common occurrence. if ( directives === undefined ) { return; } const highest = directives.length - 1; // More than a single directive means more work. if ( highest !== 0 ) { - directives.sort(FilterContainer.compareRedirectRequests); + directives.sort( + FilterContainer.compareRedirectRequests.bind(this, redirectEngine) + ); } // Redirect to highest-ranked directive const directive = directives[highest]; if ( (directive.bits & AllowAction) === 0 ) { const { token } = FilterContainer.parseRedirectRequestValue(directive.modifier); - fctxt.redirectURL = µb.redirectEngine.tokenToURL(fctxt, token); + fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token); if ( fctxt.redirectURL === undefined ) { return; } } return directives; @@ -4265,18 +4313,18 @@ FilterContainer.prototype.redirectRequest = function(fctxt) { FilterContainer.parseRedirectRequestValue = function(modifier) { if ( modifier.cache === undefined ) { modifier.cache = - vAPI.StaticFilteringParser.parseRedirectValue(modifier.value); + StaticFilteringParser.parseRedirectValue(modifier.value); } return modifier.cache; }; -FilterContainer.compareRedirectRequests = function(a, b) { +FilterContainer.compareRedirectRequests = function(redirectEngine, a, b) { const { token: atok, priority: aint, bits: abits } = FilterContainer.parseRedirectRequestValue(a.modifier); - if ( µb.redirectEngine.hasToken(atok) === false ) { return -1; } + if ( redirectEngine.hasToken(atok) === false ) { return -1; } const { token: btok, priority: bint, bits: bbits } = FilterContainer.parseRedirectRequestValue(b.modifier); - if ( µb.redirectEngine.hasToken(btok) === false ) { return 1; } + if ( redirectEngine.hasToken(btok) === false ) { return 1; } if ( abits !== bbits ) { if ( (abits & Important) !== 0 ) { return 1; } if ( (bbits & Important) !== 0 ) { return -1; } @@ -4299,7 +4347,9 @@ FilterContainer.prototype.filterQuery = function(fctxt) { if ( qpos === -1 ) { return; } let hpos = url.indexOf('#', qpos + 1); if ( hpos === -1 ) { hpos = url.length; } - const params = new Map(new self.URLSearchParams(url.slice(qpos + 1, hpos))); + const params = new Map( + new globals.URLSearchParams(url.slice(qpos + 1, hpos)) + ); const inParamCount = params.size; const out = []; for ( const directive of directives ) { @@ -4363,7 +4413,7 @@ FilterContainer.prototype.filterQuery = function(fctxt) { FilterContainer.prototype.parseQueryPruneValue = function(modifier) { if ( modifier.cache === undefined ) { modifier.cache = - vAPI.StaticFilteringParser.parseQueryPruneValue(modifier.value); + StaticFilteringParser.parseQueryPruneValue(modifier.value); } return modifier.cache; }; @@ -4406,11 +4456,11 @@ FilterContainer.prototype.getFilterCount = function() { /******************************************************************************/ -FilterContainer.prototype.enableWASM = function() { +FilterContainer.prototype.enableWASM = function(modulePath) { return Promise.all([ - bidiTrie.enableWASM(), - filterOrigin.trieContainer.enableWASM(), - FilterHostnameDict.trieContainer.enableWASM(), + bidiTrie.enableWASM(modulePath), + filterOrigin.trieContainer.enableWASM(modulePath), + FilterHostnameDict.trieContainer.enableWASM(modulePath), ]); }; @@ -4418,8 +4468,8 @@ FilterContainer.prototype.enableWASM = function() { // action: 1=test, 2=record -FilterContainer.prototype.benchmark = async function(action, target) { - const requests = await µb.loadBenchmarkDataset(); +FilterContainer.prototype.benchmark = async function(requests, options = {}) { + const { action, target, redirectEngine } = options; if ( Array.isArray(requests) === false || requests.length === 0 ) { const text = 'No dataset found to benchmark'; @@ -4429,15 +4479,16 @@ FilterContainer.prototype.benchmark = async function(action, target) { const print = log.print; - print(`Benchmarking staticNetFilteringEngine.matchString()...`); - const fctxt = µb.filteringContext.duplicate(); + print(`Benchmarking staticNetFilteringEngine.matchRequest()...`); + + const fctxt = new FilteringContext(); if ( typeof target === 'number' ) { const request = requests[target]; fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); print(`Result=${r}:`); print(`\ttype=${fctxt.type}`); print(`\turl=${fctxt.url}`); @@ -4452,7 +4503,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { if ( action === 1 ) { try { expected = JSON.parse( - vAPI.localStorage.getItem('FilterContainer.benchmark.results') + keyvalStore.getItem('FilterContainer.benchmark.results') ); } catch(ex) { } @@ -4461,7 +4512,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { recorded = []; } - const t0 = self.performance.now(); + const t0 = globals.performance.now(); let matchCount = 0; for ( let i = 0; i < requests.length; i++ ) { const request = requests[i]; @@ -4469,7 +4520,7 @@ FilterContainer.prototype.benchmark = async function(action, target) { fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); this.redirectURL = undefined; - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); matchCount += 1; if ( recorded !== undefined ) { recorded.push(r); } if ( expected !== undefined && r !== expected[i] ) { @@ -4487,15 +4538,15 @@ FilterContainer.prototype.benchmark = async function(action, target) { this.matchAndFetchModifiers(fctxt, 'csp'); } this.matchHeaders(fctxt, []); - } else { - this.redirectRequest(fctxt); + } else if ( redirectEngine !== undefined ) { + this.redirectRequest(redirectEngine, fctxt); } } - const t1 = self.performance.now(); + const t1 = globals.performance.now(); const dur = t1 - t0; if ( recorded !== undefined ) { - vAPI.localStorage.setItem( + keyvalStore.setItem( 'FilterContainer.benchmark.results', JSON.stringify(recorded) ); @@ -4519,12 +4570,12 @@ FilterContainer.prototype.benchmark = async function(action, target) { /******************************************************************************/ -FilterContainer.prototype.test = function(docURL, type, url) { - const fctxt = µb.filteringContext.duplicate(); +FilterContainer.prototype.test = async function(docURL, type, url) { + const fctxt = new FilteringContext(); fctxt.setDocOriginFromURL(docURL); fctxt.setType(type); fctxt.setURL(url); - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); console.log(`${r}`); if ( r !== 0 ) { console.log(this.toLogData()); @@ -4687,17 +4738,15 @@ FilterContainer.prototype.filterClassHistogram = function() { /******************************************************************************/ -FilterContainer.prototype.tokenHistograms = async function() { - const requests = await µb.loadBenchmarkDataset(); - +FilterContainer.prototype.tokenHistograms = async function(requests) { if ( Array.isArray(requests) === false || requests.length === 0 ) { console.info('No requests found to benchmark'); return; } console.info(`Computing token histograms...`); - const fctxt = µb.filteringContext.duplicate(); + const fctxt = new FilteringContext(); const missTokenMap = new Map(); const hitTokenMap = new Map(); const reTokens = /[0-9a-z%]{2,}/g; @@ -4707,7 +4756,7 @@ FilterContainer.prototype.tokenHistograms = async function() { fctxt.setURL(request.url); fctxt.setDocOriginFromURL(request.frameUrl); fctxt.setType(request.cpt); - const r = this.matchString(fctxt); + const r = this.matchRequest(fctxt); for ( let [ keyword ] of request.url.toLowerCase().matchAll(reTokens) ) { const token = keyword; if ( r === 0 ) { @@ -4729,8 +4778,8 @@ FilterContainer.prototype.tokenHistograms = async function() { /******************************************************************************/ -return new FilterContainer(); +const staticNetFilteringEngine = new FilterContainer(); /******************************************************************************/ -})(); +export { staticNetFilteringEngine }; diff --git a/src/js/storage.js b/src/js/storage.js index e0fb1668c..5ca65a30d 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -19,12 +19,27 @@ Home: https://github.com/gorhill/uBlock */ -/* global punycode, publicSuffixList */ - 'use strict'; /******************************************************************************/ +import '../lib/publicsuffixlist/publicsuffixlist.js'; +import '../lib/punycode.js'; + +import globals from './globals.js'; +import { hostnameFromURI } from './uri-utils.js'; +import { sparseBase64 } from './base64-custom.js'; +import { LineIterator } from './text-iterators.js'; +import { StaticFilteringParser } from './static-filtering-parser.js'; +import µBlock from './background.js'; + +import { + CompiledListReader, + CompiledListWriter, +} from './static-filtering-io.js'; + +/******************************************************************************/ + µBlock.getBytesInUse = async function() { const promises = []; let bytesInUse; @@ -242,7 +257,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µBlock.hiddenSettingsFromString = function(raw) { const out = Object.assign({}, this.hiddenSettingsDefault); - const lineIter = new this.LineIterator(raw); + const lineIter = new LineIterator(raw); while ( lineIter.eot() === false ) { const line = lineIter.next(); const matches = /^\s*(\S+)\s+(.+)$/.exec(line); @@ -561,7 +576,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { // https://github.com/gorhill/uBlock/issues/1786 if ( details.docURL === undefined ) { return; } this.cosmeticFilteringEngine.removeFromSelectorCache( - vAPI.hostnameFromURI(details.docURL) + hostnameFromURI(details.docURL) ); }; @@ -929,12 +944,12 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ µBlock.compileFilters = function(rawText, details = {}) { - const writer = new this.CompiledLineIO.Writer(); + const writer = new CompiledListWriter(); // Populate the writer with information potentially useful to the // client compilers. if ( details.assetKey ) { - writer.properties.set('assetKey', details.assetKey); + writer.properties.set('name', details.assetKey); } const expertMode = details.assetKey !== this.userFiltersPath || @@ -944,8 +959,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { // https://adblockplus.org/en/filters const staticNetFilteringEngine = this.staticNetFilteringEngine; const staticExtFilteringEngine = this.staticExtFilteringEngine; - const lineIter = new this.LineIterator(this.preparseDirectives.prune(rawText)); - const parser = new vAPI.StaticFilteringParser({ expertMode }); + const lineIter = new LineIterator(this.preparseDirectives.prune(rawText)); + const parser = new StaticFilteringParser({ expertMode }); parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH); @@ -973,7 +988,14 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { if ( parser.patternHasUnicode() && parser.toASCII() === false ) { continue; } - staticNetFilteringEngine.compile(parser, writer); + if ( staticNetFilteringEngine.compile(parser, writer) ) { continue; } + if ( staticNetFilteringEngine.error !== undefined ) { + this.logger.writeOne({ + realm: 'message', + type: 'error', + text: staticNetFilteringEngine.error + }); + } } // https://github.com/uBlockOrigin/uBlock-issues/issues/1365 @@ -993,8 +1015,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µBlock.applyCompiledFilters = function(rawText, firstparty) { if ( rawText === '' ) { return; } - const reader = new this.CompiledLineIO.Reader(rawText); - this.staticNetFilteringEngine.fromCompiledContent(reader); + const reader = new CompiledListReader(rawText); + this.staticNetFilteringEngine.fromCompiled(reader); this.staticExtFilteringEngine.fromCompiledContent(reader, { skipGenericCosmetic: this.userSettings.ignoreGenericCosmeticFilters, skipCosmetic: !firstparty && !this.userSettings.parseAllABPHideFilters @@ -1162,13 +1184,14 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ µBlock.loadPublicSuffixList = async function() { + const psl = globals.publicSuffixList; if ( this.hiddenSettings.disableWebAssembly !== true ) { - publicSuffixList.enableWASM(); + psl.enableWASM('/lib/publicsuffixlist'); } try { const result = await this.assets.get(`compiled/${this.pslAssetKey}`); - if ( publicSuffixList.fromSelfie(result.content, this.base64) ) { + if ( psl.fromSelfie(result.content, sparseBase64) ) { return; } } catch (ex) { @@ -1182,11 +1205,9 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { }; µBlock.compilePublicSuffixList = function(content) { - publicSuffixList.parse(content, punycode.toASCII); - this.assets.put( - 'compiled/' + this.pslAssetKey, - publicSuffixList.toSelfie(µBlock.base64) - ); + const psl = globals.publicSuffixList; + psl.parse(content, globals.punycode.toASCII); + this.assets.put(`compiled/${this.pslAssetKey}`, psl.toSelfie(sparseBase64)); }; /******************************************************************************/ @@ -1218,6 +1239,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { 'selfie/staticExtFilteringEngine' ), µb.staticNetFilteringEngine.toSelfie( + µb.assets, 'selfie/staticNetFilteringEngine' ), ]); @@ -1261,6 +1283,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { 'selfie/staticExtFilteringEngine' ), µb.staticNetFilteringEngine.fromSelfie( + µb.assets, 'selfie/staticNetFilteringEngine' ), ]); diff --git a/src/js/tab.js b/src/js/tab.js index b63b72863..046df4ddf 100644 --- a/src/js/tab.js +++ b/src/js/tab.js @@ -21,6 +21,17 @@ 'use strict'; +/******************************************************************************/ + +import { + domainFromHostname, + hostnameFromURI, + isNetworkURI, + originFromURI, +} from './uri-utils.js'; + +import µBlock from './background.js'; + /******************************************************************************/ /******************************************************************************/ @@ -33,26 +44,30 @@ // hostname. This way, for a specific scheme you can create scope with // rules which will apply only to that scheme. -µBlock.normalizePageURL = function(tabId, pageURL) { - if ( tabId < 0 ) { - return 'http://behind-the-scene/'; - } - const uri = this.URI.set(pageURL); - const scheme = uri.scheme; - if ( scheme === 'https' || scheme === 'http' ) { - return uri.normalizedURI(); - } +µBlock.normalizeTabURL = (( ) => { + const tabURLNormalizer = new URL('about:blank'); - let fakeHostname = scheme + '-scheme'; + return (tabId, tabURL) => { + if ( tabId < 0 ) { + return 'http://behind-the-scene/'; + } + tabURLNormalizer.href = tabURL; + const protocol = tabURLNormalizer.protocol.slice(0, -1); + if ( protocol === 'https' || protocol === 'http' ) { + return tabURLNormalizer.href; + } - if ( uri.hostname !== '' ) { - fakeHostname = uri.hostname + '.' + fakeHostname; - } else if ( scheme === 'about' && uri.path !== '' ) { - fakeHostname = uri.path + '.' + fakeHostname; - } + let fakeHostname = protocol + '-scheme'; - return `http://${fakeHostname}/`; -}; + if ( tabURLNormalizer.hostname !== '' ) { + fakeHostname = tabURLNormalizer.hostname + '.' + fakeHostname; + } else if ( protocol === 'about' && protocol.pathname !== '' ) { + fakeHostname = tabURLNormalizer.pathname + '.' + fakeHostname; + } + + return `http://${fakeHostname}/`; + }; +})(); /******************************************************************************/ @@ -114,7 +129,7 @@ // Don't block if uBO is turned off in popup's context if ( µb.getNetFilteringSwitch(targetURL) === false || - µb.getNetFilteringSwitch(µb.normalizePageURL(0, targetURL)) === false + µb.getNetFilteringSwitch(µb.normalizeTabURL(0, targetURL)) === false ) { return 0; } @@ -192,7 +207,7 @@ } fctxt.type = popupType; - const result = µb.staticNetFilteringEngine.matchString(fctxt, 0b0001); + const result = µb.staticNetFilteringEngine.matchRequest(fctxt, 0b0001); if ( result !== 0 ) { fctxt.filter = µb.staticNetFilteringEngine.toLogData(); return result; @@ -257,7 +272,7 @@ // For now, a "broad" filter is one which does not touch any part of // the hostname part of the opener URL. let popunderURL = rootOpenerURL, - popunderHostname = µb.URI.hostnameFromURI(popunderURL); + popunderHostname = hostnameFromURI(popunderURL); if ( popunderHostname === '' ) { return 0; } result = mapPopunderResult( @@ -270,7 +285,7 @@ // https://github.com/gorhill/uBlock/issues/1598 // Try to find a match against origin part of the opener URL. - popunderURL = µb.URI.originFromURI(popunderURL); + popunderURL = originFromURI(popunderURL); if ( popunderURL === '' ) { return 0; } return mapPopunderResult( @@ -305,7 +320,7 @@ // https://github.com/gorhill/uBlock/issues/1538 if ( µb.getNetFilteringSwitch( - µb.normalizePageURL(openerTabId, rootOpenerURL) + µb.normalizeTabURL(openerTabId, rootOpenerURL) ) === false ) { return; @@ -662,11 +677,11 @@ housekeep itself. } const stackEntry = this.stack[this.stack.length - 1]; this.rawURL = stackEntry.url; - this.normalURL = µb.normalizePageURL(this.tabId, this.rawURL); - this.origin = µb.URI.originFromURI(this.normalURL); - this.rootHostname = µb.URI.hostnameFromURI(this.origin); + this.normalURL = µb.normalizeTabURL(this.tabId, this.rawURL); + this.origin = originFromURI(this.normalURL); + this.rootHostname = hostnameFromURI(this.origin); this.rootDomain = - µb.URI.domainFromHostname(this.rootHostname) || + domainFromHostname(this.rootHostname) || this.rootHostname; }; @@ -794,10 +809,10 @@ housekeep itself. const entry = new TabContext(vAPI.noTabId); entry.stack.push(new StackEntry('', true)); entry.rawURL = ''; - entry.normalURL = µb.normalizePageURL(entry.tabId); - entry.origin = µb.URI.originFromURI(entry.normalURL); - entry.rootHostname = µb.URI.hostnameFromURI(entry.origin); - entry.rootDomain = µb.URI.domainFromHostname(entry.rootHostname); + entry.normalURL = µb.normalizeTabURL(entry.tabId); + entry.origin = originFromURI(entry.normalURL); + entry.rootHostname = hostnameFromURI(entry.origin); + entry.rootDomain = domainFromHostname(entry.rootHostname); } // Context object, typically to be used to feed filtering engines. @@ -894,7 +909,7 @@ vAPI.Tabs = class extends vAPI.Tabs { pageStore.setFrameURL(details); if ( µb.canInjectScriptletsNow && - µb.URI.isNetworkURI(url) && + isNetworkURI(url) && pageStore.getNetFilteringSwitch() ) { µb.scriptletFilteringEngine.injectNow(details); diff --git a/src/js/text-encode.js b/src/js/text-encode.js index 614db6481..3eb6dc166 100644 --- a/src/js/text-encode.js +++ b/src/js/text-encode.js @@ -23,6 +23,10 @@ /******************************************************************************/ +import µBlock from './background.js'; + +/******************************************************************************/ + µBlock.textEncode = (function() { if ( µBlock.canFilterResponseData !== true ) { return; } diff --git a/src/js/text-iterators.js b/src/js/text-iterators.js new file mode 100644 index 000000000..e2986ddae --- /dev/null +++ b/src/js/text-iterators.js @@ -0,0 +1,92 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +class LineIterator { + constructor(text, offset) { + this.text = text; + this.textLen = this.text.length; + this.offset = offset || 0; + } + next(offset) { + if ( offset !== undefined ) { + this.offset += offset; + } + let lineEnd = this.text.indexOf('\n', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.text.indexOf('\r', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.textLen; + } + } + const line = this.text.slice(this.offset, lineEnd); + this.offset = lineEnd + 1; + return line; + } + peek(n) { + const offset = this.offset; + return this.text.slice(offset, offset + n); + } + charCodeAt(offset) { + return this.text.charCodeAt(this.offset + offset); + } + eot() { + return this.offset >= this.textLen; + } +} + +/******************************************************************************/ + +// The field iterator is less CPU-intensive than when using native +// String.split(). + +class FieldIterator { + constructor(sep) { + this.text = ''; + this.sep = sep; + this.sepLen = sep.length; + this.offset = 0; + } + first(text) { + this.text = text; + this.offset = 0; + return this.next(); + } + next() { + let end = this.text.indexOf(this.sep, this.offset); + if ( end === -1 ) { + end = this.text.length; + } + const field = this.text.slice(this.offset, end); + this.offset = end + this.sepLen; + return field; + } + remainder() { + return this.text.slice(this.offset); + } +} + +/******************************************************************************/ + +export { LineIterator, FieldIterator }; diff --git a/src/js/traffic.js b/src/js/traffic.js index 0f383aa0d..d76805ca3 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -23,9 +23,12 @@ /******************************************************************************/ -// Start isolation from global scope +import { + entityFromDomain, + isNetworkURI, +} from './uri-utils.js'; -µBlock.webRequest = (( ) => { +import µBlock from './background.js'; /******************************************************************************/ @@ -37,20 +40,11 @@ let dontCacheResponseHeaders = vAPI.webextFlavor.soup.has('firefox'); -// https://github.com/gorhill/uMatrix/issues/967#issuecomment-373002011 -// This can be removed once Firefox 60 ESR is released. -let cantMergeCSPHeaders = - vAPI.webextFlavor.soup.has('firefox') && vAPI.webextFlavor.major < 59; - - // The real actual webextFlavor value may not be set in stone, so listen // for possible future changes. window.addEventListener('webextFlavor', function() { dontCacheResponseHeaders = vAPI.webextFlavor.soup.has('firefox'); - cantMergeCSPHeaders = - vAPI.webextFlavor.soup.has('firefox') && - vAPI.webextFlavor.major < 59; }, { once: true }); // https://github.com/uBlockOrigin/uBlock-issues/issues/1553 @@ -269,7 +263,7 @@ const shouldStrictBlock = function(fctxt, loggerEnabled) { const snfe = µb.staticNetFilteringEngine; // Explicit filtering: `document` option - const rs = snfe.matchString(fctxt, 0b0011); + const rs = snfe.matchRequest(fctxt, 0b0011); const is = rs === 1 && snfe.isBlockImportant(); let lds; if ( rs !== 0 || loggerEnabled ) { @@ -291,7 +285,7 @@ const shouldStrictBlock = function(fctxt, loggerEnabled) { // Implicit filtering: no `document` option fctxt.type = 'no_type'; - let rg = snfe.matchString(fctxt, 0b0011); + let rg = snfe.matchRequest(fctxt, 0b0011); fctxt.type = 'main_frame'; const ig = rg === 1 && snfe.isBlockImportant(); let ldg; @@ -382,7 +376,7 @@ const onBeforeBehindTheSceneRequest = function(fctxt) { if ( fctxt.tabOrigin.endsWith('-scheme') === false && - µb.URI.isNetworkURI(fctxt.tabOrigin) || + isNetworkURI(fctxt.tabOrigin) || µb.userSettings.advancedUserEnabled || fctxt.itype === fctxt.CSP_REPORT ) { @@ -836,7 +830,7 @@ const filterDocument = (( ) => { url: fctxt.url, hostname: hostname, domain: domain, - entity: µb.URI.entityFromDomain(domain), + entity: entityFromDomain(domain), selectors: undefined, buffer: null, mime: 'text/html', @@ -997,17 +991,6 @@ const injectCSP = function(fctxt, pageStore, responseHeaders) { // Firefox 58/webext and less can't merge CSP headers, so we will merge // them here. - if ( cantMergeCSPHeaders ) { - const i = headerIndexFromName( - 'content-security-policy', - responseHeaders - ); - if ( i !== -1 ) { - cspSubsets.unshift(responseHeaders[i].value.trim()); - responseHeaders.splice(i, 1); - } - } - responseHeaders.push({ name: 'Content-Security-Policy', value: cspSubsets.join(', ') @@ -1139,7 +1122,9 @@ const strictBlockBypasser = { /******************************************************************************/ -return { +// Export + +µBlock.webRequest = { start: (( ) => { vAPI.net = new vAPI.Net(); vAPI.net.suspend(); @@ -1162,7 +1147,3 @@ return { }; /******************************************************************************/ - -})(); - -/******************************************************************************/ diff --git a/src/js/ublock.js b/src/js/ublock.js index 4284d15b4..bef20f8a3 100644 --- a/src/js/ublock.js +++ b/src/js/ublock.js @@ -21,13 +21,13 @@ 'use strict'; +/******************************************************************************/ + +import { hostnameFromURI } from './uri-utils.js'; +import µBlock from './background.js'; + /******************************************************************************/ /******************************************************************************/ - -{ - -// ***************************************************************************** -// start of local namespace // https://github.com/chrisaljoudi/uBlock/issues/405 // Be more flexible with whitelist syntax @@ -91,7 +91,7 @@ const matchBucket = function(url, hostname, bucket, start) { /******************************************************************************/ µBlock.getNetFilteringSwitch = function(url) { - const hostname = this.URI.hostnameFromURI(url); + const hostname = hostnameFromURI(url); let key = hostname; for (;;) { if ( matchBucket(url, hostname, this.netWhitelist.get(key)) !== -1 ) { @@ -121,7 +121,7 @@ const matchBucket = function(url, hostname, bucket, start) { const netWhitelist = this.netWhitelist; const pos = url.indexOf('#'); let targetURL = pos !== -1 ? url.slice(0, pos) : url; - const targetHostname = this.URI.hostnameFromURI(targetURL); + const targetHostname = hostnameFromURI(targetURL); let key = targetHostname; let directive = scope === 'page' ? targetURL : targetHostname; @@ -281,12 +281,6 @@ const matchBucket = function(url, hostname, bucket, start) { µBlock.reWhitelistBadHostname = /[^a-z0-9.\-_\[\]:]/; µBlock.reWhitelistHostnameExtractor = /([a-z0-9.\-_\[\]]+)(?::[\d*]+)?\/(?:[^\x00-\x20\/]|$)[^\x00-\x20]*$/; -// end of local namespace -// ***************************************************************************** - -} - -/******************************************************************************/ /******************************************************************************/ µBlock.changeUserSettings = function(name, value) { diff --git a/src/js/uri-utils.js b/src/js/uri-utils.js new file mode 100644 index 000000000..99cbfdd7c --- /dev/null +++ b/src/js/uri-utils.js @@ -0,0 +1,126 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2014-present Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +'use strict'; + +/******************************************************************************/ + +import '../lib/publicsuffixlist/publicsuffixlist.js'; +import '../lib/punycode.js'; + +import globals from './globals.js'; + +/******************************************************************************/ + +// Originally: +// https://github.com/gorhill/uBlock/blob/8b5733a58d3acf9fb62815e14699c986bd1c2fdc/src/js/uritools.js + +const psl = globals.publicSuffixList; +const punycode = globals.punycode; + +const reCommonHostnameFromURL = + /^https?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])\//; +const reAuthorityFromURI = + /^(?:[^:\/?#]+:)?(\/\/[^\/?#]+)/; +const reHostFromNakedAuthority = + /^[0-9a-z._-]+[0-9a-z]$/i; +const reHostFromAuthority = + /^(?:[^@]*@)?([^:]+)(?::\d*)?$/; +const reIPv6FromAuthority = + /^(?:[^@]*@)?(\[[0-9a-f:]+\])(?::\d*)?$/i; +const reMustNormalizeHostname = + /[^0-9a-z._-]/; +const reOriginFromURI = + /^(?:[^:\/?#]+:)\/\/[^\/?#]+/; +const reHostnameFromNetworkURL = + /^(?:http|ws|ftp)s?:\/\/([0-9a-z_][0-9a-z._-]*[0-9a-z])(?::\d+)?\//; +const reIPAddressNaive = + /^\d+\.\d+\.\d+\.\d+$|^\[[\da-zA-Z:]+\]$/; + +/******************************************************************************/ + +const domainFromHostname = function(hostname) { + return reIPAddressNaive.test(hostname) + ? hostname + : psl.getDomain(hostname); +}; + +const domainFromURI = function(uri) { + if ( !uri ) { return ''; } + return domainFromHostname(hostnameFromURI(uri)); +}; + +const entityFromDomain = function(domain) { + const pos = domain.indexOf('.'); + return pos !== -1 ? domain.slice(0, pos) + '.*' : ''; +}; + +const hostnameFromURI = function(uri) { + let matches = reCommonHostnameFromURL.exec(uri); + if ( matches !== null ) { return matches[1]; } + matches = reAuthorityFromURI.exec(uri); + if ( matches === null ) { return ''; } + const authority = matches[1].slice(2); + if ( reHostFromNakedAuthority.test(authority) ) { + return authority.toLowerCase(); + } + matches = reHostFromAuthority.exec(authority); + if ( matches === null ) { + matches = reIPv6FromAuthority.exec(authority); + if ( matches === null ) { return ''; } + } + let hostname = matches[1]; + while ( hostname.endsWith('.') ) { + hostname = hostname.slice(0, -1); + } + if ( reMustNormalizeHostname.test(hostname) ) { + hostname = punycode.toASCII(hostname.toLowerCase()); + } + return hostname; +}; + +const hostnameFromNetworkURL = function(url) { + const matches = reHostnameFromNetworkURL.exec(url); + return matches !== null ? matches[1] : ''; +}; + +const originFromURI = function(uri) { + const matches = reOriginFromURI.exec(uri); + return matches !== null ? matches[0].toLowerCase() : ''; +}; + +const isNetworkURI = function(uri) { + return reNetworkURI.test(uri); +}; + +const reNetworkURI = /^(?:ftps?|https?|wss?):\/\//; + +/******************************************************************************/ + +export { + domainFromHostname, + domainFromURI, + entityFromDomain, + hostnameFromNetworkURL, + hostnameFromURI, + isNetworkURI, + originFromURI, +}; diff --git a/src/js/uritools.js b/src/js/uritools.js deleted file mode 100644 index cb0a17883..000000000 --- a/src/js/uritools.js +++ /dev/null @@ -1,380 +0,0 @@ -/******************************************************************************* - - uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-present Raymond Hill - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see {http://www.gnu.org/licenses/}. - - Home: https://github.com/gorhill/uBlock -*/ - -'use strict'; - -/******************************************************************************* - -RFC 3986 as reference: http://tools.ietf.org/html/rfc3986#appendix-A - -Naming convention from https://en.wikipedia.org/wiki/URI_scheme#Examples - -*/ - -/******************************************************************************/ - -µBlock.URI = (( ) => { - -/******************************************************************************/ - -// Favorite regex tool: http://regex101.com/ - -// Ref: -// I removed redundant capture groups: capture less = peform faster. See -// -// Performance improvements welcomed. -// jsperf: -const reRFC3986 = /^([^:\/?#]+:)?(\/\/[^\/?#]*)?([^?#]*)(\?[^#]*)?(#.*)?/; - -// Derived -const reSchemeFromURI = /^[^:\/?#]+:/; -const reOriginFromURI = /^(?:[^:\/?#]+:)\/\/[^\/?#]+/; -const rePathFromURI = /^(?:[^:\/?#]+:)?(?:\/\/[^\/?#]*)?([^?#]*)/; - -// These are to parse authority field, not parsed by above official regex -// IPv6 is seen as an exception: a non-compatible IPv6 is first tried, and -// if it fails, the IPv6 compatible regex istr used. This helps -// peformance by avoiding the use of a too complicated regex first. - -// https://github.com/gorhill/httpswitchboard/issues/211 -// "While a hostname may not contain other characters, such as the -// "underscore character (_), other DNS names may contain the underscore" -const reHostPortFromAuthority = /^(?:[^@]*@)?([^:]*)(:\d*)?$/; -const reIPv6PortFromAuthority = /^(?:[^@]*@)?(\[[0-9a-f:]*\])(:\d*)?$/i; - -const reHostFromNakedAuthority = /^[0-9a-z._-]+[0-9a-z]$/i; - -// Coarse (but fast) tests -const reValidHostname = /^([a-z\d]+(-*[a-z\d]+)*)(\.[a-z\d]+(-*[a-z\d])*)*$/; - -/******************************************************************************/ - -const reset = function(o) { - o.scheme = ''; - o.hostname = ''; - o._ipv4 = undefined; - o._ipv6 = undefined; - o.port = ''; - o.path = ''; - o.query = ''; - o.fragment = ''; - return o; -}; - -const resetAuthority = function(o) { - o.hostname = ''; - o._ipv4 = undefined; - o._ipv6 = undefined; - o.port = ''; - return o; -}; - -/******************************************************************************/ - -// This will be exported - -const URI = { - scheme: '', - authority: '', - hostname: '', - _ipv4: undefined, - _ipv6: undefined, - port: '', - domain: undefined, - path: '', - query: '', - fragment: '', - schemeBit: (1 << 0), - userBit: (1 << 1), - passwordBit: (1 << 2), - hostnameBit: (1 << 3), - portBit: (1 << 4), - pathBit: (1 << 5), - queryBit: (1 << 6), - fragmentBit: (1 << 7), - allBits: (0xFFFF) -}; - -URI.authorityBit = (URI.userBit | URI.passwordBit | URI.hostnameBit | URI.portBit); -URI.normalizeBits = (URI.schemeBit | URI.hostnameBit | URI.pathBit | URI.queryBit); - -/******************************************************************************/ - -// See: https://en.wikipedia.org/wiki/URI_scheme#Examples -// URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] -// -// foo://example.com:8042/over/there?name=ferret#nose -// \_/ \______________/\_________/ \_________/ \__/ -// | | | | | -// scheme authority path query fragment -// | _____________________|__ -// / \ / \ -// urn:example:animal:ferret:nose - -URI.set = function(uri) { - if ( uri === undefined ) { - return reset(URI); - } - let matches = reRFC3986.exec(uri); - if ( !matches ) { - return reset(URI); - } - this.scheme = matches[1] !== undefined ? matches[1].slice(0, -1) : ''; - this.authority = matches[2] !== undefined ? matches[2].slice(2).toLowerCase() : ''; - this.path = matches[3] !== undefined ? matches[3] : ''; - - // - // "In general, a URI that uses the generic syntax for authority - // "with an empty path should be normalized to a path of '/'." - if ( this.authority !== '' && this.path === '' ) { - this.path = '/'; - } - this.query = matches[4] !== undefined ? matches[4].slice(1) : ''; - this.fragment = matches[5] !== undefined ? matches[5].slice(1) : ''; - - // Assume very simple authority, i.e. just a hostname (highest likelihood - // case for µBlock) - if ( reHostFromNakedAuthority.test(this.authority) ) { - this.hostname = this.authority; - this.port = ''; - return this; - } - // Authority contains more than just a hostname - matches = reHostPortFromAuthority.exec(this.authority); - if ( !matches ) { - matches = reIPv6PortFromAuthority.exec(this.authority); - if ( !matches ) { - return resetAuthority(URI); - } - } - this.hostname = matches[1] !== undefined ? matches[1] : ''; - // http://en.wikipedia.org/wiki/FQDN - if ( this.hostname.endsWith('.') ) { - this.hostname = this.hostname.slice(0, -1); - } - this.port = matches[2] !== undefined ? matches[2].slice(1) : ''; - return this; -}; - -/******************************************************************************/ - -// URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] -// -// foo://example.com:8042/over/there?name=ferret#nose -// \_/ \______________/\_________/ \_________/ \__/ -// | | | | | -// scheme authority path query fragment -// | _____________________|__ -// / \ / \ -// urn:example:animal:ferret:nose - -URI.assemble = function(bits) { - if ( bits === undefined ) { - bits = this.allBits; - } - const s = []; - if ( this.scheme && (bits & this.schemeBit) ) { - s.push(this.scheme, ':'); - } - if ( this.hostname && (bits & this.hostnameBit) ) { - s.push('//', this.hostname); - } - if ( this.port && (bits & this.portBit) ) { - s.push(':', this.port); - } - if ( this.path && (bits & this.pathBit) ) { - s.push(this.path); - } - if ( this.query && (bits & this.queryBit) ) { - s.push('?', this.query); - } - if ( this.fragment && (bits & this.fragmentBit) ) { - s.push('#', this.fragment); - } - return s.join(''); -}; - -/******************************************************************************/ - -URI.originFromURI = function(uri) { - const matches = reOriginFromURI.exec(uri); - return matches !== null ? matches[0].toLowerCase() : ''; -}; - -/******************************************************************************/ - -URI.schemeFromURI = function(uri) { - const matches = reSchemeFromURI.exec(uri); - if ( !matches ) { return ''; } - return matches[0].slice(0, -1).toLowerCase(); -}; - -/******************************************************************************/ - -URI.hostnameFromURI = vAPI.hostnameFromURI; -URI.domainFromHostname = vAPI.domainFromHostname; - -URI.domain = function() { - return this.domainFromHostname(this.hostname); -}; - -/******************************************************************************/ - -URI.entityFromDomain = function(domain) { - const pos = domain.indexOf('.'); - return pos !== -1 ? domain.slice(0, pos) + '.*' : ''; -}; - -/******************************************************************************/ - -URI.pathFromURI = function(uri) { - const matches = rePathFromURI.exec(uri); - return matches !== null ? matches[1] : ''; -}; - -/******************************************************************************/ - -URI.domainFromURI = function(uri) { - if ( !uri ) { return ''; } - return this.domainFromHostname(this.hostnameFromURI(uri)); -}; - -/******************************************************************************/ - -URI.isNetworkURI = function(uri) { - return reNetworkURI.test(uri); -}; - -const reNetworkURI = /^(?:ftps?|https?|wss?):\/\//; - -/******************************************************************************/ - -URI.isNetworkScheme = function(scheme) { - return reNetworkScheme.test(scheme); -}; - -const reNetworkScheme = /^(?:ftps?|https?|wss?)$/; - -/******************************************************************************/ - -// Normalize the way µBlock expects it - -URI.normalizedURI = function() { - // Will be removed: - // - port - // - user id/password - // - fragment - return this.assemble(this.normalizeBits); -}; - -/******************************************************************************/ - -URI.rootURL = function() { - if ( !this.hostname ) { return ''; } - return this.assemble(this.schemeBit | this.hostnameBit); -}; - -/******************************************************************************/ - -URI.isValidHostname = function(hostname) { - try { - return reValidHostname.test(hostname); - } - catch (e) { - } - return false; -}; - -/******************************************************************************/ - -// Return the parent domain. For IP address, there is no parent domain. - -URI.parentHostnameFromHostname = function(hostname) { - // `locahost` => `` - // `example.org` => `example.org` - // `www.example.org` => `example.org` - // `tomato.www.example.org` => `example.org` - const domain = this.domainFromHostname(hostname); - - // `locahost` === `` => bye - // `example.org` === `example.org` => bye - // `www.example.org` !== `example.org` => stay - // `tomato.www.example.org` !== `example.org` => stay - if ( domain === '' || domain === hostname ) { return; } - - // Parent is hostname minus first label - return hostname.slice(hostname.indexOf('.') + 1); -}; - -/******************************************************************************/ - -// Return all possible parent hostnames which can be derived from `hostname`, -// ordered from direct parent up to domain inclusively. - -URI.parentHostnamesFromHostname = function(hostname) { - // TODO: I should create an object which is optimized to receive - // the list of hostnames by making it reusable (junkyard etc.) and which - // has its own element counter property in order to avoid memory - // alloc/dealloc. - const domain = this.domainFromHostname(hostname); - if ( domain === '' || domain === hostname ) { - return []; - } - const nodes = []; - for (;;) { - const pos = hostname.indexOf('.'); - if ( pos < 0 ) { break; } - hostname = hostname.slice(pos + 1); - nodes.push(hostname); - if ( hostname === domain ) { break; } - } - return nodes; -}; - -/******************************************************************************/ - -// Return all possible hostnames which can be derived from `hostname`, -// ordered from self up to domain inclusively. - -URI.allHostnamesFromHostname = function(hostname) { - const nodes = this.parentHostnamesFromHostname(hostname); - nodes.unshift(hostname); - return nodes; -}; - -/******************************************************************************/ - -URI.toString = function() { - return this.assemble(); -}; - -/******************************************************************************/ - -// Export - -return URI; - -/******************************************************************************/ - -})(); - -/******************************************************************************/ - diff --git a/src/js/url-net-filtering.js b/src/js/url-net-filtering.js index aa6a9e232..c5f1e2283 100644 --- a/src/js/url-net-filtering.js +++ b/src/js/url-net-filtering.js @@ -23,21 +23,20 @@ /******************************************************************************/ -// The purpose of log filtering is to create ad hoc filtering rules, to -// diagnose and assist in the creation of custom filters. - -µBlock.URLNetFiltering = (( ) => { +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; /******************************************************************************* -buckets: map of [hostname + type] - bucket: array of rule entries, sorted from shorter to longer url -rule entry: { url, action } + The purpose of log filtering is to create ad hoc filtering rules, to + diagnose and assist in the creation of custom filters. + + buckets: map of [hostname + type] + bucket: array of rule entries, sorted from shorter to longer url + rule entry: { url, action } *******************************************************************************/ -/******************************************************************************/ - const actionToNameMap = { 1: 'block', 2: 'allow', @@ -330,7 +329,7 @@ URLNetFiltering.prototype.toString = function() { URLNetFiltering.prototype.fromString = function(text) { this.reset(); - const lineIter = new µBlock.LineIterator(text); + const lineIter = new LineIterator(text); while ( lineIter.eot() === false ) { this.addFromRuleParts(lineIter.next().trim().split(/\s+/)); } @@ -371,15 +370,11 @@ URLNetFiltering.prototype.removeFromRuleParts = function(parts) { /******************************************************************************/ -return URLNetFiltering; - -/******************************************************************************/ - -})(); - -/******************************************************************************/ - -µBlock.sessionURLFiltering = new µBlock.URLNetFiltering(); -µBlock.permanentURLFiltering = new µBlock.URLNetFiltering(); +// Export + +µBlock.URLNetFiltering = URLNetFiltering; + +µBlock.sessionURLFiltering = new URLNetFiltering(); +µBlock.permanentURLFiltering = new URLNetFiltering(); /******************************************************************************/ diff --git a/src/js/utils.js b/src/js/utils.js index 47e84d557..955ce8ceb 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -23,6 +23,11 @@ /******************************************************************************/ +import { LineIterator } from './text-iterators.js'; +import µBlock from './background.js'; + +/******************************************************************************/ + µBlock.formatCount = function(count) { if ( typeof count !== 'number' ) { return ''; @@ -57,183 +62,6 @@ /******************************************************************************/ -µBlock.LineIterator = class { - constructor(text, offset) { - this.text = text; - this.textLen = this.text.length; - this.offset = offset || 0; - } - next(offset) { - if ( offset !== undefined ) { - this.offset += offset; - } - let lineEnd = this.text.indexOf('\n', this.offset); - if ( lineEnd === -1 ) { - lineEnd = this.text.indexOf('\r', this.offset); - if ( lineEnd === -1 ) { - lineEnd = this.textLen; - } - } - const line = this.text.slice(this.offset, lineEnd); - this.offset = lineEnd + 1; - return line; - } - peek(n) { - const offset = this.offset; - return this.text.slice(offset, offset + n); - } - charCodeAt(offset) { - return this.text.charCodeAt(this.offset + offset); - } - eot() { - return this.offset >= this.textLen; - } -}; - -/******************************************************************************/ - -// The field iterator is less CPU-intensive than when using native -// String.split(). - -µBlock.FieldIterator = class { - constructor(sep) { - this.text = ''; - this.sep = sep; - this.sepLen = sep.length; - this.offset = 0; - } - first(text) { - this.text = text; - this.offset = 0; - return this.next(); - } - next() { - let end = this.text.indexOf(this.sep, this.offset); - if ( end === -1 ) { - end = this.text.length; - } - const field = this.text.slice(this.offset, end); - this.offset = end + this.sepLen; - return field; - } - remainder() { - return this.text.slice(this.offset); - } -}; - -/******************************************************************************/ - -// https://www.reddit.com/r/uBlockOrigin/comments/oq6kt5/ubo_loads_generic_filter_instead_of_specific/ -// Ensure blocks of content are sorted in ascending id order, such that the -// specific cosmetic filters will be found (and thus reported) before the -// generic ones. - -µBlock.CompiledLineIO = { - serialize: JSON.stringify, - unserialize: JSON.parse, - blockStartPrefix: '#block-start-', // ensure no special regex characters - blockEndPrefix: '#block-end-', // ensure no special regex characters - - Writer: class { - constructor() { - this.io = µBlock.CompiledLineIO; - this.blockId = undefined; - this.block = undefined; - this.stringifier = this.io.serialize; - this.blocks = new Map(); - this.properties = new Map(); - } - push(args) { - this.block.push(this.stringifier(args)); - } - last() { - if ( Array.isArray(this.block) && this.block.length !== 0 ) { - return this.block[this.block.length - 1]; - } - } - select(blockId) { - if ( blockId === this.blockId ) { return; } - this.blockId = blockId; - this.block = this.blocks.get(blockId); - if ( this.block === undefined ) { - this.blocks.set(blockId, (this.block = [])); - } - return this; - } - toString() { - const result = []; - const sortedBlocks = - Array.from(this.blocks).sort((a, b) => a[0] - b[0]); - for ( const [ id, lines ] of sortedBlocks ) { - if ( lines.length === 0 ) { continue; } - result.push( - this.io.blockStartPrefix + id, - lines.join('\n'), - this.io.blockEndPrefix + id - ); - } - return result.join('\n'); - } - }, - - Reader: class { - constructor(raw, blockId) { - this.io = µBlock.CompiledLineIO; - this.block = ''; - this.len = 0; - this.offset = 0; - this.line = ''; - this.parser = this.io.unserialize; - this.blocks = new Map(); - this.properties = new Map(); - let reBlockStart = new RegExp( - `^${this.io.blockStartPrefix}(\\d+)\\n`, - 'gm' - ); - let match = reBlockStart.exec(raw); - while ( match !== null ) { - let beg = match.index + match[0].length; - let end = raw.indexOf(this.io.blockEndPrefix + match[1], beg); - this.blocks.set(parseInt(match[1], 10), raw.slice(beg, end)); - reBlockStart.lastIndex = end; - match = reBlockStart.exec(raw); - } - if ( blockId !== undefined ) { - this.select(blockId); - } - } - next() { - if ( this.offset === this.len ) { - this.line = ''; - return false; - } - let pos = this.block.indexOf('\n', this.offset); - if ( pos !== -1 ) { - this.line = this.block.slice(this.offset, pos); - this.offset = pos + 1; - } else { - this.line = this.block.slice(this.offset); - this.offset = this.len; - } - return true; - } - select(blockId) { - this.block = this.blocks.get(blockId) || ''; - this.len = this.block.length; - this.offset = 0; - return this; - } - fingerprint() { - return this.line; - } - args() { - return this.parser(this.line); - } - } -}; - -/******************************************************************************/ - µBlock.openNewTab = function(details) { if ( details.url.startsWith('logger-ui.html') ) { if ( details.shiftKey ) { @@ -374,228 +202,6 @@ /******************************************************************************/ -// Custom base64 codecs. These codecs are meant to encode/decode typed arrays -// to/from strings. - -// https://github.com/uBlockOrigin/uBlock-issues/issues/461 -// Provide a fallback encoding for Chromium 59 and less by issuing a plain -// JSON string. The fallback can be removed once min supported version is -// above 59. - -// TODO: rename µBlock.base64 to µBlock.SparseBase64, now that -// µBlock.DenseBase64 has been introduced. -// TODO: Should no longer need to test presence of TextEncoder/TextDecoder. - -{ - const valToDigit = new Uint8Array(64); - const digitToVal = new Uint8Array(128); - { - const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@%'; - for ( let i = 0, n = chars.length; i < n; i++ ) { - const c = chars.charCodeAt(i); - valToDigit[i] = c; - digitToVal[c] = i; - } - } - - // The sparse base64 codec is best for buffers which contains a lot of - // small u32 integer values. Those small u32 integer values are better - // represented with stringified integers, because small values can be - // represented with fewer bits than the usual base64 codec. For example, - // 0 become '0 ', i.e. 16 bits instead of 48 bits with official base64 - // codec. - - µBlock.base64 = { - magic: 'Base64_1', - - encode: function(arrbuf, arrlen) { - const inputLength = (arrlen + 3) >>> 2; - const inbuf = new Uint32Array(arrbuf, 0, inputLength); - const outputLength = this.magic.length + 7 + inputLength * 7; - const outbuf = new Uint8Array(outputLength); - // magic bytes - let j = 0; - for ( let i = 0; i < this.magic.length; i++ ) { - outbuf[j++] = this.magic.charCodeAt(i); - } - // array size - let v = inputLength; - do { - outbuf[j++] = valToDigit[v & 0b111111]; - v >>>= 6; - } while ( v !== 0 ); - outbuf[j++] = 0x20 /* ' ' */; - // array content - for ( let i = 0; i < inputLength; i++ ) { - v = inbuf[i]; - do { - outbuf[j++] = valToDigit[v & 0b111111]; - v >>>= 6; - } while ( v !== 0 ); - outbuf[j++] = 0x20 /* ' ' */; - } - if ( typeof TextDecoder === 'undefined' ) { - return JSON.stringify( - Array.from(new Uint32Array(outbuf.buffer, 0, j >>> 2)) - ); - } - const textDecoder = new TextDecoder(); - return textDecoder.decode(new Uint8Array(outbuf.buffer, 0, j)); - }, - - decode: function(instr, arrbuf) { - if ( instr.charCodeAt(0) === 0x5B /* '[' */ ) { - const inbuf = JSON.parse(instr); - if ( arrbuf instanceof ArrayBuffer === false ) { - return new Uint32Array(inbuf); - } - const outbuf = new Uint32Array(arrbuf); - outbuf.set(inbuf); - return outbuf; - } - if ( instr.startsWith(this.magic) === false ) { - throw new Error('Invalid µBlock.base64 encoding'); - } - const inputLength = instr.length; - const outputLength = this.decodeSize(instr) >> 2; - const outbuf = arrbuf instanceof ArrayBuffer === false - ? new Uint32Array(outputLength) - : new Uint32Array(arrbuf); - let i = instr.indexOf(' ', this.magic.length) + 1; - if ( i === -1 ) { - throw new Error('Invalid µBlock.base64 encoding'); - } - // array content - let j = 0; - for (;;) { - if ( j === outputLength || i >= inputLength ) { break; } - let v = 0, l = 0; - for (;;) { - const c = instr.charCodeAt(i++); - if ( c === 0x20 /* ' ' */ ) { break; } - v += digitToVal[c] << l; - l += 6; - } - outbuf[j++] = v; - } - if ( i < inputLength || j < outputLength ) { - throw new Error('Invalid µBlock.base64 encoding'); - } - return outbuf; - }, - - decodeSize: function(instr) { - if ( instr.startsWith(this.magic) === false ) { return 0; } - let v = 0, l = 0, i = this.magic.length; - for (;;) { - const c = instr.charCodeAt(i++); - if ( c === 0x20 /* ' ' */ ) { break; } - v += digitToVal[c] << l; - l += 6; - } - return v << 2; - }, - }; - - // The dense base64 codec is best for typed buffers which values are - // more random. For example, buffer contents as a result of compression - // contain less repetitive values and thus the content is more - // random-looking. - - // TODO: Investigate that in Firefox, creating a new Uint8Array from the - // ArrayBuffer fails, the content of the resulting Uint8Array is - // non-sensical. WASM-related? - - µBlock.denseBase64 = { - magic: 'DenseBase64_1', - - encode: function(input) { - const m = input.length % 3; - const n = input.length - m; - let outputLength = n / 3 * 4; - if ( m !== 0 ) { - outputLength += m + 1; - } - const output = new Uint8Array(outputLength); - let j = 0; - for ( let i = 0; i < n; i += 3) { - const i1 = input[i+0]; - const i2 = input[i+1]; - const i3 = input[i+2]; - output[j+0] = valToDigit[ i1 >>> 2]; - output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; - output[j+2] = valToDigit[i2 << 2 & 0b111100 | i3 >>> 6]; - output[j+3] = valToDigit[i3 & 0b111111 ]; - j += 4; - } - if ( m !== 0 ) { - const i1 = input[n]; - output[j+0] = valToDigit[i1 >>> 2]; - if ( m === 1 ) { // 1 value - output[j+1] = valToDigit[i1 << 4 & 0b110000]; - } else { // 2 values - const i2 = input[n+1]; - output[j+1] = valToDigit[i1 << 4 & 0b110000 | i2 >>> 4]; - output[j+2] = valToDigit[i2 << 2 & 0b111100 ]; - } - } - const textDecoder = new TextDecoder(); - const b64str = textDecoder.decode(output); - return this.magic + b64str; - }, - - decode: function(instr, arrbuf) { - if ( instr.startsWith(this.magic) === false ) { - throw new Error('Invalid µBlock.denseBase64 encoding'); - } - const outputLength = this.decodeSize(instr); - const outbuf = arrbuf instanceof ArrayBuffer === false - ? new Uint8Array(outputLength) - : new Uint8Array(arrbuf); - const inputLength = instr.length - this.magic.length; - let i = this.magic.length; - let j = 0; - const m = inputLength & 3; - const n = i + inputLength - m; - while ( i < n ) { - const i1 = digitToVal[instr.charCodeAt(i+0)]; - const i2 = digitToVal[instr.charCodeAt(i+1)]; - const i3 = digitToVal[instr.charCodeAt(i+2)]; - const i4 = digitToVal[instr.charCodeAt(i+3)]; - i += 4; - outbuf[j+0] = i1 << 2 | i2 >>> 4; - outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; - outbuf[j+2] = i3 << 6 & 0b11000000 | i4; - j += 3; - } - if ( m !== 0 ) { - const i1 = digitToVal[instr.charCodeAt(i+0)]; - const i2 = digitToVal[instr.charCodeAt(i+1)]; - outbuf[j+0] = i1 << 2 | i2 >>> 4; - if ( m === 3 ) { - const i3 = digitToVal[instr.charCodeAt(i+2)]; - outbuf[j+1] = i2 << 4 & 0b11110000 | i3 >>> 2; - } - } - return outbuf; - }, - - decodeSize: function(instr) { - if ( instr.startsWith(this.magic) === false ) { return 0; } - const inputLength = instr.length - this.magic.length; - const m = inputLength & 3; - const n = inputLength - m; - let outputLength = (n >>> 2) * 3; - if ( m !== 0 ) { - outputLength += m - 1; - } - return outputLength; - }, - }; -} - -/******************************************************************************/ - // The requests.json.gz file can be downloaded from: // https://cdn.cliqz.com/adblocking/requests_top500.json.gz // @@ -654,7 +260,7 @@ datasetPromise = µBlock.assets.fetchText(datasetURL).then(details => { console.info(`Parsing benchmark dataset...`); const requests = []; - const lineIter = new µBlock.LineIterator(details.content); + const lineIter = new LineIterator(details.content); while ( lineIter.eot() === false ) { let request; try { diff --git a/src/lib/publicsuffixlist/publicsuffixlist.js b/src/lib/publicsuffixlist/publicsuffixlist.js index 9963c1f88..a073d49ac 100644 --- a/src/lib/publicsuffixlist/publicsuffixlist.js +++ b/src/lib/publicsuffixlist/publicsuffixlist.js @@ -16,6 +16,8 @@ /* jshint browser:true, esversion:6, laxbreak:true, undef:true, unused:true */ /* globals WebAssembly, console, exports:true, module */ +'use strict'; + /******************************************************************************* Reference: @@ -45,8 +47,6 @@ (function(context) { // >>>>>>>> start of anonymous namespace -'use strict'; - /******************************************************************************* Tree encoding in array buffer: @@ -81,13 +81,11 @@ let hostnameArg = EMPTY_STRING; /******************************************************************************/ const fireChangedEvent = function() { - if ( - window instanceof Object && - window.dispatchEvent instanceof Function && - window.CustomEvent instanceof Function - ) { - window.dispatchEvent(new CustomEvent('publicSuffixListChanged')); - } + if ( typeof window !== 'object' ) { return; } + if ( window instanceof Object === false ) { return; } + if ( window.dispatchEvent instanceof Function === false ) { return; } + if ( window.CustomEvent instanceof Function === false ) { return; } + window.dispatchEvent(new CustomEvent('publicSuffixListChanged')); }; /******************************************************************************/ @@ -531,22 +529,9 @@ const fromSelfie = function(selfie, decoder) { // used should the WASM module be unavailable for whatever reason. const enableWASM = (function() { - // The directory from which the current script was fetched should also - // contain the related WASM file. The script is fetched from a trusted - // location, and consequently so will be the related WASM file. - let workingDir; - { - const url = new URL(document.currentScript.src); - const match = /[^\/]+$/.exec(url.pathname); - if ( match !== null ) { - url.pathname = url.pathname.slice(0, match.index); - } - workingDir = url.href; - } - let memory; - return function() { + return function(modulePath) { if ( getPublicSuffixPosWASM instanceof Function ) { return Promise.resolve(true); } @@ -568,7 +553,7 @@ const enableWASM = (function() { } return fetch( - workingDir + 'wasm/publicsuffixlist.wasm', + `${modulePath}/wasm/publicsuffixlist.wasm`, { mode: 'same-origin' } ).then(response => { const pageCount = pslBuffer8 !== undefined @@ -624,8 +609,6 @@ const disableWASM = function() { /******************************************************************************/ -context = context || window; - context.publicSuffixList = { version: '2.0', parse, @@ -644,4 +627,14 @@ if ( typeof module !== 'undefined' ) { /******************************************************************************/ // <<<<<<<< end of anonymous namespace -})(this); +})( + (root => { + if ( root !== undefined ) { return root; } + // jshint ignore:start + if ( typeof self !== 'undefined' ) { return self; } + if ( typeof window !== 'undefined' ) { return window; } + if ( typeof global !== 'undefined' ) { return global; } + // jshint ignore:end + throw new Error('unable to locate global object'); + })(this) +); diff --git a/src/lib/punycode.js b/src/lib/punycode.js index f3866b8e6..18a94eac6 100644 --- a/src/lib/punycode.js +++ b/src/lib/punycode.js @@ -6,7 +6,7 @@ !exports.nodeType && exports; var freeModule = typeof module == 'object' && module && !module.nodeType && module; - var freeGlobal = typeof global == 'object' && global; + var freeGlobal = typeof global == 'object' && global || self; if ( freeGlobal.global === freeGlobal || freeGlobal.window === freeGlobal || diff --git a/src/lib/regexanalyzer/regex.js b/src/lib/regexanalyzer/regex.js index 788f03ee8..1decf2534 100644 --- a/src/lib/regexanalyzer/regex.js +++ b/src/lib/regexanalyzer/regex.js @@ -9,9 +9,7 @@ **/ !function( root, name, factory ){ "use strict"; -if ( ('undefined'!==typeof Components)&&('object'===typeof Components.classes)&&('object'===typeof Components.classesByID)&&Components.utils&&('function'===typeof Components.utils['import']) ) /* XPCOM */ - (root.$deps = root.$deps||{}) && (root.EXPORTED_SYMBOLS = [name]) && (root[name] = root.$deps[name] = factory.call(root)); -else if ( ('object'===typeof module)&&module.exports ) /* CommonJS */ +if ( ('object'===typeof module)&&module.exports ) /* CommonJS */ (module.$deps = module.$deps||{}) && (module.exports = module.$deps[name] = factory.call(root)); else if ( ('undefined'!==typeof System)&&('function'===typeof System.register)&&('function'===typeof System['import']) ) /* ES6 module */ System.register(name,[],function($__export){$__export(name, factory.call(root));}); @@ -19,7 +17,11 @@ else if ( ('function'===typeof define)&&define.amd&&('function'===typeof require define(name,['module'],function(module){factory.moduleUri = module.uri; return factory.call(root);}); else if ( !(name in root) ) /* Browser/WebWorker/.. */ (root[name] = factory.call(root)||1)&&('function'===typeof(define))&&define.amd&&define(function(){return root[name];} ); -}( /* current root */ 'undefined' !== typeof self ? self : this, +}( /* current root */ (( ) => { + if ( typeof globalThis !== 'undefined' ) { return globalThis; } + if ( typeof self !== 'undefined' ) { return self; } + if ( typeof global !== 'undefined' ) { return global; } + })(), /* module name */ "Regex", /* module factory */ function ModuleFactory__Regex( undef ){ "use strict"; diff --git a/src/logger-ui.html b/src/logger-ui.html index 884a01339..7ea8f46ad 100644 --- a/src/logger-ui.html +++ b/src/logger-ui.html @@ -212,8 +212,8 @@ - - + + diff --git a/src/web_accessible_resources/epicker-ui.html b/src/web_accessible_resources/epicker-ui.html index 8ab76e32b..a99e0d764 100644 --- a/src/web_accessible_resources/epicker-ui.html +++ b/src/web_accessible_resources/epicker-ui.html @@ -61,15 +61,12 @@ - - - - + diff --git a/tools/copy-common-files.sh b/tools/copy-common-files.sh index 7500512ea..83da91b91 100644 --- a/tools/copy-common-files.sh +++ b/tools/copy-common-files.sh @@ -8,7 +8,11 @@ bash ./tools/make-assets.sh $DES cp -R src/css $DES/ cp -R src/img $DES/ -cp -R src/js $DES/ +mkdir $DES/js +cp -R src/js/*.js $DES/js/ +cp -R src/js/codemirror $DES/js/ +cp -R src/js/scriptlets $DES/js/ +cp -R src/js/wasm $DES/js/ cp -R src/lib $DES/ cp -R src/web_accessible_resources $DES/ cp -R src/_locales $DES/ diff --git a/tools/import-war.py b/tools/import-war.py deleted file mode 100755 index e4f4f57fe..000000000 --- a/tools/import-war.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 - -import base64 -import hashlib -import os -import re -import sys - -if len(sys.argv) == 1 or not sys.argv[1]: - raise SystemExit('Build dir missing.') - -# resource_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..') -build_dir = os.path.abspath(sys.argv[1]) - -# Read list of resource tokens to convert -to_import = set() -with open('./src/web_accessible_resources/to-import.txt', 'r') as f: - for line in f: - line = line.strip() - if len(line) != 0 and line[0] != '#': - to_import.add(line) - -# https://github.com/gorhill/uBlock/issues/3636 -safe_exts = { 'javascript': 'js', 'plain': 'txt' } - -imported = [] - -# scan the file until a resource to import is found -def find_next_resource(f): - for line in f: - line = line.strip() - if len(line) == 0 or line[0] == '#': - continue - parts = line.partition(' ') - if parts[0] in to_import: - return (parts[0], parts[2].strip()) - return ('', '') - -def safe_filename_from_token(token, mime): - h = hashlib.md5() - h.update(bytes(token, 'utf-8')) - name = h.hexdigest() - # extract file extension from mime - match = re.search('^[^/]+/([^\s;]+)', mime) - if match: - ext = match.group(1) - if ext in safe_exts: - ext = safe_exts[ext] - name += '.' + ext - return name - -def import_resource(f, token, mime): - isBinary = mime.endswith(';base64') - lines = [] - for line in f: - if line.strip() == '': - break - if line.lstrip()[0] == '#': - continue - if isBinary: - line = line.strip() - lines.append(line) - filename = safe_filename_from_token(token, mime) - filepath = os.path.join(build_dir, 'web_accessible_resources', filename) - filedata = ''.join(lines) - if isBinary: - filedata = base64.b64decode(filedata) - else: - filedata = bytes(filedata, 'utf-8') - with open(filepath, 'wb') as fo: - fo.write(filedata) - imported.append(token + '\n\t' + filename) - -# Read content of the resources to convert -# - At this point, it is assumed resources.txt has been imported into the -# package. -resources_filename = os.path.join(build_dir, 'assets/ublock/resources.txt') -with open(resources_filename, 'r') as f: - while True: - token, mime = find_next_resource(f) - if token == '': - break - import_resource(f, token, mime) - -# Output associations -content = '' -with open('./src/web_accessible_resources/imported.txt', 'r') as f: - content = f.read() + '\n'.join(imported) - filename = os.path.join(build_dir, 'web_accessible_resources/imported.txt') - with open(filename, 'w') as f: - f.write(content) - diff --git a/tools/make-browser.sh b/tools/make-browser.sh new file mode 100755 index 000000000..700157d19 --- /dev/null +++ b/tools/make-browser.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +DES=dist/build/uBlock0.browser + +mkdir -p $DES/js +cp src/js/base64-custom.js $DES/js +cp src/js/biditrie.js $DES/js +cp src/js/filtering-context.js $DES/js +cp src/js/globals.js $DES/js +cp src/js/hntrie.js $DES/js +cp src/js/static-filtering-parser.js $DES/js +cp src/js/static-net-filtering.js $DES/js +cp src/js/static-filtering-io.js $DES/js +cp src/js/text-iterators.js $DES/js +cp src/js/uri-utils.js $DES/js + +mkdir -p $DES/js/wasm +cp -R src/js/wasm $DES/js/ + +mkdir -p $DES/lib +cp -R src/lib/punycode.js $DES/lib/ +cp -R src/lib/publicsuffixlist $DES/lib/ +cp -R src/lib/regexanalyzer $DES/lib/ + +mkdir -p $DES/data +cp -R ../uAssets/thirdparties/publicsuffix.org/list/* \ + $DES/data +cp -R ../uAssets/thirdparties/easylist-downloads.adblockplus.org/* \ + $DES/data + +cp platform/browser/*.html $DES/ +cp platform/browser/*.js $DES/ +cp LICENSE.txt $DES/ diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh index f32bd2330..047146026 100755 --- a/tools/make-chromium.sh +++ b/tools/make-chromium.sh @@ -13,9 +13,9 @@ bash ./tools/copy-common-files.sh $DES # Chromium-specific echo "*** uBlock0.chromium: Copying chromium-specific files" -cp platform/chromium/*.js $DES/js/ -cp platform/chromium/*.html $DES/ -cp platform/chromium/*.json $DES/ +cp platform/chromium/*.js $DES/js/ +cp platform/chromium/*.html $DES/ +cp platform/chromium/*.json $DES/ # Chrome store-specific cp -R $DES/_locales/nb $DES/_locales/no diff --git a/tools/make-firefox.sh b/tools/make-firefox.sh index 31ec7ae20..b50eae886 100755 --- a/tools/make-firefox.sh +++ b/tools/make-firefox.sh @@ -14,11 +14,11 @@ bash ./tools/copy-common-files.sh $DES # Firefox-specific echo "*** uBlock0.firefox: Copying firefox-specific files" -cp platform/firefox/*.json $DES/ -cp platform/firefox/*.js $DES/js/ +cp platform/firefox/*.json $DES/ +cp platform/firefox/*.js $DES/js/ # Firefox store-specific -cp -R $DES/_locales/nb $DES/_locales/no +cp -R $DES/_locales/nb $DES/_locales/no # Firefox/webext-specific rm $DES/img/icon_128.png diff --git a/tools/make-nodejs.sh b/tools/make-nodejs.sh new file mode 100755 index 000000000..d217ba953 --- /dev/null +++ b/tools/make-nodejs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +DES=dist/build/uBlock0.nodejs + +mkdir -p $DES/js +cp src/js/base64-custom.js $DES/js +cp src/js/biditrie.js $DES/js +cp src/js/filtering-context.js $DES/js +cp src/js/globals.js $DES/js +cp src/js/hntrie.js $DES/js +cp src/js/static-filtering-parser.js $DES/js +cp src/js/static-net-filtering.js $DES/js +cp src/js/static-filtering-io.js $DES/js +cp src/js/text-iterators.js $DES/js +cp src/js/uri-utils.js $DES/js + +mkdir -p $DES/lib +cp -R src/lib/punycode.js $DES/lib/ +cp -R src/lib/publicsuffixlist $DES/lib/ +cp -R src/lib/regexanalyzer $DES/lib/ + +mkdir -p $DES/data +cp -R ../uAssets/thirdparties/publicsuffix.org/list/* \ + $DES/data +cp -R ../uAssets/thirdparties/easylist-downloads.adblockplus.org/* \ + $DES/data + +cp platform/nodejs/*.js $DES/ +cp platform/nodejs/*.json $DES/ +cp LICENSE.txt $DES/ diff --git a/tools/make-opera.sh b/tools/make-opera.sh index 8a6c7443a..0e95e1a41 100755 --- a/tools/make-opera.sh +++ b/tools/make-opera.sh @@ -13,12 +13,12 @@ bash ./tools/copy-common-files.sh $DES # Chromium-specific echo "*** uBlock0.opera: Copying chromium-specific files" -cp platform/chromium/*.js $DES/js/ -cp platform/chromium/*.html $DES/ +cp platform/chromium/*.js $DES/js/ +cp platform/chromium/*.html $DES/ # Opera-specific echo "*** uBlock0.opera: Copying opera-specific files" -cp platform/opera/manifest.json $DES/ +cp platform/opera/manifest.json $DES/ rm -r $DES/_locales/az rm -r $DES/_locales/cv diff --git a/tools/make-safari-meta.py b/tools/make-safari-meta.py deleted file mode 100644 index ad6a63cd4..000000000 --- a/tools/make-safari-meta.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -import os -import json -import sys -from io import open -from time import time -from shutil import rmtree -from collections import OrderedDict - -if len(sys.argv) == 1 or not sys.argv[1]: - raise SystemExit('Build dir missing.') - - -def mkdirs(path): - try: - os.makedirs(path) - finally: - return os.path.exists(path) - -pj = os.path.join -build_dir = os.path.abspath(sys.argv[1]) - -description = '' - -# locales -locale_dir = pj(build_dir, '_locales') - -for alpha2 in sorted(os.listdir(locale_dir)): - locale_path = pj(locale_dir, alpha2, 'messages.json') - with open(locale_path, encoding='utf-8') as f: - string_data = json.load(f, object_pairs_hook=OrderedDict) - - if alpha2 == 'en': - description = string_data['extShortDesc']['message'] - - for string_name in string_data: - string_data[string_name] = string_data[string_name]['message'] - - rmtree(pj(locale_dir, alpha2)) - - alpha2 = alpha2.replace('_', '-') - locale_path = pj(locale_dir, alpha2 + '.json') - - mkdirs(pj(locale_dir)) - - with open(locale_path, 'wb') as f: - f.write(json.dumps(string_data, ensure_ascii=False).encode('utf8')) - - -# update Info.plist -proj_dir = pj(os.path.split(os.path.abspath(__file__))[0], '..') -chromium_manifest = pj(proj_dir, 'platform', 'chromium', 'manifest.json') - -with open(chromium_manifest, encoding='utf-8') as m: - manifest = json.load(m) - -manifest['buildNumber'] = int(time()) -manifest['description'] = description - -info_plist = pj(build_dir, 'Info.plist') - -with open(info_plist, 'r+t', encoding='utf-8', newline='\n') as f: - info_plist = f.read() - f.seek(0) - - f.write(info_plist.format(**manifest)) - -# update Update.plist -update_plist = pj(proj_dir, 'platform', 'safari', 'Update.plist') -update_plist_build = pj(build_dir, '..', os.path.basename(update_plist)) - -with open(update_plist_build, 'wt', encoding='utf-8', newline='\n') as f: - with open(update_plist, encoding='utf-8') as u: - update_plist = u.read() - - f.write(update_plist.format(**manifest)) diff --git a/tools/make-safari.sh b/tools/make-safari.sh deleted file mode 100755 index db96811eb..000000000 --- a/tools/make-safari.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# -# This script assumes an OS X or *NIX environment - -echo "*** uBlock.safariextension: Copying files..." - -DES=dist/build/uBlock.safariextension -rm -rf $DES -mkdir -p $DES - -cp -R assets $DES/ -rm $DES/assets/*.sh -cp -R src/css $DES/ -cp -R src/img $DES/ -cp -R src/js $DES/ -cp -R src/lib $DES/ -cp -R src/_locales $DES/ -cp src/*.html $DES/ -mv $DES/img/icon_128.png $DES/Icon.png -cp platform/safari/*.js $DES/js/ -cp -R platform/safari/img $DES/ -cp platform/safari/Info.plist $DES/ -cp platform/safari/Settings.plist $DES/ -cp LICENSE.txt $DES/ - -echo "*** uBlock.safariextension: Generating Info.plist..." -python tools/make-safari-meta.py $DES/ - -if [ "$1" = all ]; then - echo "*** Use Safari's Extension Builder to create the signed uBlock extension package -- can't automate it." -fi - -echo "*** uBlock.safariextension: Done." diff --git a/tools/make-thunderbird.sh b/tools/make-thunderbird.sh index 16a17121f..590a2bce9 100755 --- a/tools/make-thunderbird.sh +++ b/tools/make-thunderbird.sh @@ -13,7 +13,7 @@ echo "*** uBlock0.thunderbird: copying common files" bash ./tools/copy-common-files.sh $DES echo "*** uBlock0.firefox: Copying firefox-specific files" -cp platform/firefox/*.js $DES/js/ +cp platform/firefox/*.js $DES/js/ echo "*** uBlock0.firefox: Copying thunderbird-specific files" cp platform/thunderbird/manifest.json $DES/ diff --git a/tools/make-webext-meta.py b/tools/make-webext-meta.py deleted file mode 100644 index 8639d2b37..000000000 --- a/tools/make-webext-meta.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 - -import os -import json -import re -import sys - -if len(sys.argv) == 1 or not sys.argv[1]: - raise SystemExit('Build dir missing.') - -proj_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..') -build_dir = os.path.abspath(sys.argv[1]) - -version = '' -with open(os.path.join(proj_dir, 'dist', 'version')) as f: - version = f.read().strip() - -webext_manifest = {} -webext_manifest_file = os.path.join(build_dir, 'manifest.json') -with open(webext_manifest_file, encoding='utf-8') as f2: - webext_manifest = json.load(f2) - -webext_manifest['version'] = version - -match = re.search('^\d+\.\d+\.\d+\.\d+$', version) -if match: - webext_manifest['name'] += ' development build' - webext_manifest['short_name'] += ' dev build' - webext_manifest['browser_action']['default_title'] += ' dev build' -else: - # https://bugzilla.mozilla.org/show_bug.cgi?id=1459007 - # By design Firefox opens the sidebar with new installation of - # uBO when sidebar_action is present in the manifest. - # Remove sidebarAction support for stable release of uBO. - del webext_manifest['sidebar_action'] - -with open(webext_manifest_file, mode='w', encoding='utf-8') as f2: - json.dump(webext_manifest, f2, indent=2, separators=(',', ': '), sort_keys=True) - f2.write('\n') diff --git a/tools/make-webext.sh b/tools/make-webext.sh deleted file mode 100755 index 80e66a0f4..000000000 --- a/tools/make-webext.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# -# This script assumes a linux environment - -# https://github.com/uBlockOrigin/uBlock-issues/issues/217 -set -e - -echo "*** uBlock0.webext: Creating web store package" - -DES=dist/build/uBlock0.webext -rm -rf $DES -mkdir -p $DES - -echo "*** uBlock0.webext: copying common files" -bash ./tools/copy-common-files.sh $DES - -cp -R $DES/_locales/nb $DES/_locales/no - -cp platform/webext/manifest.json $DES/ - -# https://github.com/uBlockOrigin/uBlock-issues/issues/407 -echo "*** uBlock0.webext: concatenating vapi-webrequest.js" -cat platform/chromium/vapi-webrequest.js > /tmp/vapi-webrequest.js -grep -v "^'use strict';$" platform/firefox/vapi-webrequest.js >> /tmp/vapi-webrequest.js -mv /tmp/vapi-webrequest.js $DES/js/vapi-webrequest.js - -echo "*** uBlock0.webext: Generating meta..." -python3 tools/make-webext-meta.py $DES/ - -if [ "$1" = all ]; then - echo "*** uBlock0.webext: Creating package..." - pushd $DES > /dev/null - zip ../$(basename $DES).xpi -qr * - popd > /dev/null -elif [ -n "$1" ]; then - echo "*** uBlock0.webext: Creating versioned package..." - pushd $DES > /dev/null - zip ../$(basename $DES).xpi -qr * -O ../uBlock0_"$1".webext.xpi - popd > /dev/null -fi - -echo "*** uBlock0.webext: Package done."