Run tests on request data (#3828)

platform/npm/.npmignore

@@ -0,0 +1,2 @@
+assets/
+tests/

platform/npm/package-lock.json

@@ -1267,6 +1267,11 @@
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
       "dev": true
     },
+    "scaling-palm-tree": {
+      "version": "github:mjethani/scaling-palm-tree#15cf1ab37e038771e1ff8005edc46d95f176739f",
+      "from": "github:mjethani/scaling-palm-tree#15cf1ab37e038771e1ff8005edc46d95f176739f",
+      "dev": true
+    },
     "semver": {
       "version": "7.3.5",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",

platform/npm/package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "scripts": {
     "build": "node build.js",
-    "lint": "eslint js/ *.js tests/",
+    "lint": "eslint js/ *.js tests/*.js",
     "test": "c8 --include=index.js --include=js/**/*.js node test.js --mocha"
   },
   "repository": {
@@ -33,6 +33,7 @@
     "c8": "^7.8.0",
     "eslint": "^7.32.0",
     "esm-world": "github:mjethani/esm-world#0e5a77a5c0cb22de28616bba9ed7247dee218fb3",
-    "mocha": "^9.0.3"
+    "mocha": "^9.0.3",
+    "scaling-palm-tree": "github:mjethani/scaling-palm-tree#15cf1ab37e038771e1ff8005edc46d95f176739f"
   }
 }

platform/npm/test.js

@@ -32,7 +32,7 @@ import { promisify } from 'util';
 /******************************************************************************/
 
 async function spawnMocha() {
-    await promisify(spawn)('mocha', [ '--experimental-vm-modules', '--no-warnings', 'tests' ], { stdio: [ 'inherit', 'inherit', 'inherit' ] });
+    await promisify(spawn)('mocha', [ '--experimental-vm-modules', '--no-warnings', 'tests', '--reporter', 'progress' ], { stdio: [ 'inherit', 'inherit', 'inherit' ] });
 }
 
 async function main() {

platform/npm/tests/request-data.js

@@ -0,0 +1,105 @@
+/*******************************************************************************
+
+    uBlock Origin - a browser extension to block requests.
+    Copyright (C) 2014-present Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+'use strict';
+
+/******************************************************************************/
+
+import { strict as assert } from 'assert';
+import { readFile } from 'fs/promises';
+import { createRequire } from 'module';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const require = createRequire(import.meta.url);
+
+const requests = require('scaling-palm-tree/requests.json');
+const results = require('./data/results.json');
+
+async function read(path) {
+    return readFile(resolve(__dirname, path), 'utf8');
+}
+
+describe('Request data', () => {
+    const typeMap = {
+        document: 'sub_frame',
+        stylesheet: 'stylesheet',
+        image: 'image',
+        media: 'media',
+        font: 'font',
+        script: 'script',
+        xhr: 'xmlhttprequest',
+        fetch: 'xmlhttprequest',
+        websocket: 'websocket',
+        ping: 'ping',
+
+        other: 'other',
+        eventsource: 'other',
+        manifest: 'other',
+        texttrack: 'other',
+    };
+
+    let engine = null;
+
+    before(async () => {
+        const { StaticNetFilteringEngine } = await import('../index.js');
+
+        engine = await StaticNetFilteringEngine.create();
+
+        await engine.useLists([
+            read('./data/assets/ublock/badware.txt')
+                .then(raw => ({ name: 'badware', raw })),
+            read('./data/assets/ublock/filters.txt')
+                .then(raw => ({ name: 'filters', raw })),
+            read('./data/assets/ublock/filters-2020.txt')
+                .then(raw => ({ name: 'filters-2020', raw })),
+            read('./data/assets/ublock/filters-2021.txt')
+                .then(raw => ({ name: 'filters-2021', raw })),
+            read('./data/assets/ublock/privacy.txt')
+                .then(raw => ({ name: 'privacy', raw })),
+            read('./data/assets/ublock/resource-abuse.txt')
+                .then(raw => ({ name: 'resource-abuse', raw })),
+            read('./data/assets/ublock/unbreak.txt')
+                .then(raw => ({ name: 'unbreak.txt', raw })),
+            read('./data/assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt')
+                .then(raw => ({ name: 'easylist', raw })),
+            read('./data/assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt')
+                .then(raw => ({ name: 'easyprivacy', raw })),
+            read('./data/assets/thirdparties/pgl.yoyo.org/as/serverlist')
+                .then(raw => ({ name: 'PGL', raw })),
+            read('./data/assets/thirdparties/urlhaus-filter/urlhaus-filter-online.txt')
+                .then(raw => ({ name: 'urlhaus', raw })),
+        ]);
+    });
+
+    for ( let i = 0; i < requests.length; i++ ) {
+        const { url, frameUrl, cpt } = requests[i];
+        const request = { url, originURL: frameUrl, type: typeMap[cpt] };
+
+        const expected = results[i];
+
+        it(`should ${expected === 1 ? 'block' : 'allow'} ${request.type} URL ${request.url} from origin ${request.originURL}`, () => {
+            assert.equal(engine.matchRequest(request), expected);
+        });
+    }
+});

tools/make-npm.sh

@@ -18,29 +18,19 @@ fi
 rm -rf $DES
 
 ./tools/make-nodejs.sh $DES
-
-UASSETS=submodules/uAssets
-
-# https://github.com/uBlockOrigin/uBlock-issues/issues/1664#issuecomment-888332409
-mkdir -p $DES/assets/thirdparties/publicsuffix.org/list
-THIRDPARTY=$UASSETS/thirdparties/publicsuffix.org
-node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/list/effective_tld_names.dat', 'utf8'))" \
-    > $DES/assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat
-
-mkdir -p $DES/data
-THIRDPARTY=$UASSETS/thirdparties/easylist-downloads.adblockplus.org
-node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easylist.txt', 'utf8'))" \
-    > $DES/data/easylist.json
-node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easyprivacy.txt', 'utf8'))" \
-    > $DES/data/easyprivacy.json
+./tools/make-assets.sh $DES
 
 # Target-specific
-cp    platform/npm/*.json  $DES/
-cp    platform/npm/.*.json $DES/
-cp    platform/npm/*.js    $DES/
-cp -R platform/npm/tests   $DES/
+cp    platform/npm/.npmignore  $DES/
+cp    platform/npm/*.json      $DES/
+cp    platform/npm/.*.json     $DES/
+cp    platform/npm/*.js        $DES/
+cp -R platform/npm/tests       $DES/
 
 cd $DES
+cd tests/data
+tar xzf bundle.tgz
+cd -
 npm run build
 tarballname=$(npm pack 2> /dev/null)
 if [ "$1" ]; then