Rewrite static filtering parser

This commit is a rewrite of the static filtering parser into a
tree-based data structure, for easier maintenance and better
abstraction of parsed filters.

This simplifies greatly syntax coloring of filters and also
simplify extending filter syntax.

The minimum version of Chromium-based browsers has been raised
to version 73 because of usage of String.matchAll().

.jshintrc

@@ -2,7 +2,7 @@
 	"browser": true,
 	"devel": true,
 	"eqeqeq": true,
-	"esversion": 8,
+	"esversion": 9,
 	"globals": {
 		"chrome": false,     // global variable in Chromium, Chrome, Opera
 		"globalThis": false,

platform/browser/main.js

@@ -29,7 +29,7 @@ import punycode from './lib/punycode.js';
 import staticNetFilteringEngine from './js/static-net-filtering.js';
 import { FilteringContext } from './js/filtering-context.js';
 import { LineIterator } from './js/text-utils.js';
-import { StaticFilteringParser } from './js/static-filtering-parser.js';
+import * as sfp from './js/static-filtering-parser.js';
 
 import {
     CompiledListReader,
@@ -40,10 +40,11 @@ import {
 
 function compileList(rawText, writer) {
     const lineIter = new LineIterator(rawText);
-    const parser = new StaticFilteringParser(true);
+    const parser = new sfp.AstFilterParser({
-    const compiler = staticNetFilteringEngine.createCompiler(parser);
+        interactive: true,
-
+        maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
-    parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH);
+    });
+    const compiler = staticNetFilteringEngine.createCompiler();
 
     while ( lineIter.eot() === false ) {
         let line = lineIter.next();
@@ -52,13 +53,10 @@ function compileList(rawText, writer) {
             if ( lineIter.peek(4) !== '    ' ) { break; }
             line = line.slice(0, -2).trim() + lineIter.next().trim();
         }
-        parser.analyze(line);
+        parser.parse(line);
 
-        if ( parser.shouldIgnore() ) { continue; }
+        if ( parser.isFilter() === false ) { continue; }
-        if ( parser.category !== parser.CATStaticNetFilter ) { continue; }
+        if ( parser.isNetworkFilter() === false ) { continue; }
-        if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
-            continue;
-        }
         if ( compiler.compile(parser, writer) ) { continue; }
         if ( compiler.error !== undefined ) {
             console.info(JSON.stringify({

platform/chromium/manifest.json

@@ -74,7 +74,7 @@
   },
   "incognito": "split",
   "manifest_version": 2,
-  "minimum_chrome_version": "66.0",
+  "minimum_chrome_version": "73.0",
   "name": "uBlock Origin",
   "options_ui": {
     "page": "dashboard.html",

platform/mv3/make-rulesets.js

@@ -30,8 +30,8 @@ import process from 'process';
 import { createHash } from 'crypto';
 import redirectResourcesMap from './js/redirect-resources.js';
 import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js';
-import { StaticFilteringParser } from './js/static-filtering-parser.js';
 import { fnameFromFileId } from './js/utils.js';
+import * as sfp from './js/static-filtering-parser.js';
 
 /******************************************************************************/
 
@@ -202,7 +202,7 @@ async function fetchAsset(assetDetails) {
             );
         }
         parts = await Promise.all(newParts);
-        parts = StaticFilteringParser.utils.preparser.expandIncludes(parts, env);
+        parts = sfp.utils.preparser.expandIncludes(parts, env);
     }
     const text = parts.join('\n');
 

platform/nodejs/index.js

@@ -38,7 +38,7 @@ import publicSuffixList from './lib/publicsuffixlist/publicsuffixlist.js';
 import snfe from './js/static-net-filtering.js';
 import { FilteringContext } from './js/filtering-context.js';
 import { LineIterator } from './js/text-utils.js';
-import { StaticFilteringParser } from './js/static-filtering-parser.js';
+import * as sfp from './js/static-filtering-parser.js';
 
 import {
     CompiledListReader,
@@ -117,7 +117,9 @@ function compileList({ name, raw }, compiler, writer, options = {}) {
         writer.properties.set('name', name);
     }
 
-    const { parser } = compiler;
+    const parser = new sfp.AstFilterParser({
+        maxTokenLength: snfe.MAX_TOKEN_LENGTH,
+    });
 
     while ( lineIter.eot() === false ) {
         let line = lineIter.next();
@@ -125,13 +127,10 @@ function compileList({ name, raw }, compiler, writer, options = {}) {
             if ( lineIter.peek(4) !== '    ' ) { break; }
             line = line.slice(0, -2).trim() + lineIter.next().trim();
         }
-        parser.analyze(line);
+        parser.parse(line);
-        if ( parser.shouldIgnore() ) { continue; }
+        if ( parser.isFilter() === false ) { continue; }
-        if ( parser.category !== parser.CATStaticNetFilter ) { continue; }
+        if ( parser.isNetworkFilter() === false ) { continue; }
-        if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
+        if ( compiler.compile(parser, writer) ) { continue; }
-            continue;
-        }
-        if ( compiler.compile(writer) ) { continue; }
         if ( compiler.error !== undefined && events !== undefined ) {
             options.events.push({
                 type: 'error',
@@ -164,7 +163,7 @@ async function useLists(lists, options = {}) {
         if ( typeof compiled !== 'string' || compiled === '' ) {
             const writer = new CompiledListWriter();
             if ( compiler === null ) {
-                compiler = snfe.createCompiler(new StaticFilteringParser());
+                compiler = snfe.createCompiler();
             }
             compiled = compileList(list, compiler, writer, options);
         }

platform/opera/manifest.json

@@ -73,7 +73,7 @@
   },
   "incognito": "split",
   "manifest_version": 2,
-  "minimum_opera_version": "53.0",
+  "minimum_opera_version": "60.0",
   "name": "uBlock Origin",
   "options_page": "dashboard.html",
   "permissions": [

src/css/codemirror.css

@@ -108,6 +108,12 @@
     text-decoration-style: solid;
     text-decoration-line: underline;
     }
+.cm-s-default .cm-unicode {
+    text-underline-position: under;
+    text-decoration-color: var(--sf-unicode-ink);
+    text-decoration-style: dashed;
+    text-decoration-line: underline;
+    }
 .cm-s-default .cm-tag {
     color: var(--sf-tag-ink);
     }

src/css/themes/default.css

@@ -284,6 +284,7 @@
     --sf-notice-ink: var(--ink-4);
     --sf-readonly-ink: var(--ink-3);
     --sf-tag-ink: #006e2e /* h:135 S:100 Luv:40 */;
+    --sf-unicode-ink: var(--ink-1);
     --sf-value-ink: #974900 /* h:30 S:100 Luv:40 */;
     --sf-variable-ink: var(--ink-1);
     --sf-warning-ink: #e49d00; /* h:50 S:100 Luv:70 */

src/js/assets.js

@@ -26,8 +26,8 @@
 import cacheStorage from './cachestorage.js';
 import logger from './logger.js';
 import µb from './background.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
 import { i18n$ } from './i18n.js';
+import * as sfp from './static-filtering-parser.js';
 
 /******************************************************************************/
 
@@ -269,7 +269,7 @@ assets.fetchFilterList = async function(mainlistURL) {
             }
             if ( result instanceof Object === false ) { continue; }
             const content = result.content;
-            const slices = StaticFilteringParser.utils.preparser.splitter(
+            const slices = sfp.utils.preparser.splitter(
                 content,
                 vAPI.webextFlavor.env
             );

src/js/background.js

@@ -176,8 +176,8 @@ const µBlock = {  // jshint ignore:line
 
     // Read-only
     systemSettings: {
-        compiledMagic: 52,  // Increase when compiled format changes
+        compiledMagic: 54,  // Increase when compiled format changes
-        selfieMagic: 52,    // Increase when selfie format changes
+        selfieMagic: 54,    // Increase when selfie format changes
     },
 
     // https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501

src/js/codemirror/ubo-static-filtering.js

@@ -25,7 +25,7 @@
 
 /******************************************************************************/
 
-import { StaticFilteringParser } from '../static-filtering-parser.js';
+import * as sfp from '../static-filtering-parser.js';
 
 /******************************************************************************/
 
@@ -39,339 +39,219 @@ let hintHelperRegistered = false;
 /******************************************************************************/
 
 CodeMirror.defineMode('ubo-static-filtering', function() {
-    if ( StaticFilteringParser instanceof Object === false ) { return; }
+    if ( sfp.AstFilterParser instanceof Object === false ) { return; }
-    const parser = new StaticFilteringParser({
+    const astParser = new sfp.AstFilterParser({
         interactive: true,
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
     });
+    const astWalker = astParser.getWalker();
+    let currentWalkerNode = 0;
+    let lastNetOptionType = 0;
 
-    const reURL = /\bhttps?:\/\/\S+/;
+    const redirectTokenStyle = node => {
-    const rePreparseDirectives = /^!#(?:if|endif|include )\b/;
+        const rawToken = astParser.getNodeString(node);
-    const rePreparseIfDirective = /^(!#if ?)(.*)$/;
+        const { token } = sfp.parseRedirectValue(rawToken);
-    let parserSlot = 0;
+        return redirectNames.has(token) ? 'value' : 'value warning';
-    let netOptionValueMode = false;
-
-    const colorCommentSpan = function(stream) {
-        const { string, pos } = stream;
-        if ( rePreparseDirectives.test(string) === false ) {
-            const match = reURL.exec(string.slice(pos));
-            if ( match !== null ) {
-                if ( match.index === 0 ) {
-                    stream.pos += match[0].length;
-                    return 'comment link';
-                }
-                stream.pos += match.index;
-                return 'comment';
-            }
-            stream.skipToEnd();
-            return 'comment';
-        }
-        const match = rePreparseIfDirective.exec(string);
-        if ( match === null ) {
-            stream.skipToEnd();
-            return 'directive';
-        }
-        if ( pos < match[1].length ) {
-            stream.pos += match[1].length;
-            return 'directive';
-        }
-        stream.skipToEnd();
-        if ( match[1].endsWith(' ') === false ) {
-            return 'error strong';
-        }
-        if ( preparseDirectiveTokens.size === 0 ) {
-            return 'positive strong';
-        }
-        let token = match[2];
-        const not = token.startsWith('!');
-        if ( not ) {
-            token = token.slice(1);
-        }
-        if ( preparseDirectiveTokens.has(token) === false ) {
-            return 'error strong';
-        }
-        if ( not !== preparseDirectiveTokens.get(token) ) {
-            return 'positive strong';
-        }
-        return 'negative strong';
     };
 
-    const colorExtHTMLPatternSpan = function(stream) {
+    const colorFromAstNode = function() {
-        const { i } = parser.patternSpan;
+        if ( astParser.nodeIsEmptyString(currentWalkerNode) ) { return '+'; }
-        if ( stream.pos === parser.slices[i+1] ) {
+        if ( astParser.getNodeFlags(currentWalkerNode, sfp.NODE_FLAG_ERROR) !== 0 ) {
-            stream.pos += 1;
-            return 'def';
-        }
-        stream.skipToEnd();
-        return 'variable';
-    };
-
-    const colorExtScriptletPatternSpan = function(stream) {
-        const { pos, string } = stream;
-        const { i, len } = parser.patternSpan;
-        const patternBeg = parser.slices[i+1];
-        if ( pos === patternBeg ) {
-            stream.pos = pos + 4;
-            return 'def';
-        }
-        if ( len > 3 ) {
-            if ( pos === patternBeg + 4 ) {
-                const match = /^[^,)]+/.exec(string.slice(pos));
-                const token = match && match[0].trim();
-                if ( token && scriptletNames.has(token) === false ) {
-                    stream.pos = pos + match[0].length;
-                    return 'warning';
-                }
-            }
-            const r = parser.slices[i+len+1] - 1;
-            if ( pos < r ) {
-                stream.pos = r;
-                return 'variable';
-            }
-            if ( pos === r ) {
-                stream.pos = pos + 1;
-                return 'def';
-            }
-        }
-        stream.skipToEnd();
-        return 'variable';
-    };
-
-    const colorExtPatternSpan = function(stream) {
-        if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
-            return colorExtScriptletPatternSpan(stream);
-        }
-        if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
-            return colorExtHTMLPatternSpan(stream);
-        }
-        stream.skipToEnd();
-        return 'variable';
-    };
-
-    const colorExtSpan = function(stream) {
-        if ( parserSlot < parser.optionsAnchorSpan.i ) {
-            const style = (parser.slices[parserSlot] & parser.BITComma) === 0
-                ? 'value'
-                : 'def';
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
-            return style;
-        }
-        if (
-            parserSlot >= parser.optionsAnchorSpan.i &&
-            parserSlot < parser.patternSpan.i
-        ) {
-            const style = (parser.flavorBits & parser.BITFlavorException) !== 0
-                ? 'tag'
-                : 'def';
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
-            return `${style} strong`;
-        }
-        if (
-            parserSlot >= parser.patternSpan.i &&
-            parserSlot < parser.rightSpaceSpan.i
-        ) {
-            return colorExtPatternSpan(stream);
-        }
-        stream.skipToEnd();
-        return null;
-    };
-
-    const colorNetOptionValueSpan = function(stream, bits) {
-        const { pos, string } = stream;
-        let style;
-        // Warn about unknown redirect tokens.
-        if (
-            string.charCodeAt(pos - 1) === 0x3D /* '=' */ &&
-            /[$,](redirect(-rule)?|rewrite)=$/.test(string.slice(0, pos))
-        ) {
-            style = 'value';
-            const end = parser.skipUntil(
-                parserSlot,
-                parser.commentSpan.i,
-                parser.BITComma
-            );
-            const raw = parser.strFromSlices(parserSlot, end - 3);
-            const { token } = StaticFilteringParser.parseRedirectValue(raw);
-            if ( redirectNames.has(token) === false ) {
-                style += ' warning';
-            }
-            stream.pos += raw.length;
-            parserSlot = end;
-            return style;
-        }
-        if ( (bits & parser.BITTilde) !== 0 ) {
-            style = 'keyword strong';
-        } else if ( (bits & parser.BITPipe) !== 0 ) {
-            style = 'def';
-        }
-        stream.pos += parser.slices[parserSlot+2];
-        parserSlot += 3;
-        return style || 'value';
-    };
-
-    // https://github.com/uBlockOrigin/uBlock-issues/issues/760#issuecomment-951146371
-    //   Quick fix: auto-escape commas.
-    const colorNetOptionSpan = function(stream) {
-        const [ slotBits, slotPos, slotLen ] =
-            parser.slices.slice(parserSlot, parserSlot+3);
-        if ( (slotBits & parser.BITComma) !== 0 ) {
-            if ( /^,\d*?\}/.test(parser.raw.slice(slotPos)) === false ) {
-                netOptionValueMode = false;
-                stream.pos += slotLen;
-                parserSlot += 3;
-                return 'def strong';
-            }
-        }
-        if ( netOptionValueMode ) {
-            return colorNetOptionValueSpan(stream, slotBits);
-        }
-        if ( (slotBits & parser.BITTilde) !== 0 ) {
-            stream.pos += slotLen;
-            parserSlot += 3;
-            return 'keyword strong';
-        }
-        if ( (slotBits & parser.BITEqual) !== 0 ) {
-            netOptionValueMode = true;
-            stream.pos += slotLen;
-            parserSlot += 3;
-            return 'def';
-        }
-        parserSlot = parser.skipUntil(
-            parserSlot,
-            parser.commentSpan.i,
-            parser.BITComma | parser.BITEqual
-        );
-        stream.pos = parser.slices[parserSlot+1];
-        return 'def';
-    };
-
-    const colorNetSpan = function(stream) {
-        if ( parserSlot < parser.exceptionSpan.i ) {
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
-            return null;
-        }
-        if (
-            parserSlot === parser.exceptionSpan.i &&
-            parser.exceptionSpan.len !== 0
-        ) {
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
-            return 'tag strong';
-        }
-        if (
-            parserSlot === parser.patternLeftAnchorSpan.i &&
-            parser.patternLeftAnchorSpan.len !== 0 ||
-            parserSlot === parser.patternRightAnchorSpan.i &&
-            parser.patternRightAnchorSpan.len !== 0
-        ) {
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
-            return 'keyword strong';
-        }
-        if (
-            parserSlot >= parser.patternSpan.i &&
-            parserSlot < parser.optionsAnchorSpan.i
-        ) {
-            if ( parser.patternIsRegex() ) {
-                stream.pos = parser.slices[parser.optionsAnchorSpan.i+1];
-                parserSlot = parser.optionsAnchorSpan.i;
-                return parser.patternIsTokenizable()
-                    ? 'variable notice'
-                    : 'variable warning';
-            }
-            if ( (parser.slices[parserSlot] & (parser.BITAsterisk | parser.BITCaret)) !== 0 ) {
-                stream.pos += parser.slices[parserSlot+2];
-                parserSlot += 3;
-                return 'keyword strong';
-            }
-            const nextSlot = parser.skipUntil(
-                parserSlot + 3,
-                parser.patternRightAnchorSpan.i,
-                parser.BITAsterisk | parser.BITCaret
-            );
-            stream.pos = parser.slices[nextSlot+1];
-            parserSlot = nextSlot;
-            return 'variable';
-        }
-        if (
-            parserSlot === parser.optionsAnchorSpan.i &&
-            parserSlot < parser.optionsSpan.i !== 0
-        ) {
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
-            return 'def strong';
-        }
-        if (
-            parserSlot >= parser.optionsSpan.i &&
-            parserSlot < parser.commentSpan.i
-        ) {
-            return colorNetOptionSpan(stream);
-        }
-        if (
-            parserSlot >= parser.commentSpan.i &&
-            parser.commentSpan.len !== 0
-        ) {
-            stream.skipToEnd();
-            return 'comment';
-        }
-        stream.skipToEnd();
-        return null;
-    };
-
-    const colorSpan = function(stream) {
-        if ( parser.category === parser.CATNone || parser.shouldIgnore() ) {
-            stream.skipToEnd();
-            return 'comment';
-        }
-        if ( parser.category === parser.CATComment ) {
-            return colorCommentSpan(stream);
-        }
-        if ( (parser.slices[parserSlot] & parser.BITError) !== 0 ) {
-            stream.pos += parser.slices[parserSlot+2];
-            parserSlot += 3;
             return 'error';
         }
-        if ( (parser.slices[parserSlot] & parser.BITIgnore) !== 0 ) {
+        const nodeType = astParser.getNodeType(currentWalkerNode);
-            stream.pos += parser.slices[parserSlot+2];
+        switch ( nodeType ) {
-            parserSlot += 3;
+            case sfp.NODE_TYPE_WHITESPACE:
-            return 'comment';
+                return '';
-        }
+            case sfp.NODE_TYPE_COMMENT:
-        if ( parser.category === parser.CATStaticExtFilter ) {
+                if ( astWalker.canGoDown() ) { break; }
-            const style = colorExtSpan(stream) || '';
+                return 'comment';
-            let flavor = '';
+            case sfp.NODE_TYPE_COMMENT_URL:
-            if ( (parser.flavorBits & parser.BITFlavorExtCosmetic) !== 0 ) {
+                return 'comment link';
-                flavor = 'line-cm-ext-dom';
+            case sfp.NODE_TYPE_IGNORE:
-            } else if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
+                return 'comment';
-                flavor = 'line-cm-ext-js';
+            case sfp.NODE_TYPE_PREPARSE_DIRECTIVE:
-            } else if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
+            case sfp.NODE_TYPE_PREPARSE_DIRECTIVE_VALUE:
-                flavor = 'line-cm-ext-html';
+                return 'directive';
+            case sfp.NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE: {
+                if ( preparseDirectiveTokens.size === 0 ) {
+                    return 'positive strong';
+                }
+                const raw = astParser.getNodeString(currentWalkerNode);
+                const not = raw.startsWith('!');
+                const token = not ? raw.slice(1) : raw;
+                if ( preparseDirectiveTokens.has(token) === false ) {
+                    return 'error strong';
+                }
+                return not === preparseDirectiveTokens.get(token)
+                    ? 'negative strong'
+                    : 'positive strong';
             }
-            return `${flavor} ${style}`.trim();
+            case sfp.NODE_TYPE_EXT_OPTIONS_ANCHOR:
+                return astParser.getFlags(sfp.AST_FLAG_IS_EXCEPTION)
+                    ? 'tag strong'
+                    : 'def strong';
+            case sfp.NODE_TYPE_EXT_DECORATION:
+                return 'def';
+            case sfp.NODE_TYPE_EXT_PATTERN_RAW:
+                if ( astWalker.canGoDown() ) { break; }
+                return 'variable';
+            case sfp.NODE_TYPE_EXT_PATTERN_COSMETIC:
+            case sfp.NODE_TYPE_EXT_PATTERN_HTML:
+                return 'variable';
+            case sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER:
+            case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET:
+                if ( astWalker.canGoDown() ) { break; }
+                return 'variable';
+            case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN: {
+                const token = astParser.getNodeString(currentWalkerNode);
+                if ( scriptletNames.has(token) === false ) {
+                    return 'warning';
+                }
+                return 'variable';
+            }
+            case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG:
+                return 'variable';
+            case sfp.NODE_TYPE_NET_EXCEPTION:
+                return 'tag strong';
+            case sfp.NODE_TYPE_NET_PATTERN:
+                if ( astWalker.canGoDown() ) { break; }
+                if ( astParser.isRegexPattern() ) {
+                    if ( astParser.getNodeFlags(currentWalkerNode, sfp.NODE_FLAG_PATTERN_UNTOKENIZABLE) !== 0 ) {
+                        return 'variable warning';
+                    }
+                    return 'variable notice';
+                }
+                return 'variable';
+            case sfp.NODE_TYPE_NET_PATTERN_PART:
+                return 'variable';
+            case sfp.NODE_TYPE_NET_PATTERN_PART_SPECIAL:
+                return 'keyword strong';
+            case sfp.NODE_TYPE_NET_PATTERN_PART_UNICODE:
+                return 'variable unicode';
+            case sfp.NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR:
+            case sfp.NODE_TYPE_NET_PATTERN_LEFT_ANCHOR:
+            case sfp.NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_NOT:
+                return 'keyword strong';
+            case sfp.NODE_TYPE_NET_OPTIONS_ANCHOR:
+            case sfp.NODE_TYPE_NET_OPTION_SEPARATOR:
+                lastNetOptionType = 0;
+                return 'def strong';
+            case sfp.NODE_TYPE_NET_OPTION_NAME_UNKNOWN:
+                lastNetOptionType = 0;
+                return 'error';
+            case sfp.NODE_TYPE_NET_OPTION_NAME_1P:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_3P:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_ALL:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CSP:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CSS:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_DOC:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_EMPTY:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FONT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FROM:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_MP4:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_NOOP:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_PING:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_XHR:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET:
+                lastNetOptionType = nodeType;
+                return 'def';
+            case sfp.NODE_TYPE_NET_OPTION_ASSIGN:
+                return 'def';
+            case sfp.NODE_TYPE_NET_OPTION_VALUE:
+                if ( astWalker.canGoDown() ) { break; }
+                switch ( lastNetOptionType ) {
+                    case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
+                    case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
+                        return redirectTokenStyle(currentWalkerNode);
+                    default:
+                        break;
+                }
+                return 'value';
+            case sfp.NODE_TYPE_OPTION_VALUE_NOT:
+                return 'keyword strong';
+            case sfp.NODE_TYPE_OPTION_VALUE_DOMAIN:
+                return 'value';
+            case sfp.NODE_TYPE_OPTION_VALUE_SEPARATOR:
+                return 'def';
+            default:
+                break;
         }
-        if ( parser.category === parser.CATStaticNetFilter ) {
+        return '+';
-            const style = colorNetSpan(stream);
-            return style ? `line-cm-net ${style}` : 'line-cm-net';
-        }
-        stream.skipToEnd();
-        return null;
     };
 
     return {
         lineComment: '!',
         token: function(stream) {
-            let style = '';
             if ( stream.sol() ) {
-                parser.analyze(stream.string);
+                astParser.parse(stream.string);
-                parser.analyzeExtra();
+                if ( astParser.getFlags(sfp.AST_FLAG_UNSUPPORTED) !== 0 ) {
-                parserSlot = 0;
+                    stream.skipToEnd();
-                netOptionValueMode = false;
+                    return 'error';
+                }
+                if ( astParser.getType() === sfp.AST_TYPE_NONE ) {
+                    stream.skipToEnd();
+                    return 'comment';
+                }
+                currentWalkerNode = astWalker.reset();
+            } else {
+                currentWalkerNode = astWalker.next();
             }
-            style += colorSpan(stream) || '';
+            let style = '';
-            if ( (parser.flavorBits & parser.BITFlavorError) !== 0 ) {
+            while ( currentWalkerNode !== 0 ) {
-                style += ' line-background-error';
+                style = colorFromAstNode(stream);
+                if ( style !== '+' ) { break; }
+                currentWalkerNode = astWalker.next();
+            }
+            if ( style === '+' ) {
+                stream.skipToEnd();
+                return null;
+            }
+            stream.pos = astParser.getNodeStringEnd(currentWalkerNode);
+            if ( astParser.isNetworkFilter() ) {
+                return style ? `line-cm-net ${style}` : 'line-cm-net';
+            }
+            if ( astParser.isExtendedFilter() ) {
+                let flavor = '';
+                if ( astParser.isCosmeticFilter() ) {
+                    flavor = 'line-cm-ext-dom';
+                } else if ( astParser.isScriptletFilter() ) {
+                    flavor = 'line-cm-ext-js';
+                } else if ( astParser.isHtmlFilter() ) {
+                    flavor = 'line-cm-ext-html';
+                }
+                if ( flavor !== '' ) {
+                    style = `${flavor} ${style}`;
+                }
             }
             style = style.trim();
             return style !== '' ? style : null;
@@ -409,9 +289,7 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
                 initHints();
             }
         },
-        get parser() {
+        parser: astParser,
-            return parser;
-        },
     };
 });
 
@@ -421,11 +299,14 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
 //   https://codemirror.net/demo/complete.html
 
 const initHints = function() {
-    if ( StaticFilteringParser instanceof Object === false ) { return; }
+    if ( sfp.AstFilterParser instanceof Object === false ) { return; }
 
-    const parser = new StaticFilteringParser();
+    const astParser = new sfp.AstFilterParser({
+        interactive: true,
+        nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
+    });
     const proceduralOperatorNames = new Map(
-        Array.from(parser.proceduralOperatorTokens)
+        Array.from(sfp.proceduralOperatorTokens)
              .filter(item => (item[1] & 0b01) !== 0)
     );
     const excludedHints = new Set([
@@ -560,16 +441,16 @@ const initHints = function() {
         }
         const assignPos = seedRight.indexOf('=');
         if ( assignPos !== -1 ) { seedRight = seedRight.slice(0, assignPos); }
-        const isException = parser.isException();
+        const isException = astParser.isException();
         const hints = [];
-        for ( let [ text, bits ] of parser.netOptionTokenDescriptors ) {
+        for ( let [ text, desc ] of sfp.netOptionTokenDescriptors ) {
             if ( excludedHints.has(text) ) { continue; }
-            if ( isNegated && (bits & parser.OPTCanNegate) === 0 ) { continue; }
+            if ( isNegated && desc.canNegate !== true ) { continue; }
             if ( isException ) {
-                if ( (bits & parser.OPTBlockOnly) !== 0 ) { continue; }
+                if ( desc.blockOnly ) { continue; }
             } else {
-                if ( (bits & parser.OPTAllowOnly) !== 0 ) { continue; }
+                if ( desc.allowOnly ) { continue; }
-                if ( (assignPos === -1) && (bits & parser.OPTMustAssign) !== 0 ) {
+                if ( (assignPos === -1) && desc.mustAssign ) {
                     text += '=';
                 }
             }
@@ -588,8 +469,11 @@ const initHints = function() {
     };
 
     const getNetHints = function(cursor, line) {
+        const patternNode = astParser.getBranchFromType(sfp.NODE_TYPE_NET_PATTERN_RAW);
+        if ( patternNode === 0 ) { return; }
+        const patternEnd = astParser.getNodeStringEnd(patternNode);
         const beg = cursor.ch;
-        if ( beg <= parser.slices[parser.optionsAnchorSpan.i+1] ) {
+        if ( beg <= patternEnd ) {
             return getNetPatternHints(cursor, line);
         }
         const lineBefore = line.slice(0, beg);
@@ -650,7 +534,7 @@ const initHints = function() {
         const matchRight = /^([^)]*)/.exec(line.slice(beg));
         if ( matchLeft === null || matchRight === null ) { return; }
         const hints = [];
-        for ( const hint of parser.removableHTTPHeaders ) {
+        for ( const hint of sfp.removableHTTPHeaders ) {
             hints.push(hint);
         }
         return pickBestHints(cursor, matchLeft[1], matchRight[1], hints);
@@ -697,29 +581,29 @@ const initHints = function() {
     CodeMirror.registerHelper('hint', 'ubo-static-filtering', function(cm) {
         const cursor = cm.getCursor();
         const line = cm.getLine(cursor.line);
-        parser.analyze(line);
+        astParser.parse(line);
-        if ( parser.category === parser.CATStaticExtFilter ) {
+        if ( astParser.isExtendedFilter() ) {
+            const anchorNode = astParser.getBranchFromType(sfp.NODE_TYPE_EXT_OPTIONS_ANCHOR);
+            if ( anchorNode === 0 ) { return; }
             let hints;
-            if ( cursor.ch <= parser.slices[parser.optionsAnchorSpan.i+1] ) {
+            if ( cursor.ch <= astParser.getNodeStringBeg(anchorNode) ) {
                 hints = getOriginHints(cursor, line);
-            } else if ( parser.hasFlavor(parser.BITFlavorExtScriptlet) ) {
+            } else if ( astParser.isScriptletFilter() ) {
                 hints = getExtScriptletHints(cursor, line);
-            } else if ( parser.hasFlavor(parser.BITFlavorExtResponseHeader) ) {
+            } else if ( astParser.isResponseheaderFilter() ) {
                 hints = getExtHeaderHints(cursor, line);
             } else {
                 hints = getExtSelectorHints(cursor, line);
             }
             return hints;
         }
-        if ( parser.category === parser.CATStaticNetFilter ) {
+        if ( astParser.isNetworkFilter() ) {
             return getNetHints(cursor, line);
         }
-        if ( parser.category === parser.CATComment ) {
+        if ( astParser.isComment() ) {
             return getCommentHints(cursor, line);
         }
-        if ( parser.category === parser.CATNone ) {
+        return getOriginHints(cursor, line);
-            return getOriginHints(cursor, line);
-        }
     });
 };
 

src/js/cosmetic-filtering.js

@@ -322,7 +322,7 @@ FilterContainer.prototype.compile = function(parser, writer) {
     //   Negated hostname means the filter applies to all non-negated hostnames
     //   of same filter OR globally if there is no non-negated hostnames.
     let applyGlobally = true;
-    for ( const { hn, not, bad } of parser.extOptions() ) {
+    for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
         if ( bad ) { continue; }
         if ( not === false ) {
             applyGlobally = false;

src/js/epicker-ui.js

@@ -26,7 +26,7 @@
 import './codemirror/ubo-static-filtering.js';
 
 import { hostnameFromURI } from './uri-utils.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
+import * as sfp from './static-filtering-parser.js';
 
 /******************************************************************************/
 /******************************************************************************/
@@ -110,13 +110,12 @@ const rawFilterFromTextarea = function() {
 const filterFromTextarea = function() {
     const filter = rawFilterFromTextarea();
     if ( filter === '' ) { return ''; }
-    const sfp = staticFilteringParser;
+    const parser = staticFilteringParser;
-    sfp.analyze(filter);
+    parser.parse(filter);
-    sfp.analyzeExtra();
+    if ( parser.isFilter() === false ) { return '!'; }
-    if ( sfp.shouldDiscard() ) { return '!'; }
+    if ( parser.isExtendedFilter() ) {
-    if ( sfp.category === sfp.CATStaticExtFilter ) {
+        if ( parser.isCosmeticFilter() === false ) { return '!'; }
-        if ( sfp.hasFlavor(sfp.BITFlavorExtCosmetic) === false ) { return '!'; }
+    } else if ( parser.isNetworkFilter() === false ) {
-    } else if ( sfp.category !== sfp.CATStaticNetFilter ) {
         return '!';
     }
     return filter;
@@ -829,7 +828,7 @@ const startPicker = function() {
     $id('candidateFilters').addEventListener('click', onCandidateClicked);
     $stor('#resultsetDepth input').addEventListener('input', onDepthChanged);
     $stor('#resultsetSpecificity input').addEventListener('input', onSpecificityChanged);
-    staticFilteringParser = new StaticFilteringParser({
+    staticFilteringParser = new sfp.AstFilterParser({
         interactive: true,
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
     });

src/js/html-filtering.js

@@ -26,8 +26,8 @@
 import logger from './logger.js';
 import µb from './background.js';
 import { sessionFirewall } from './filtering-engines.js';
-
 import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
+import * as sfp from './static-filtering-parser.js';
 
 /******************************************************************************/
 
@@ -314,7 +314,11 @@ htmlFilteringEngine.freeze = function() {
 };
 
 htmlFilteringEngine.compile = function(parser, writer) {
-    const { raw, compiled, exception } = parser.result;
+    const isException = parser.isException();
+    const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_HTML);
+    const headerName = parser.getNodeString(root);
+
+    const { raw, compiled } = parser.result;
     if ( compiled === undefined ) {
         const who = writer.properties.get('name') || '?';
         logger.writeOne({
@@ -329,7 +333,7 @@ htmlFilteringEngine.compile = function(parser, writer) {
 
     // Only exception filters are allowed to be global.
     if ( parser.hasOptions() === false ) {
-        if ( exception ) {
+        if ( isException ) {
             writer.push([ 64, '', 1, compiled ]);
         }
         return;
@@ -337,10 +341,10 @@ htmlFilteringEngine.compile = function(parser, writer) {
 
     // TODO: Mind negated hostnames, they are currently discarded.
 
-    for ( const { hn, not, bad } of parser.extOptions() ) {
+    for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
         if ( bad ) { continue; }
         let kind = 0;
-        if ( exception ) {
+        if ( isException ) {
             if ( not ) { continue; }
             kind |= 0b01;
         }

src/js/httpheader-filtering.js

@@ -27,8 +27,8 @@ import logger from './logger.js';
 import µb from './background.js';
 import { entityFromDomain } from './uri-utils.js';
 import { sessionFirewall } from './filtering-engines.js';
-
 import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
+import * as sfp from './static-filtering-parser.js';
 
 /******************************************************************************/
 
@@ -88,16 +88,17 @@ httpheaderFilteringEngine.freeze = function() {
 httpheaderFilteringEngine.compile = function(parser, writer) {
     writer.select('HTTPHEADER_FILTERS');
 
-    const { compiled, exception } = parser.result;
+    const isException = parser.isException();
-    const headerName = compiled.slice(15, -1);
+    const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER);
+    const headerName = parser.getNodeString(root);
 
     // Tokenless is meaningful only for exception filters.
-    if ( headerName === '' && exception === false ) { return; }
+    if ( headerName === '' && isException === false ) { return; }
 
     // Only exception filters are allowed to be global.
     if ( parser.hasOptions() === false ) {
-        if ( exception ) {
+        if ( isException ) {
-            writer.push([ 64, '', 1, compiled ]);
+            writer.push([ 64, '', 1, headerName ]);
         }
         return;
     }
@@ -106,16 +107,16 @@ httpheaderFilteringEngine.compile = function(parser, writer) {
     //   Ignore instances of exception filter with negated hostnames,
     //   because there is no way to create an exception to an exception.
 
-    for ( const { hn, not, bad } of parser.extOptions() ) {
+    for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
         if ( bad ) { continue; }
         let kind = 0;
-        if ( exception ) {
+        if ( isException ) {
             if ( not ) { continue; }
             kind |= 1;
         } else if ( not ) {
             kind |= 1;
         }
-        writer.push([ 64, hn, kind, compiled ]);
+        writer.push([ 64, hn, kind, headerName ]);
     }
 };
 

src/js/messaging.js

@@ -43,7 +43,7 @@ import { denseBase64 } from './base64-custom.js';
 import { dnrRulesetFromRawLists } from './static-dnr-filtering.js';
 import { i18n$ } from './i18n.js';
 import { redirectEngine } from './redirect-engine.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
+import * as sfp from './static-filtering-parser.js';
 
 import {
     permanentFirewall,
@@ -1515,9 +1515,9 @@ const onMessage = function(request, sender, callback) {
         if ( (request.hintUpdateToken || 0) === 0 ) {
             response.redirectResources = redirectEngine.getResourceDetails();
             response.preparseDirectiveTokens =
-                StaticFilteringParser.utils.preparser.getTokens(vAPI.webextFlavor.env);
+                sfp.utils.preparser.getTokens(vAPI.webextFlavor.env);
             response.preparseDirectiveHints =
-                StaticFilteringParser.utils.preparser.getHints();
+                sfp.utils.preparser.getHints();
             response.expertMode = µb.hiddenSettings.filterAuthorMode;
         }
         if ( request.hintUpdateToken !== µb.pageStoresToken ) {

src/js/redirect-resources.js

@@ -117,13 +117,6 @@ export default new Map([
     [ 'monkeybroker.js', {
         alias: 'd3pkae9owd2lcf.cloudfront.net/mb105.js',
     } ],
-    [ 'noeval.js', {
-        data: 'text',
-    } ],
-    [ 'noeval-silent.js', {
-        alias: 'silent-noeval.js',
-        data: 'text',
-    } ],
     [ 'nobab.js', {
         alias: 'bab-defuser.js',
         data: 'text',
@@ -131,6 +124,13 @@ export default new Map([
     [ 'nobab2.js', {
         data: 'text',
     } ],
+    [ 'noeval.js', {
+        data: 'text',
+    } ],
+    [ 'noeval-silent.js', {
+        alias: 'silent-noeval.js',
+        data: 'text',
+    } ],
     [ 'nofab.js', {
         alias: 'fuckadblock.js-3.2.0',
         data: 'text',
@@ -145,6 +145,9 @@ export default new Map([
         alias: 'noopmp4-1s',
         data: 'blob',
     } ],
+    [ 'noop.css', {
+        data: 'text',
+    } ],
     [ 'noop.html', {
         alias: 'noopframe',
     } ],

src/js/reverselookup.js

@@ -26,8 +26,8 @@
 import staticNetFilteringEngine from './static-net-filtering.js';
 import µb from './background.js';
 import { CompiledListWriter } from './static-filtering-io.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
 import { i18n$ } from './i18n.js';
+import * as sfp from './static-filtering-parser.js';
 
 import {
     domainFromHostname,
@@ -134,14 +134,14 @@ const fromNetFilter = async function(rawFilter) {
     if ( typeof rawFilter !== 'string' || rawFilter === '' ) { return; }
 
     const writer = new CompiledListWriter();
-    const parser = new StaticFilteringParser({
+    const parser = new sfp.AstFilterParser({
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
+        maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
     });
-    parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH);
+    parser.parse(rawFilter);
-    parser.analyze(rawFilter);
 
-    const compiler = staticNetFilteringEngine.createCompiler(parser);
+    const compiler = staticNetFilteringEngine.createCompiler();
-    if ( compiler.compile(writer) === false ) { return; }
+    if ( compiler.compile(parser, writer) === false ) { return; }
 
     await initWorker();
 

src/js/scriptlet-filtering.js

@@ -27,8 +27,8 @@ import logger from './logger.js';
 import µb from './background.js';
 import { redirectEngine } from './redirect-engine.js';
 import { sessionFirewall } from './filtering-engines.js';
-
 import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
+import * as sfp from './static-filtering-parser.js';
 
 import {
     domainFromHostname,
@@ -117,25 +117,28 @@ const contentscriptCode = (( ) => {
 // TODO: Probably should move this into StaticFilteringParser
 // https://github.com/uBlockOrigin/uBlock-issues/issues/1031
 //   Normalize scriptlet name to its canonical, unaliased name.
-const normalizeRawFilter = function(rawFilter) {
+const normalizeRawFilter = function(parser) {
-    const rawToken = rawFilter.slice(4, -1);
+    const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET);
-    const rawEnd = rawToken.length;
+    const walker = parser.getWalker(root);
-    let end = rawToken.indexOf(',');
+    const args = [];
-    if ( end === -1 ) { end = rawEnd; }
+    for ( let node = walker.next(); node !== 0; node = walker.next() ) {
-    const token = rawToken.slice(0, end).trim();
+        switch ( parser.getNodeType(node) ) {
-    const alias = token.endsWith('.js') ? token.slice(0, -3) : token;
+            case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN:
-    let normalized = redirectEngine.aliases.get(`${alias}.js`);
+            case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG:
-    normalized = normalized === undefined
+                args.push(parser.getNodeString(node));
-        ? alias
+                break;
-        : normalized.slice(0, -3);
+            default:
-    let beg = end + 1;
+                break;
-    while ( beg < rawEnd ) {
+        }
-        end = rawToken.indexOf(',', beg);
-        if ( end === -1 ) { end = rawEnd; }
-        normalized += ', ' + rawToken.slice(beg, end).trim();
-        beg = end + 1;
     }
-    return `+js(${normalized})`;
+    walker.dispose();
+    if ( args.length !== 0 ) {
+        const full = `${args[0]}.js`;
+        if ( redirectEngine.aliases.has(full) ) {
+            args[0] = redirectEngine.aliases.get(full).slice(0, -3);
+        }
+    }
+    return `+js(${args.join(', ')})`;
 };
 
 const lookupScriptlet = function(rawToken, reng, toInject) {
@@ -228,14 +231,14 @@ scriptletFilteringEngine.compile = function(parser, writer) {
     writer.select('SCRIPTLET_FILTERS');
 
     // Only exception filters are allowed to be global.
-    const { raw, exception } = parser.result;
+    const isException = parser.isException();
-    const normalized = normalizeRawFilter(raw);
+    const normalized = normalizeRawFilter(parser);
 
     // Tokenless is meaningful only for exception filters.
-    if ( normalized === '+js()' && exception === false ) { return; }
+    if ( normalized === '+js()' && isException === false ) { return; }
 
     if ( parser.hasOptions() === false ) {
-        if ( exception ) {
+        if ( isException ) {
             writer.push([ 32, '', 1, normalized ]);
         }
         return;
@@ -245,10 +248,10 @@ scriptletFilteringEngine.compile = function(parser, writer) {
     //   Ignore instances of exception filter with negated hostnames,
     //   because there is no way to create an exception to an exception.
 
-    for ( const { hn, not, bad } of parser.extOptions() ) {
+    for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
         if ( bad ) { continue; }
         let kind = 0;
-        if ( exception ) {
+        if ( isException ) {
             if ( not ) { continue; }
             kind |= 1;
         } else if ( not ) {

src/js/static-dnr-filtering.js

@@ -25,7 +25,7 @@
 
 import staticNetFilteringEngine from './static-net-filtering.js';
 import { LineIterator } from './text-utils.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
+import * as sfp from './static-filtering-parser.js';
 
 import {
     CompiledListReader,
@@ -87,19 +87,17 @@ const keyFromSelector = selector => {
 /******************************************************************************/
 
 function addExtendedToDNR(context, parser) {
-    if ( parser.category !== parser.CATStaticExtFilter ) { return false; }
+    if ( parser.isExtendedFilter() === false ) { return false; }
 
     // Scriptlet injection
-    if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
+    if ( parser.isScriptletFilter() ) {
-        if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) {
-            return;
-        }
         if ( parser.hasOptions() === false ) { return; }
         if ( context.scriptletFilters === undefined ) {
             context.scriptletFilters = new Map();
         }
-        const { raw, exception } = parser.result;
+        const exception = parser.isException();
-        for ( const { hn, not, bad } of parser.extOptions() ) {
+        const raw = parser.getTypeString(sfp.NODE_TYPE_EXT_PATTERN_RAW);
+        for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
             if ( bad ) { continue; }
             if ( exception ) { continue; }
             let details = context.scriptletFilters.get(raw);
@@ -166,7 +164,7 @@ function addExtendedToDNR(context, parser) {
     if ( context.specificCosmeticFilters === undefined ) {
         context.specificCosmeticFilters = new Map();
     }
-    for ( const { hn, not, bad } of parser.extOptions() ) {
+    for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
         if ( bad ) { continue; }
         let { compiled, exception, raw } = parser.result;
         if ( exception ) { continue; }
@@ -209,15 +207,13 @@ function addToDNR(context, list) {
     const env = context.env || [];
     const writer = new CompiledListWriter();
     const lineIter = new LineIterator(
-        StaticFilteringParser.utils.preparser.prune(list.text, env)
+        sfp.utils.preparser.prune(list.text, env)
     );
-    const parser = new StaticFilteringParser({
+    const parser = new sfp.AstFilterParser({
         nativeCssHas: env.includes('native_css_has'),
+        badTypes: [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE ],
     });
-    const compiler = staticNetFilteringEngine.createCompiler(parser);
+    const compiler = staticNetFilteringEngine.createCompiler();
-
-    // Can't enforce `redirect-rule=` with DNR
-    compiler.excludeOptions([ parser.OPTTokenRedirectRule ]);
 
     writer.properties.set('name', list.name);
     compiler.start(writer);
@@ -229,22 +225,18 @@ function addToDNR(context, list) {
             line = line.slice(0, -2).trim() + lineIter.next().trim();
         }
 
-        parser.analyze(line);
+        parser.parse(line);
 
-        if ( parser.shouldIgnore() ) { continue; }
+        if ( parser.isFilter() === false ) { continue; }
+        if ( parser.hasError() ) { continue; }
 
-        if ( parser.category !== parser.CATStaticNetFilter ) {
+        if ( parser.isExtendedFilter() ) {
             addExtendedToDNR(context, parser);
             continue;
         }
+        if ( parser.isNetworkFilter() === false ) { continue; }
 
-        // https://github.com/gorhill/uBlock/issues/2599
+        if ( compiler.compile(parser, writer) ) { continue; }
-        //   convert hostname to punycode if needed
-        if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
-            continue;
-        }
-
-        if ( compiler.compile(writer) ) { continue; }
 
         if ( compiler.error !== undefined ) {
             context.invalid.add(compiler.error);

src/js/static-ext-filtering.js

@@ -95,26 +95,25 @@ staticExtFilteringEngine.freeze = function() {
 };
 
 staticExtFilteringEngine.compile = function(parser, writer) {
-    if ( parser.category !== parser.CATStaticExtFilter ) { return false; }
+    if ( parser.isExtendedFilter() === false ) { return false; }
 
-    if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) {
+    if ( parser.hasError() ) {
-        const who = writer.properties.get('name') || '?';
         logger.writeOne({
             realm: 'message',
             type: 'error',
-            text: `Invalid extended filter in ${who}: ${parser.raw}`
+            text: `Invalid extended filter in ${writer.properties.get('name') || '?'}: ${parser.raw}`
         });
         return true;
     }
 
     // Scriptlet injection
-    if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
+    if ( parser.isScriptletFilter() ) {
         scriptletFilteringEngine.compile(parser, writer);
         return true;
     }
 
     // Response header filtering
-    if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
+    if ( parser.isResponseheaderFilter() ) {
         httpheaderFilteringEngine.compile(parser, writer);
         return true;
     }
@@ -122,13 +121,22 @@ staticExtFilteringEngine.compile = function(parser, writer) {
     // HTML filtering
     // TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML
     //       filtering syntax.
-    if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
+    if ( parser.isHtmlFilter() ) {
         htmlFilteringEngine.compile(parser, writer);
         return true;
     }
 
     // Cosmetic filtering
-    cosmeticFilteringEngine.compile(parser, writer);
+    if ( parser.isCosmeticFilter() ) {
+        cosmeticFilteringEngine.compile(parser, writer);
+        return true;
+    }
+
+    logger.writeOne({
+        realm: 'message',
+        type: 'error',
+        text: `Unknown extended filter in ${writer.properties.get('name') || '?'}: ${parser.raw}`
+    });
     return true;
 };
 

src/js/static-net-filtering.js

@@ -29,8 +29,8 @@ import { queueTask, dropTask } from './tasks.js';
 import BidiTrieContainer from './biditrie.js';
 import HNTrieContainer from './hntrie.js';
 import { sparseBase64 } from './base64-custom.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
 import { CompiledListReader } from './static-filtering-io.js';
+import * as sfp from './static-filtering-parser.js';
 
 import {
     domainFromHostname,
@@ -178,6 +178,24 @@ const typeValueToDNRTypeName = [
     'other',
 ];
 
+const MODIFIER_TYPE_REDIRECT = 1;
+const MODIFIER_TYPE_REDIRECTRULE = 2;
+const MODIFIER_TYPE_REMOVEPARAM = 3;
+const MODIFIER_TYPE_CSP = 4;
+
+const modifierTypeFromName = new Map([
+    [ 'redirect', MODIFIER_TYPE_REDIRECT ],
+    [ 'redirect-rule', MODIFIER_TYPE_REDIRECTRULE ],
+    [ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ],
+    [ 'csp', MODIFIER_TYPE_CSP ],
+]);
+
+const modifierNameFromType = new Map([
+    [ MODIFIER_TYPE_REDIRECT, 'redirect' ],
+    [ MODIFIER_TYPE_REDIRECTRULE, 'redirect-rule' ],
+    [ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ],
+    [ MODIFIER_TYPE_CSP, 'csp' ],
+]);
 
 //const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111;
 
@@ -1244,7 +1262,7 @@ class FilterRegex {
         if ( rule.condition === undefined ) {
             rule.condition = {};
         }
-        if ( StaticFilteringParser.utils.regex.isRE2(args[1]) === false ) {
+        if ( sfp.utils.regex.isRE2(args[1]) === false ) {
             dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`);
         }
         rule.condition.regexFilter = args[1];
@@ -2001,7 +2019,7 @@ class FilterModifier {
 
     static dnrFromCompiled(args, rule) {
         rule.__modifierAction = args[1];
-        rule.__modifierType = StaticFilteringParser.netOptionTokenNames.get(args[2]);
+        rule.__modifierType = modifierNameFromType.get(args[2]);
         rule.__modifierValue = args[3];
     }
 
@@ -2010,7 +2028,7 @@ class FilterModifier {
     }
 
     static logData(idata, details) {
-        let opt = StaticFilteringParser.netOptionTokenNames.get(filterData[idata+2]);
+        let opt = modifierNameFromType.get(filterData[idata+2]);
         const refs = filterRefs[filterData[idata+3]];
         if ( refs.value !== '' ) {
             opt += `=${refs.value}`;
@@ -2019,7 +2037,7 @@ class FilterModifier {
     }
 
     static dumpInfo(idata) {
-        const s = StaticFilteringParser.netOptionTokenNames.get(filterData[idata+2]);
+        const s = modifierNameFromType.get(filterData[idata+2]);
         const refs = filterRefs[filterData[idata+3]];
         if ( refs.value === '' ) { return s; }
         return `${s}=${refs.value}`;
@@ -2797,7 +2815,7 @@ class FilterOnHeaders {
     static match(idata) {
         const refs = filterRefs[filterData[idata+1]];
         if ( refs.$parsed === null ) {
-            refs.$parsed = StaticFilteringParser.parseHeaderValue(refs.headerOpt);
+            refs.$parsed = sfp.parseHeaderValue(refs.headerOpt);
         }
         const { bad, name, not, re, value } = refs.$parsed;
         if ( bad ) { return false; }
@@ -3017,39 +3035,42 @@ const urlTokenizer = new (class {
 /******************************************************************************/
 
 class FilterCompiler {
-    constructor(parser, other = undefined) {
+    constructor(other = undefined) {
-        this.parser = parser;
         if ( other !== undefined ) {
             return Object.assign(this, other);
         }
-        this.reBadCSP = /(?:=|;)\s*report-(?:to|uri)\b/;
         this.reToken = /[%0-9A-Za-z]+/g;
         this.fromDomainOptList = [];
         this.toDomainOptList = [];
         this.tokenIdToNormalizedType = new Map([
-            [ parser.OPTTokenCname, bitFromType('cname') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_CNAME, bitFromType('cname') ],
-            [ parser.OPTTokenCss, bitFromType('stylesheet') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_CSS, bitFromType('stylesheet') ],
-            [ parser.OPTTokenDoc, bitFromType('main_frame') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_DOC, bitFromType('main_frame') ],
-            [ parser.OPTTokenFont, bitFromType('font') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_FONT, bitFromType('font') ],
-            [ parser.OPTTokenFrame, bitFromType('sub_frame') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_FRAME, bitFromType('sub_frame') ],
-            [ parser.OPTTokenGenericblock, bitFromType('unsupported') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK, bitFromType('unsupported') ],
-            [ parser.OPTTokenGhide, bitFromType('generichide') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE, bitFromType('generichide') ],
-            [ parser.OPTTokenImage, bitFromType('image') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE, bitFromType('image') ],
-            [ parser.OPTTokenInlineFont, bitFromType('inline-font') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT, bitFromType('inline-font') ],
-            [ parser.OPTTokenInlineScript, bitFromType('inline-script') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT, bitFromType('inline-script') ],
-            [ parser.OPTTokenMedia, bitFromType('media') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA, bitFromType('media') ],
-            [ parser.OPTTokenObject, bitFromType('object') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT, bitFromType('object') ],
-            [ parser.OPTTokenOther, bitFromType('other') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_OTHER, bitFromType('other') ],
-            [ parser.OPTTokenPing, bitFromType('ping') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_PING, bitFromType('ping') ],
-            [ parser.OPTTokenPopunder, bitFromType('popunder') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER, bitFromType('popunder') ],
-            [ parser.OPTTokenPopup, bitFromType('popup') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_POPUP, bitFromType('popup') ],
-            [ parser.OPTTokenScript, bitFromType('script') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT, bitFromType('script') ],
-            [ parser.OPTTokenShide, bitFromType('specifichide') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE, bitFromType('specifichide') ],
-            [ parser.OPTTokenXhr, bitFromType('xmlhttprequest') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_XHR, bitFromType('xmlhttprequest') ],
-            [ parser.OPTTokenWebrtc, bitFromType('unsupported') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC, bitFromType('unsupported') ],
-            [ parser.OPTTokenWebsocket, bitFromType('websocket') ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET, bitFromType('websocket') ],
+        ]);
+        this.modifierIdToNormalizedId = new Map([
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_CSP, MODIFIER_TYPE_CSP ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT, MODIFIER_TYPE_REDIRECT ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE, MODIFIER_TYPE_REDIRECTRULE ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ],
         ]);
-        this.excludedOptionSet = new Set();
         // These top 100 "bad tokens" are collated using the "miss" histogram
         // from tokenHistograms(). The "score" is their occurrence among the
         // 200K+ URLs used in the benchmark and executed against default
@@ -3182,6 +3203,7 @@ class FilterCompiler {
         this.denyallowOpt = '';
         this.headerOpt = undefined;
         this.isPureHostname = false;
+        this.isGeneric = false;
         this.isRegex = false;
         this.strictParty = 0;
         this.token = '*';
@@ -3203,7 +3225,7 @@ class FilterCompiler {
     }
 
     clone() {
-        return new FilterCompiler(this.parser, this);
+        return new FilterCompiler(this);
     }
 
     normalizeRegexSource(s) {
@@ -3215,12 +3237,6 @@ class FilterCompiler {
         return '';
     }
 
-    excludeOptions(options) {
-        for ( const option of options ) {
-            this.excludedOptionSet.add(option);
-        }
-    }
-
     processMethodOption(value) {
         for ( const method of value.split('|') ) {
             if ( method.charCodeAt(0) === 0x7E /* '~' */ ) {
@@ -3264,21 +3280,12 @@ class FilterCompiler {
         this.party |= firstParty ? FirstParty : ThirdParty;
     }
 
-    processHostnameList(s, modeBits, out = []) {
+    processHostnameList(iter, out = []) {
-        let beg = 0;
-        let slen = s.length;
         let i = 0;
-        while ( beg < slen ) {
+        for ( const { hn, not, bad } of iter ) {
-            let end = s.indexOf('|', beg);
+            if ( bad ) { return ''; }
-            if ( end === -1 ) { end = slen; }
+            out[i] = not ? `~${hn}` : hn;
-            const hn = this.parser.normalizeHostnameValue(
+            i += 1;
-                s.slice(beg, end),
-                modeBits
-            );
-            if ( hn !== undefined ) {
-                out[i] = hn; i += 1;
-            }
-            beg = end + 1;
         }
         out.length = i;
         return i === 1 ? out[0] : out.join('|');
@@ -3286,146 +3293,206 @@ class FilterCompiler {
 
     processModifierOption(modifier, value) {
         if ( this.modifyType !== undefined ) { return false; }
-        this.modifyType = modifier;
+        const normalized = this.modifierIdToNormalizedId.get(modifier);
+        if ( normalized === undefined ) { return false; }
+        this.modifyType = normalized;
         this.modifyValue = value || '';
         return true;
     }
 
-    processOptions() {
+    processCspOption(value) {
-        const { parser } = this;
+        this.modifyType = MODIFIER_TYPE_CSP;
-        for ( let { id, val, not } of parser.netOptions() ) {
+        this.modifyValue = value || '';
-            switch ( id ) {
+        this.optionUnitBits |= this.CSP_BIT;
-            case parser.OPTToken1p:
+        return true;
-                this.processPartyOption(true, not);
+    }
+
+    processOptionWithValue(parser, id) {
+        switch ( id ) {
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CSP:
+                if ( this.processCspOption(parser.getNetOptionValue(id)) === false ) { return false; }
                 break;
-            case parser.OPTToken1pStrict:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW:
-                this.strictParty = this.strictParty === -1 ? 0 : 1;
+                this.denyallowOpt = this.processHostnameList(
-                this.optionUnitBits |= this.STRICT_PARTY_BIT;
+                    parser.getNetFilterDenyallowOptionIterator(),
+                );
+                if ( this.denyallowOpt === '' ) { return false; }
+                this.optionUnitBits |= this.DENYALLOW_BIT;
                 break;
-            case parser.OPTToken3p:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FROM:
-                this.processPartyOption(false, not);
-                break;
-            case parser.OPTToken3pStrict:
-                this.strictParty = this.strictParty === 1 ? 0 : -1;
-                this.optionUnitBits |= this.STRICT_PARTY_BIT;
-                break;
-            case parser.OPTTokenAll:
-                this.processTypeOption(-1);
-                break;
-            // https://github.com/uBlockOrigin/uAssets/issues/192
-            case parser.OPTTokenBadfilter:
-                this.badFilter = true;
-                break;
-            case parser.OPTTokenCsp:
-                if ( this.processModifierOption(id, val) === false ) {
-                    return false;
-                }
-                if ( val !== undefined && this.reBadCSP.test(val) ) {
-                    return false;
-                }
-                this.optionUnitBits |= this.CSP_BIT;
-                break;
-            // https://github.com/gorhill/uBlock/issues/2294
-            //   Detect and discard filter if domain option contains
-            //   nonsensical characters.
-            case parser.OPTTokenFrom:
                 this.fromDomainOpt = this.processHostnameList(
-                    val,
+                    parser.getNetFilterFromOptionIterator(),
-                    0b1010,
                     this.fromDomainOptList
                 );
                 if ( this.fromDomainOpt === '' ) { return false; }
                 this.optionUnitBits |= this.FROM_BIT;
                 break;
-            case parser.OPTTokenTo:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER: {
+                this.headerOpt = parser.getNetOptionValue(id) || '';
+                this.optionUnitBits |= this.HEADER_BIT;
+                break;
+            }
+            case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD:
+                this.processMethodOption(parser.getNetOptionValue(id));
+                this.optionUnitBits |= this.METHOD_BIT;
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
+                if ( this.action === AllowAction ) {
+                    id = sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE;
+                }
+                if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
+                    return false;
+                }
+                this.optionUnitBits |= this.REDIRECT_BIT;
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
+                if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
+                    return false;
+                }
+                this.optionUnitBits |= this.REDIRECT_BIT;
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
+                if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
+                    return false;
+                }
+                this.optionUnitBits |= this.REMOVEPARAM_BIT;
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
                 this.toDomainOpt = this.processHostnameList(
-                    val,
+                    parser.getNetFilterToOptionIterator(),
-                    0b1010,
                     this.toDomainOptList
                 );
                 if ( this.toDomainOpt === '' ) { return false; }
                 this.optionUnitBits |= this.TO_BIT;
                 break;
-            case parser.OPTTokenDenyAllow:
+            default:
-                this.denyallowOpt = this.processHostnameList(val, 0b0000);
-                if ( this.denyallowOpt === '' ) { return false; }
-                this.optionUnitBits |= this.DENYALLOW_BIT;
                 break;
-            // https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/
+        }
-            //   Add support for `elemhide`. Rarely used but it happens.
+        return true;
-            case parser.OPTTokenEhide:
+    }
-                this.processTypeOption(parser.OPTTokenShide, not);
+
-                this.processTypeOption(parser.OPTTokenGhide, not);
+    process(parser) {
+        // important!
+        this.reset();
+
+        if ( parser.hasError() ) {
+            return this.FILTER_INVALID;
+        }
+
+        if ( parser.isException() ) {
+            this.action = AllowAction;
+        }
+
+        if ( parser.isLeftHnAnchored() ) {
+            this.anchor |= 0b100;
+        } else if ( parser.isLeftAnchored() ) {
+            this.anchor |= 0b010;
+        }
+        if ( parser.isRightAnchored() ) {
+            this.anchor |= 0b001;
+        }
+
+        this.pattern = parser.getNetPattern();
+        if ( parser.isHostnamePattern() ) {
+            this.isPureHostname = true;
+        } else if ( parser.isGenericPattern() ) {
+            this.isGeneric = true;
+        } else if ( parser.isRegexPattern() ) {
+            this.isRegex = true;
+        }
+
+        for ( const type of parser.getNodeTypes() ) {
+            switch ( type ) {
+            case sfp.NODE_TYPE_NET_OPTION_NAME_1P:
+                this.processPartyOption(true, parser.isNegatedOption(type));
                 break;
-            case parser.OPTTokenHeader:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P:
-                this.headerOpt = val !== undefined ? val : '';
+                this.strictParty = this.strictParty === -1 ? 0 : 1;
-                this.optionUnitBits |= this.HEADER_BIT;
+                this.optionUnitBits |= this.STRICT_PARTY_BIT;
                 break;
-            case parser.OPTTokenImportant:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_3P:
-                if ( this.action === AllowAction ) { return false; }
+                this.processPartyOption(false, parser.isNegatedOption(type));
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P:
+                this.strictParty = this.strictParty === 1 ? 0 : -1;
+                this.optionUnitBits |= this.STRICT_PARTY_BIT;
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_ALL:
+                this.processTypeOption(-1);
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER:
+                this.badFilter = true;
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CSS:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_DOC:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FONT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_PING:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_XHR:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET:
+                this.processTypeOption(type, parser.isNegatedOption(type));
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_CSP:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_FROM:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
+                if ( this.processOptionWithValue(parser, type) === false ) {
+                    return this.FILTER_INVALID;
+                }
+                break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE: {
+                const not = parser.isNegatedOption(type);
+                this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE, not);
+                this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE, not);
+                break;
+            }
+            case sfp.NODE_TYPE_NET_OPTION_NAME_EMPTY: {
+                const id = this.action === AllowAction
+                    ? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE
+                    : sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT;
+                if ( this.processModifierOption(id, 'empty') === false ) {
+                    return this.FILTER_INVALID;
+                }
+                this.optionUnitBits |= this.REDIRECT_BIT;
+                break;
+            }
+            case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT:
                 this.optionUnitBits |= this.IMPORTANT_BIT;
                 this.action = BlockImportant;
                 break;
-            // Used by Adguard:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE:
-            // https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#empty-modifier
-            case parser.OPTTokenEmpty:
-                id = this.action === AllowAction
-                    ? parser.OPTTokenRedirectRule
-                    : parser.OPTTokenRedirect;
-                if ( this.processModifierOption(id, 'empty') === false ) {
-                    return false;
-                }
-                this.optionUnitBits |= this.REDIRECT_BIT;
-                break;
-            case parser.OPTTokenMatchCase:
                 this.patternMatchCase = true;
                 break;
-            case parser.OPTTokenMp4:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_MP4: {
-                id = this.action === AllowAction
+                const id = this.action === AllowAction
-                    ? parser.OPTTokenRedirectRule
+                    ? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE
-                    : parser.OPTTokenRedirect;
+                    : sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT;
                 if ( this.processModifierOption(id, 'noopmp4-1s') === false ) {
-                    return false;
+                    return this.FILTER_INVALID;
                 }
                 this.optionUnitBits |= this.REDIRECT_BIT;
                 break;
-            case parser.OPTTokenNoop:
+            }
-                break;
-            case parser.OPTTokenRemoveparam:
-                if ( this.processModifierOption(id, val) === false ) {
-                    return false;
-                }
-                this.optionUnitBits |= this.REMOVEPARAM_BIT;
-                break;
-            case parser.OPTTokenRedirect:
-                if ( this.action === AllowAction ) {
-                    id = parser.OPTTokenRedirectRule;
-                }
-                if ( this.processModifierOption(id, val) === false ) {
-                    return false;
-                }
-                this.optionUnitBits |= this.REDIRECT_BIT;
-                break;
-            case parser.OPTTokenRedirectRule:
-                if ( this.excludedOptionSet.has(parser.OPTTokenRedirectRule) ) {
-                    return false;
-                }
-                if ( this.processModifierOption(id, val) === false ) {
-                    return false;
-                }
-                this.optionUnitBits |= this.REDIRECT_BIT;
-                break;
-            case parser.OPTTokenMethod:
-                this.processMethodOption(val);
-                this.optionUnitBits |= this.METHOD_BIT;
-                break;
-            case parser.OPTTokenInvalid:
-                return false;
             default:
-                if ( this.tokenIdToNormalizedType.has(id) === false ) {
-                    return false;
-                }
-                this.processTypeOption(id, not);
                 break;
             }
         }
@@ -3452,10 +3519,10 @@ class FilterCompiler {
         }
 
         // CSP directives implicitly apply only to document/subdocument.
-        if ( this.modifyType === this.parser.OPTTokenCsp ) {
+        if ( this.modifyType === MODIFIER_TYPE_CSP ) {
             if ( this.typeBits === 0 ) {
-                this.processTypeOption(this.parser.OPTTokenDoc, false);
+                this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_DOC, false);
-                this.processTypeOption(this.parser.OPTTokenFrame, false);
+                this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_FRAME, false);
             }
         }
 
@@ -3464,85 +3531,29 @@ class FilterCompiler {
         //   toggle off `unsupported` bit.
         if ( this.typeBits & unsupportedTypeBit ) {
             this.typeBits &= ~unsupportedTypeBit;
-            if ( this.typeBits === 0 ) { return false; }
+            if ( this.typeBits === 0 ) { return this.FILTER_UNSUPPORTED; }
         }
 
-        return true;
-    }
-
-    process() {
-        // important!
-        this.reset();
-
-        if ( this.parser.hasError() ) {
-            return this.FILTER_INVALID;
-        }
-
-        // Filters which pattern is a single character other than `*` and have
-        // no narrowing options are discarded as invalid.
-        if ( this.parser.patternIsDubious() ) {
-            return this.FILTER_INVALID;
-        }
-
-        // block or allow filter?
-        // Important: this must be executed before parsing options
-        if ( this.parser.isException() ) {
-            this.action = AllowAction;
-        }
-
-        this.isPureHostname = this.parser.patternIsPlainHostname();
-
         // Plain hostname? (from HOSTS file)
-        if ( this.isPureHostname && this.parser.hasOptions() === false ) {
+        if ( this.isPureHostname && parser.hasOptions() === false ) {
-            this.pattern = this.parser.patternToLowercase();
             this.anchor |= 0b100;
             return this.FILTER_OK;
         }
 
-        // options
-        if ( this.parser.hasOptions() && this.processOptions() === false ) {
-            return this.FILTER_UNSUPPORTED;
-        }
-
         // regex?
-        if ( this.parser.patternIsRegex() ) {
+        if ( this.isRegex ) {
-            this.isRegex = true;
-            // https://github.com/gorhill/uBlock/issues/1246
-            //   If the filter is valid, use the corrected version of the
-            //   source string -- this ensure reverse-lookup will work fine.
-            this.pattern = this.normalizeRegexSource(this.parser.getNetPattern());
-            if ( this.pattern === '' ) {
-                return this.FILTER_UNSUPPORTED;
-            }
             return this.FILTER_OK;
         }
 
-        const pattern = this.parser.patternIsMatchAll()
+        if ( this.isGeneric ) {
-            ? '*'
+            this.wildcardPos = this.pattern.indexOf('*');
-            : this.parser.patternToLowercase();
+            this.caretPos = this.pattern.indexOf('^');
-
-        if ( this.parser.patternIsLeftHostnameAnchored() ) {
-            this.anchor |= 0b100;
-        } else if ( this.parser.patternIsLeftAnchored() ) {
-            this.anchor |= 0b010;
-        }
-        if ( this.parser.patternIsRightAnchored() ) {
-            this.anchor |= 0b001;
         }
 
-        if ( this.parser.patternHasWildcard() ) {
+        if ( this.pattern.length > 1024 ) {
-            this.wildcardPos = pattern.indexOf('*');
-        }
-
-        if ( this.parser.patternHasCaret() ) {
-            this.caretPos = pattern.indexOf('^');
-        }
-
-        if ( pattern.length > 1024 ) {
             return this.FILTER_UNSUPPORTED;
         }
 
-        this.pattern = pattern;
         return this.FILTER_OK;
     }
 
@@ -3556,9 +3567,7 @@ class FilterCompiler {
 
     makeToken() {
         if ( this.pattern === '*' ) {
-            if ( this.modifyType !== this.parser.OPTTokenRemoveparam ) {
+            if ( this.modifyType !== MODIFIER_TYPE_REMOVEPARAM ) { return; }
-                return;
-            }
             return this.extractTokenFromQuerypruneValue();
         }
         if ( this.isRegex ) {
@@ -3607,7 +3616,7 @@ class FilterCompiler {
     //   Mind `\b` directives: `/\bads\b/` should result in token being `ads`,
     //   not `bads`.
     extractTokenFromRegex(pattern) {
-        pattern = StaticFilteringParser.utils.regex.toTokenizableStr(pattern);
+        pattern = sfp.utils.regex.toTokenizableStr(pattern);
         this.reToken.lastIndex = 0;
         let bestToken;
         let bestBadness = 0x7FFFFFFF;
@@ -3682,8 +3691,8 @@ class FilterCompiler {
                s.charCodeAt(l-2) === 0x2E /* '.' */;
     }
 
-    compile(writer) {
+    compile(parser, writer) {
-        const r = this.process();
+        const r = this.process(parser);
 
         // Ignore non-static network filters
         if ( r === this.FILTER_INVALID ) { return false; }
@@ -3691,7 +3700,7 @@ class FilterCompiler {
         // Ignore filters with unsupported options
         if ( r === this.FILTER_UNSUPPORTED ) {
             const who = writer.properties.get('name') || '?';
-            this.error = `Invalid network filter in ${who}: ${this.parser.raw}`;
+            this.error = `Invalid network filter in ${who}: ${parser.raw}`;
             return false;
         }
 
@@ -3704,8 +3713,8 @@ class FilterCompiler {
         // Reminder:
         //   `redirect=` is a combination of a `redirect-rule` filter and a
         //   block filter.
-        if ( this.modifyType === this.parser.OPTTokenRedirect ) {
+        if ( this.modifyType === MODIFIER_TYPE_REDIRECT ) {
-            this.modifyType = this.parser.OPTTokenRedirectRule;
+            this.modifyType = MODIFIER_TYPE_REDIRECTRULE;
             const parsedBlock = this.clone();
             parsedBlock.modifyType = undefined;
             parsedBlock.optionUnitBits &= ~this.REDIRECT_BIT;
@@ -3943,11 +3952,6 @@ FilterContainer.prototype.prime = function() {
         keyvalStore.getItem('SNFE.destHNTrieContainer.trieDetails')
     );
     bidiTriePrime();
-    // Remove entries with obsolete name.
-    // TODO: Remove before publishing 1.41.0
-    keyvalStore.removeItem('SNFE.filterOrigin.trieDetails');
-    keyvalStore.removeItem('SNFE.FilterHostnameDict.trieDetails');
-    keyvalStore.removeItem('SNFE.filterDocOrigin.trieDetails');
 };
 
 /******************************************************************************/
@@ -4741,8 +4745,8 @@ FilterContainer.prototype.unserialize = async function(s) {
 
 /******************************************************************************/
 
-FilterContainer.prototype.createCompiler = function(parser) {
+FilterContainer.prototype.createCompiler = function() {
-    return new FilterCompiler(parser);
+    return new FilterCompiler();
 };
 
 /******************************************************************************/
@@ -4768,7 +4772,7 @@ FilterContainer.prototype.fromCompiled = function(reader) {
 
 FilterContainer.prototype.matchAndFetchModifiers = function(
     fctxt,
-    modifierType
+    modifierName
 ) {
     const typeBits = typeNameToTypeValue[fctxt.type] || otherTypeBitValue;
 
@@ -4811,7 +4815,7 @@ FilterContainer.prototype.matchAndFetchModifiers = function(
 
     const results = [];
     const env = {
-        type: StaticFilteringParser.netOptionTokenIds.get(modifierType) || 0,
+        type: modifierTypeFromName.get(modifierName) || 0,
         bits: 0,
         th: 0,
         iunit: 0,
@@ -5223,7 +5227,7 @@ FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) {
 function parseRedirectRequestValue(directive) {
     if ( directive.cache === null ) {
         directive.cache =
-            StaticFilteringParser.parseRedirectValue(directive.value);
+            sfp.parseRedirectValue(directive.value);
     }
     return directive.cache;
 }
@@ -5336,7 +5340,7 @@ FilterContainer.prototype.filterQuery = function(fctxt) {
 function parseQueryPruneValue(directive) {
     if ( directive.cache === null ) {
         directive.cache =
-            StaticFilteringParser.parseQueryPruneValue(directive.value);
+            sfp.parseQueryPruneValue(directive.value);
     }
     return directive.cache;
 }

src/js/storage.js

@@ -40,8 +40,8 @@ import { hostnameFromURI } from './uri-utils.js';
 import { i18n, i18n$ } from './i18n.js';
 import { redirectEngine } from './redirect-engine.js';
 import { sparseBase64 } from './base64-custom.js';
-import { StaticFilteringParser } from './static-filtering-parser.js';
 import { ubolog, ubologSet } from './console.js';
+import * as sfp from './static-filtering-parser.js';
 
 import {
     permanentFirewall,
@@ -1007,20 +1007,16 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
     const expertMode =
         details.assetKey !== this.userFiltersPath ||
         this.hiddenSettings.filterAuthorMode !== false;
-    // Useful references:
+    const parser = new sfp.AstFilterParser({
-    //    https://adblockplus.org/en/filter-cheatsheet
-    //    https://adblockplus.org/en/filters
-    const parser = new StaticFilteringParser({
         expertMode,
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
+        maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
     });
     const compiler = staticNetFilteringEngine.createCompiler(parser);
     const lineIter = new LineIterator(
-        parser.utils.preparser.prune(rawText, vAPI.webextFlavor.env)
+        sfp.utils.preparser.prune(rawText, vAPI.webextFlavor.env)
     );
 
-    parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH);
-
     compiler.start(writer);
 
     while ( lineIter.eot() === false ) {
@@ -1031,23 +1027,19 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
             line = line.slice(0, -2).trim() + lineIter.next().trim();
         }
 
-        parser.analyze(line);
+        parser.parse(line);
 
-        if ( parser.shouldIgnore() ) { continue; }
+        if ( parser.isFilter() === false ) { continue; }
+        if ( parser.hasError() ) { continue; }
 
-        if ( parser.category === parser.CATStaticExtFilter ) {
+        if ( parser.isExtendedFilter() ) {
             staticExtFilteringEngine.compile(parser, writer);
             continue;
         }
 
-        if ( parser.category !== parser.CATStaticNetFilter ) { continue; }
+        if ( parser.isNetworkFilter() === false ) { continue; }
 
-        // https://github.com/gorhill/uBlock/issues/2599
+        if ( compiler.compile(parser, writer) ) { continue; }
-        //   convert hostname to punycode if needed
-        if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
-            continue;
-        }
-        if ( compiler.compile(writer) ) { continue; }
         if ( compiler.error !== undefined ) {
             logger.writeOne({
                 realm: 'message',