Add CoreMirror's code-folding ability to list editor/viewer

Related issue:
- https://github.com/uBlockOrigin/uBlock-issues/issues/1134

CodeMirror's code folding reference:
- https://codemirror.net/doc/manual.html#addon_foldcode

This commit adds support for code-folding to the filter
list editor/viewer.

The following blocks of code are foldable by clicking the
corresponding marker in the gutter:

- !#if/#endif blocks
- !#include blocks

Addtionally, the following changes:

- The `!#include` line is now preserved when importing a
  sublist
- The `!#if` directives will be syntax-colored according
  to whether they evaluate to true or false on the current
  platform
- Double-clicking on a foldable line in the gutter will
  select the content of the foldable block
- Minor visual improvement to matching brackets

src/1p-filters.html

@@ -6,6 +6,7 @@
 <title>uBlock — Your filters</title>
 
 <link rel="stylesheet" href="lib/codemirror/lib/codemirror.css">
+<link rel="stylesheet" href="lib/codemirror/addon/fold/foldgutter.css">
 <link rel="stylesheet" href="lib/codemirror/addon/hint/show-hint.css">
 <link rel="stylesheet" href="lib/codemirror/addon/search/matchesonscrollbar.css">
 
@@ -42,6 +43,8 @@
 <script src="lib/codemirror/addon/display/panel.js"></script>
 <script src="lib/codemirror/addon/edit/closebrackets.js"></script>
 <script src="lib/codemirror/addon/edit/matchbrackets.js"></script>
+<script src="lib/codemirror/addon/fold/foldcode.js"></script>
+<script src="lib/codemirror/addon/fold/foldgutter.js"></script>
 <script src="lib/codemirror/addon/hint/show-hint.js"></script>
 <script src="lib/codemirror/addon/scroll/annotatescrollbar.js"></script>
 <script src="lib/codemirror/addon/search/matchesonscrollbar.js"></script>

src/asset-viewer.html

@@ -5,6 +5,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title data-i18n="assetViewerPageName"></title>
 <link rel="stylesheet" href="lib/codemirror/lib/codemirror.css">
+<link rel="stylesheet" href="lib/codemirror/addon/fold/foldgutter.css">
 <link rel="stylesheet" href="lib/codemirror/addon/search/matchesonscrollbar.css">
 <link rel="stylesheet" type="text/css" href="css/themes/default.css">
 <link rel="stylesheet" href="css/common.css">
@@ -31,6 +32,8 @@ body {
 <script src="lib/codemirror/lib/codemirror.js"></script>
 <script src="lib/codemirror/addon/display/panel.js"></script>
 <script src="lib/codemirror/addon/edit/matchbrackets.js"></script>
+<script src="lib/codemirror/addon/fold/foldcode.js"></script>
+<script src="lib/codemirror/addon/fold/foldgutter.js"></script>
 <script src="lib/codemirror/addon/scroll/annotatescrollbar.js"></script>
 <script src="lib/codemirror/addon/search/matchesonscrollbar.js"></script>
 <script src="lib/codemirror/addon/search/searchcursor.js"></script>

src/css/codemirror.css

@@ -52,6 +52,7 @@ div.CodeMirror span.CodeMirror-matchingbracket {
     color: unset;
     }
 .CodeMirror-matchingbracket {
+    background-color: #afa;
     color: inherit !important;
     font-weight: bold;
     }

src/js/1p-filters.js

@@ -36,6 +36,8 @@ const cmEditor = new CodeMirror(document.getElementById('userFilters'), {
         'Ctrl-Space': 'autocomplete',
         'Tab': 'toggleComment',
     },
+    foldGutter: true,
+    gutters: [ 'CodeMirror-linenumbers', 'CodeMirror-foldgutter' ],
     lineNumbers: true,
     lineWrapping: true,
     matchBrackets: true,

src/js/asset-viewer.js

@@ -32,6 +32,8 @@
 
     const cmEditor = new CodeMirror(document.getElementById('content'), {
         autofocus: true,
+        foldGutter: true,
+        gutters: [ 'CodeMirror-linenumbers', 'CodeMirror-foldgutter' ],
         lineNumbers: true,
         lineWrapping: true,
         matchBrackets: true,

src/js/assets.js

@@ -276,7 +276,7 @@ api.fetchFilterList = async function(mainlistURL) {
                     if ( sublistURLs.has(subURL) ) { continue; }
                     sublistURLs.add(subURL);
                     out.push(
-                        slice.slice(lastIndex, match.index),
+                        slice.slice(lastIndex, match.index + match[0].length),
                         `! >>>>>>>> ${subURL}`,
                         api.fetchText(subURL),
                         `! <<<<<<<< ${subURL}`

src/js/codemirror/ubo-static-filtering.js

@@ -32,7 +32,7 @@
 
 const redirectNames = new Map();
 const scriptletNames = new Map();
-const preparseDirectiveTokens = new Set();
+const preparseDirectiveTokens = new Map();
 const preparseDirectiveHints = [];
 
 /******************************************************************************/
@@ -44,8 +44,8 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
     if ( StaticFilteringParser instanceof Object === false ) { return; }
     const parser = new StaticFilteringParser({ interactive: true });
 
-    const rePreparseDirectives = /^!#(?:if|endif|include)\b/;
-    const rePreparseIfDirective = /^(!#if !?)(.+)$/;
+    const rePreparseDirectives = /^!#(?:if|endif|include )\b/;
+    const rePreparseIfDirective = /^(!#if ?)(.*)$/;
     let parserSlot = 0;
     let netOptionValueMode = false;
 
@@ -60,17 +60,28 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
             return 'variable strong';
         }
         if ( stream.pos < match[1].length ) {
-            stream.pos = match[1].length;
+            stream.pos += match[1].length;
             return 'variable strong';
         }
         stream.skipToEnd();
-        if (
-            preparseDirectiveTokens.size === 0 ||
-            preparseDirectiveTokens.has(match[2].trim())
-        ) {
-            return 'variable strong';
+        if ( match[1].endsWith(' ') === false ) {
+            return 'error strong';
         }
-        return 'error strong';
+        if ( preparseDirectiveTokens.size === 0 ) {
+            return 'positive strong';
+        }
+        let token = match[2];
+        const not = token.startsWith('!');
+        if ( not ) {
+            token = token.slice(1);
+        }
+        if ( preparseDirectiveTokens.has(token) === false ) {
+            return 'error strong';
+        }
+        if ( not !== preparseDirectiveTokens.get(token) ) {
+            return 'positive strong';
+        }
+        return 'negative strong';
     };
 
     const colorExtHTMLPatternSpan = function(stream) {
@@ -289,8 +300,8 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
                     scriptletNames.set(name.slice(0, -3), displayText);
                 }
             }
-            details.preparseDirectiveTokens.forEach(a => {
-                preparseDirectiveTokens.add(a);
+            details.preparseDirectiveTokens.forEach(([ a, b ]) => {
+                preparseDirectiveTokens.set(a, b);
             });
             preparseDirectiveHints.push(...details.preparseDirectiveHints);
             initHints();
@@ -471,6 +482,63 @@ const initHints = function() {
 
 /******************************************************************************/
 
+CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => {
+
+    const foldIfEndif = function(startLineNo, startLine, cm) {
+        const lastLineNo = cm.lastLine();
+        let endLineNo = startLineNo;
+        let depth = 1;
+        while ( endLineNo < lastLineNo ) {
+            endLineNo += 1;
+            const line = cm.getLine(endLineNo);
+            if ( line.startsWith('!#endif') ) {
+                depth -= 1;
+                if ( depth === 0 ) {
+                    return {
+                        from: CodeMirror.Pos(startLineNo, startLine.length),
+                        to: CodeMirror.Pos(endLineNo, 0)
+                    };
+                }
+            }
+            if ( line.startsWith('!#if') ) {
+                depth += 1;
+            }
+        }
+    };
+
+    const foldInclude = function(startLineNo, startLine, cm) {
+        const lastLineNo = cm.lastLine();
+        let endLineNo = startLineNo + 1;
+        if ( endLineNo >= lastLineNo ) { return; }
+        if ( cm.getLine(endLineNo).startsWith('! >>>>>>>> ') === false ) {
+            return;
+        }
+        while ( endLineNo < lastLineNo ) {
+            endLineNo += 1;
+            const line = cm.getLine(endLineNo);
+            if ( line.startsWith('! <<<<<<<< ') ) {
+                return {
+                    from: CodeMirror.Pos(startLineNo, startLine.length),
+                    to: CodeMirror.Pos(endLineNo, line.length)
+                };
+            }
+        }
+    };
+
+    return function(cm, start) {
+        const startLineNo = start.line;
+        const startLine = cm.getLine(startLineNo);
+        if ( startLine.startsWith('!#if') ) {
+            return foldIfEndif(startLineNo, startLine, cm);
+        }
+        if ( startLine.startsWith('!#include ') ) {
+            return foldInclude(startLineNo, startLine, cm);
+        }
+    };
+})());
+
+/******************************************************************************/
+
 // <<<<< end of local scope
 }
 

src/js/dashboard-common.js

@@ -149,24 +149,37 @@ self.uBlockDashboard.patchCodeMirrorEditor = (function() {
     let lastGutterClick = 0;
     let lastGutterLine = 0;
 
-    const onGutterClicked = function(cm, line) {
+    const onGutterClicked = function(cm, line, gutter) {
+        if ( gutter !== 'CodeMirror-linenumbers' ) { return; }
+        grabFocusAsync(cm);
         const delta = Date.now() - lastGutterClick;
+        // Single click
         if ( delta >= 500 || line !== lastGutterLine ) {
             cm.setSelection(
-                { line: line, ch: 0 },
+                { line, ch: 0 },
                 { line: line + 1, ch: 0 }
             );
             lastGutterClick = Date.now();
             lastGutterLine = line;
-        } else {
-            cm.setSelection(
-                { line: 0, ch: 0 },
-                { line: cm.lineCount(), ch: 0 },
-                { scroll: false }
-            );
-            lastGutterClick = 0;
+            return;
         }
-        grabFocusAsync(cm);
+        // Double click: select fold-able block or all
+        let lineFrom = 0;
+        let lineTo = cm.lineCount();
+        const foldFn = cm.getHelper({ line, ch: 0 }, 'fold');
+        if ( foldFn instanceof Function ) {
+            const range = foldFn(cm, { line, ch: 0 });
+            if ( range !== undefined ) {
+                lineFrom = range.from.line;
+                lineTo = range.to.line + 1;
+            }
+        }
+        cm.setSelection(
+            { line: lineFrom, ch: 0 },
+            { line: lineTo, ch: 0 },
+            { scroll: false }
+        );
+        lastGutterClick = 0;
     };
 
     return function(cm) {

src/js/storage.js

@@ -862,6 +862,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
     // the content string which should alternatively be parsed and discarded.
     split: function(content) {
         const reIf = /^!#(if|endif)\b([^\n]*)(?:[\n\r]+|$)/gm;
+        const soup = vAPI.webextFlavor.soup;
         const stack = [];
         const shouldDiscard = ( ) => stack.some(v => v);
         const parts = [ 0 ];
@@ -878,10 +879,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
                 if ( target ) { expr = expr.slice(1); }
                 const token = this.tokens.get(expr);
                 const startDiscard =
-                    token === 'false' &&
-                        target === false ||
-                    token !== undefined &&
-                        vAPI.webextFlavor.soup.has(token) === target;
+                    token === 'false' && target === false ||
+                    token !== undefined && soup.has(token) === target;
                 if ( discard === false && startDiscard ) {
                     parts.push(match.index);
                     discard = true;
@@ -930,7 +929,12 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
     },
 
     getTokens: function() {
-        return Array.from(this.tokens.keys());
+        const out = new Map();
+        const soup = vAPI.webextFlavor.soup;
+        for ( const [ key, val ] of this.tokens ) {
+            out.set(key, val !== 'false' && soup.has(val));
+        }
+        return Array.from(out);
     },
 
     tokens: new Map([
@@ -947,6 +951,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
         // Compatibility with other blockers
         // https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#adguard-specific
         [ 'adguard', 'adguard' ],
+        [ 'adguard_app_windows', 'false' ],
         [ 'adguard_ext_chromium', 'chromium' ],
         [ 'adguard_ext_edge', 'edge' ],
         [ 'adguard_ext_firefox', 'firefox' ],

src/lib/codemirror/addon/fold/foldcode.js

@@ -0,0 +1,152 @@
+// CodeMirror, copyright (c) by Marijn Haverbeke and others
+// Distributed under an MIT license: https://codemirror.net/LICENSE
+
+(function(mod) {
+  if (typeof exports == "object" && typeof module == "object") // CommonJS
+    mod(require("../../lib/codemirror"));
+  else if (typeof define == "function" && define.amd) // AMD
+    define(["../../lib/codemirror"], mod);
+  else // Plain browser env
+    mod(CodeMirror);
+})(function(CodeMirror) {
+  "use strict";
+
+  function doFold(cm, pos, options, force) {
+    if (options && options.call) {
+      var finder = options;
+      options = null;
+    } else {
+      var finder = getOption(cm, options, "rangeFinder");
+    }
+    if (typeof pos == "number") pos = CodeMirror.Pos(pos, 0);
+    var minSize = getOption(cm, options, "minFoldSize");
+
+    function getRange(allowFolded) {
+      var range = finder(cm, pos);
+      if (!range || range.to.line - range.from.line < minSize) return null;
+      var marks = cm.findMarksAt(range.from);
+      for (var i = 0; i < marks.length; ++i) {
+        if (marks[i].__isFold && force !== "fold") {
+          if (!allowFolded) return null;
+          range.cleared = true;
+          marks[i].clear();
+        }
+      }
+      return range;
+    }
+
+    var range = getRange(true);
+    if (getOption(cm, options, "scanUp")) while (!range && pos.line > cm.firstLine()) {
+      pos = CodeMirror.Pos(pos.line - 1, 0);
+      range = getRange(false);
+    }
+    if (!range || range.cleared || force === "unfold") return;
+
+    var myWidget = makeWidget(cm, options);
+    CodeMirror.on(myWidget, "mousedown", function(e) {
+      myRange.clear();
+      CodeMirror.e_preventDefault(e);
+    });
+    var myRange = cm.markText(range.from, range.to, {
+      replacedWith: myWidget,
+      clearOnEnter: getOption(cm, options, "clearOnEnter"),
+      __isFold: true
+    });
+    myRange.on("clear", function(from, to) {
+      CodeMirror.signal(cm, "unfold", cm, from, to);
+    });
+    CodeMirror.signal(cm, "fold", cm, range.from, range.to);
+  }
+
+  function makeWidget(cm, options) {
+    var widget = getOption(cm, options, "widget");
+    if (typeof widget == "string") {
+      var text = document.createTextNode(widget);
+      widget = document.createElement("span");
+      widget.appendChild(text);
+      widget.className = "CodeMirror-foldmarker";
+    } else if (widget) {
+      widget = widget.cloneNode(true)
+    }
+    return widget;
+  }
+
+  // Clumsy backwards-compatible interface
+  CodeMirror.newFoldFunction = function(rangeFinder, widget) {
+    return function(cm, pos) { doFold(cm, pos, {rangeFinder: rangeFinder, widget: widget}); };
+  };
+
+  // New-style interface
+  CodeMirror.defineExtension("foldCode", function(pos, options, force) {
+    doFold(this, pos, options, force);
+  });
+
+  CodeMirror.defineExtension("isFolded", function(pos) {
+    var marks = this.findMarksAt(pos);
+    for (var i = 0; i < marks.length; ++i)
+      if (marks[i].__isFold) return true;
+  });
+
+  CodeMirror.commands.toggleFold = function(cm) {
+    cm.foldCode(cm.getCursor());
+  };
+  CodeMirror.commands.fold = function(cm) {
+    cm.foldCode(cm.getCursor(), null, "fold");
+  };
+  CodeMirror.commands.unfold = function(cm) {
+    cm.foldCode(cm.getCursor(), null, "unfold");
+  };
+  CodeMirror.commands.foldAll = function(cm) {
+    cm.operation(function() {
+      for (var i = cm.firstLine(), e = cm.lastLine(); i <= e; i++)
+        cm.foldCode(CodeMirror.Pos(i, 0), null, "fold");
+    });
+  };
+  CodeMirror.commands.unfoldAll = function(cm) {
+    cm.operation(function() {
+      for (var i = cm.firstLine(), e = cm.lastLine(); i <= e; i++)
+        cm.foldCode(CodeMirror.Pos(i, 0), null, "unfold");
+    });
+  };
+
+  CodeMirror.registerHelper("fold", "combine", function() {
+    var funcs = Array.prototype.slice.call(arguments, 0);
+    return function(cm, start) {
+      for (var i = 0; i < funcs.length; ++i) {
+        var found = funcs[i](cm, start);
+        if (found) return found;
+      }
+    };
+  });
+
+  CodeMirror.registerHelper("fold", "auto", function(cm, start) {
+    var helpers = cm.getHelpers(start, "fold");
+    for (var i = 0; i < helpers.length; i++) {
+      var cur = helpers[i](cm, start);
+      if (cur) return cur;
+    }
+  });
+
+  var defaultOptions = {
+    rangeFinder: CodeMirror.fold.auto,
+    widget: "\u2194",
+    minFoldSize: 0,
+    scanUp: false,
+    clearOnEnter: true
+  };
+
+  CodeMirror.defineOption("foldOptions", null);
+
+  function getOption(cm, options, name) {
+    if (options && options[name] !== undefined)
+      return options[name];
+    var editorOptions = cm.options.foldOptions;
+    if (editorOptions && editorOptions[name] !== undefined)
+      return editorOptions[name];
+    return defaultOptions[name];
+  }
+
+  CodeMirror.defineExtension("foldOption", function(options, name) {
+    return getOption(this, options, name);
+  });
+});

src/lib/codemirror/addon/fold/foldgutter.css

@@ -0,0 +1,20 @@
+.CodeMirror-foldmarker {
+  color: blue;
+  text-shadow: #b9f 1px 1px 2px, #b9f -1px -1px 2px, #b9f 1px -1px 2px, #b9f -1px 1px 2px;
+  font-family: arial;
+  line-height: .3;
+  cursor: pointer;
+}
+.CodeMirror-foldgutter {
+  width: .7em;
+}
+.CodeMirror-foldgutter-open,
+.CodeMirror-foldgutter-folded {
+  cursor: pointer;
+}
+.CodeMirror-foldgutter-open:after {
+  content: "\25BE";
+}
+.CodeMirror-foldgutter-folded:after {
+  content: "\25B8";
+}

src/lib/codemirror/addon/fold/foldgutter.js

@@ -0,0 +1,146 @@
+// CodeMirror, copyright (c) by Marijn Haverbeke and others
+// Distributed under an MIT license: https://codemirror.net/LICENSE
+
+(function(mod) {
+  if (typeof exports == "object" && typeof module == "object") // CommonJS
+    mod(require("../../lib/codemirror"), require("./foldcode"));
+  else if (typeof define == "function" && define.amd) // AMD
+    define(["../../lib/codemirror", "./foldcode"], mod);
+  else // Plain browser env
+    mod(CodeMirror);
+})(function(CodeMirror) {
+  "use strict";
+
+  CodeMirror.defineOption("foldGutter", false, function(cm, val, old) {
+    if (old && old != CodeMirror.Init) {
+      cm.clearGutter(cm.state.foldGutter.options.gutter);
+      cm.state.foldGutter = null;
+      cm.off("gutterClick", onGutterClick);
+      cm.off("change", onChange);
+      cm.off("viewportChange", onViewportChange);
+      cm.off("fold", onFold);
+      cm.off("unfold", onFold);
+      cm.off("swapDoc", onChange);
+    }
+    if (val) {
+      cm.state.foldGutter = new State(parseOptions(val));
+      updateInViewport(cm);
+      cm.on("gutterClick", onGutterClick);
+      cm.on("change", onChange);
+      cm.on("viewportChange", onViewportChange);
+      cm.on("fold", onFold);
+      cm.on("unfold", onFold);
+      cm.on("swapDoc", onChange);
+    }
+  });
+
+  var Pos = CodeMirror.Pos;
+
+  function State(options) {
+    this.options = options;
+    this.from = this.to = 0;
+  }
+
+  function parseOptions(opts) {
+    if (opts === true) opts = {};
+    if (opts.gutter == null) opts.gutter = "CodeMirror-foldgutter";
+    if (opts.indicatorOpen == null) opts.indicatorOpen = "CodeMirror-foldgutter-open";
+    if (opts.indicatorFolded == null) opts.indicatorFolded = "CodeMirror-foldgutter-folded";
+    return opts;
+  }
+
+  function isFolded(cm, line) {
+    var marks = cm.findMarks(Pos(line, 0), Pos(line + 1, 0));
+    for (var i = 0; i < marks.length; ++i)
+      if (marks[i].__isFold && marks[i].find().from.line == line) return marks[i];
+  }
+
+  function marker(spec) {
+    if (typeof spec == "string") {
+      var elt = document.createElement("div");
+      elt.className = spec + " CodeMirror-guttermarker-subtle";
+      return elt;
+    } else {
+      return spec.cloneNode(true);
+    }
+  }
+
+  function updateFoldInfo(cm, from, to) {
+    var opts = cm.state.foldGutter.options, cur = from;
+    var minSize = cm.foldOption(opts, "minFoldSize");
+    var func = cm.foldOption(opts, "rangeFinder");
+    cm.eachLine(from, to, function(line) {
+      var mark = null;
+      if (isFolded(cm, cur)) {
+        mark = marker(opts.indicatorFolded);
+      } else {
+        var pos = Pos(cur, 0);
+        var range = func && func(cm, pos);
+        if (range && range.to.line - range.from.line >= minSize)
+          mark = marker(opts.indicatorOpen);
+      }
+      cm.setGutterMarker(line, opts.gutter, mark);
+      ++cur;
+    });
+  }
+
+  function updateInViewport(cm) {
+    var vp = cm.getViewport(), state = cm.state.foldGutter;
+    if (!state) return;
+    cm.operation(function() {
+      updateFoldInfo(cm, vp.from, vp.to);
+    });
+    state.from = vp.from; state.to = vp.to;
+  }
+
+  function onGutterClick(cm, line, gutter) {
+    var state = cm.state.foldGutter;
+    if (!state) return;
+    var opts = state.options;
+    if (gutter != opts.gutter) return;
+    var folded = isFolded(cm, line);
+    if (folded) folded.clear();
+    else cm.foldCode(Pos(line, 0), opts.rangeFinder);
+  }
+
+  function onChange(cm) {
+    var state = cm.state.foldGutter;
+    if (!state) return;
+    var opts = state.options;
+    state.from = state.to = 0;
+    clearTimeout(state.changeUpdate);
+    state.changeUpdate = setTimeout(function() { updateInViewport(cm); }, opts.foldOnChangeTimeSpan || 600);
+  }
+
+  function onViewportChange(cm) {
+    var state = cm.state.foldGutter;
+    if (!state) return;
+    var opts = state.options;
+    clearTimeout(state.changeUpdate);
+    state.changeUpdate = setTimeout(function() {
+      var vp = cm.getViewport();
+      if (state.from == state.to || vp.from - state.to > 20 || state.from - vp.to > 20) {
+        updateInViewport(cm);
+      } else {
+        cm.operation(function() {
+          if (vp.from < state.from) {
+            updateFoldInfo(cm, vp.from, state.from);
+            state.from = vp.from;
+          }
+          if (vp.to > state.to) {
+            updateFoldInfo(cm, state.to, vp.to);
+            state.to = vp.to;
+          }
+        });
+      }
+    }, opts.updateViewportTimeSpan || 400);
+  }
+
+  function onFold(cm, from) {
+    var state = cm.state.foldGutter;
+    if (!state) return;
+    var line = from.line;
+    if (line >= state.from && line < state.to)
+      updateFoldInfo(cm, line, line + 1);
+  }
+});