Add support for diff-patching filter lists

Related discussion:
https://github.com/ameshkov/diffupdates

The benefits of diff-patching filter lists is much shorter update
schedule and significantly less bandwidth consumed.

At the moment, only default filter lists are subject to be
diff-patched.

External filter lists can make their lists diff-patchable by
following the specification link above.

Only filter lists fetched by the auto-updater are candidate for
diff-patching.

Forcing a manual update of the filter lists will prevent the
diff-patcher from kicking in until one or more lists are
auto-updated.

Some back-of-the-envelop calculations regarding the load on free
CDN solutions used by uBO to distribute its own filter lists:

Currently, for each CDN (with lists updating after days):
~560 M req/month, ~78 TB/month

With diff-patching lists on a 6-hour schedule:
~390 M req/month, 1 TB/month

Those estimates were done according to statistics shown by
jsDelivr, which is one of 4 CDNs picked randomly when a list
updates:
https://www.jsdelivr.com/package/gh/uBlockOrigin/uAssetsCDN?tab=stats

assets/assets.dev.json

@@ -163,8 +163,7 @@
 		"cdnURLs": [
 			"https://cdn.jsdelivr.net/gh/uBlockOrigin/uAssetsCDN@latest/thirdparties/easylist.txt",
 			"https://cdn.statically.io/gh/uBlockOrigin/uAssetsCDN/main/thirdparties/easylist.txt",
-			"https://ublockorigin.pages.dev/thirdparties/easylist.txt",
+			"https://ublockorigin.pages.dev/thirdparties/easylist.txt"
-			"https://easylist.to/easylist/easylist.txt"
 		],
 		"supportURL": "https://easylist.to/"
 	},
@@ -215,8 +214,7 @@
 		"cdnURLs": [
 			"https://cdn.jsdelivr.net/gh/uBlockOrigin/uAssetsCDN@latest/thirdparties/easyprivacy.txt",
 			"https://cdn.statically.io/gh/uBlockOrigin/uAssetsCDN/main/thirdparties/easyprivacy.txt",
-			"https://ublockorigin.pages.dev/thirdparties/easyprivacy.txt",
+			"https://ublockorigin.pages.dev/thirdparties/easyprivacy.txt"
-			"https://easylist.to/easylist/easyprivacy.txt"
 		],
 		"supportURL": "https://easylist.to/"
 	},

src/js/assets.js

@@ -35,6 +35,9 @@ import { ubolog } from './console.js';
 const reIsExternalPath = /^(?:[a-z-]+):\/\//;
 const reIsUserAsset = /^user-/;
 const errorCantConnectTo = i18n$('errorCantConnectTo');
+const MS_PER_HOUR = 60 * 60 * 1000;
+const MS_PER_DAY = 24 * MS_PER_HOUR;
+const EXPIRES_DEFAULT = 7;
 
 const assets = {};
 
@@ -42,16 +45,57 @@ const assets = {};
 // bandwidth of remote servers.
 let remoteServerFriendly = false;
 
+/******************************************************************************/
+
+const parseExpires = s => {
+    const matches = s.match(/(\d+)\s*([dh])?/i);
+    if ( matches === null ) { return 0; }
+    let updateAfter = parseInt(matches[1], 10);
+    if ( matches[2] === 'h' ) {
+        updateAfter = Math.ceil(updateAfter / 12) / 2;
+    }
+    return updateAfter;
+};
+
+const extractMetadataFromList = (content, fields) => {
+    const out = {};
+    const head = content.slice(0, 1024);
+    for ( let field of fields ) {
+        field = field.replace(/\s+/g, '-');
+        const re = new RegExp(`^(?:! *|# +)${field.replace(/-/g, '(?: +|-)')}: *(.+)$`, 'im');
+        const match = re.exec(head);
+        let value = match && match[1].trim() || undefined;
+        if ( value !== undefined && /^%.+%$/.test(value) ) {
+            value = undefined;
+        }
+        field = field.toLowerCase().replace(
+            /-[a-z]/g, s => s.charAt(1).toUpperCase()
+        );
+        out[field] = value;
+    }
+    // Pre-process known fields
+    if ( out.lastModified ) {
+        out.lastModified = (new Date(out.lastModified)).getTime() || 0;
+    }
+    if ( out.expires ) {
+        out.expires = Math.max(parseExpires(out.expires), 0.5);
+    }
+    if ( out.diffExpires ) {
+        out.diffExpires = Math.max(parseExpires(out.diffExpires), 0.25);
+    }
+    return out;
+};
+assets.extractMetadataFromList = extractMetadataFromList;
+
 const resourceTimeFromXhr = xhr => {
     try {
         // First lookup timestamp from content
         let assetTime = 0;
         if ( typeof xhr.response === 'string' ) {
-            const head = xhr.response.slice(0, 512);
+            const metadata = extractMetadataFromList(xhr.response, [
-            const match = /^! Last modified: (.+)$/m.exec(head);
+                'Last-Modified'
-            if ( match ) {
+            ]);
-                assetTime = (new Date(match[1])).getTime() || 0;
+            assetTime = metadata.lastModified || 0;
-            }
         }
         // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Age
         // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
@@ -91,6 +135,40 @@ const resourceIsStale = (networkDetails, cacheDetails) => {
     return false;
 };
 
+const getUpdateAfterTime = (assetKey, diff = false) => {
+    const entry = assetCacheRegistry[assetKey];
+    if ( entry ) {
+        if ( diff && typeof entry.diffExpires === 'number' ) {
+            return entry.diffExpires * MS_PER_DAY;
+        }
+        if ( typeof entry.expires === 'number' ) {
+            return entry.expires * MS_PER_DAY;
+        }
+    }
+    if ( assetSourceRegistry ) {
+        const entry = assetSourceRegistry[assetKey];
+        if ( entry && typeof entry.updateAfter === 'number' ) {
+            return entry.updateAfter * MS_PER_DAY;
+        }
+    }
+    return EXPIRES_DEFAULT * MS_PER_DAY; // default to 7-day
+};
+
+const getWriteTime = assetKey => {
+    const entry = assetCacheRegistry[assetKey];
+    if ( entry ) { return entry.writeTime || 0; }
+    return 0;
+};
+
+const isDiffUpdatableAsset = content => {
+    if ( typeof content !== 'string' ) { return false; }
+    const data = extractMetadataFromList(content, [
+        'Diff-Path',
+    ]);
+    return typeof data.diffPath === 'string' &&
+        /^[^%].*[^%]$/.test(data.diffPath);
+};
+
 const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
 
 /******************************************************************************/
@@ -252,14 +330,6 @@ assets.fetchText = async function(url) {
             details.content = '';
             details.error = 'assets.fetchText(): Not a text file';
         }
-
-        // Important: Non empty text resource must always end with a newline
-        if (
-            details.content.length !== 0 &&
-            details.content.endsWith('\n') === false
-        ) {
-            details.content += '\n';
-        }
     } catch(ex) {
         details = ex;
     }
@@ -374,6 +444,13 @@ assets.fetchFilterList = async function(mainlistURL) {
             return { url: mainlistURL, content: '', error: part.error };
         }
         resourceTime = resourceTimeFromParts(allParts, resourceTime);
+        // Skip pre-parser directives for diff-updatable assets
+        if ( allParts.length === 1 && allParts[0] instanceof Object ) {
+            if ( isDiffUpdatableAsset(allParts[0].content) ) {
+                allParts[0] = allParts[0].content;
+                break;
+            }
+        }
         allParts = processIncludeDirectives(allParts);
     } while ( allParts.some(part => typeof part !== 'string') );
     // If we reach this point, this means all fetches were successful.
@@ -457,9 +534,6 @@ function registerAssetSource(assetKey, newDict) {
     } else if ( currentDict.contentURL === undefined ) {
         currentDict.contentURL = [];
     }
-    if ( typeof currentDict.updateAfter !== 'number' ) {
-        currentDict.updateAfter = 7;
-    }
     if ( currentDict.submitter ) {
         currentDict.submitTime = Date.now(); // To detect stale entries
     }
@@ -607,7 +681,7 @@ async function assetCacheRead(assetKey, updateReadTime = false) {
 
     const reportBack = function(content) {
         if ( content instanceof Blob ) { content = ''; }
-        const details = { assetKey: assetKey, content: content };
+        const details = { assetKey, content };
         if ( content === '' ) { details.error = 'ENOTFOUND'; }
         return details;
     };
@@ -979,6 +1053,18 @@ async function getRemote(assetKey) {
             url: contentURL,
             resourceTime: result.resourceTime || 0,
         });
+
+        if ( assetDetails.content === 'filters' ) {
+            const metadata = extractMetadataFromList(result.content, [
+                'Last-Modified',
+                'Expires',
+                'Diff-Name',
+                'Diff-Path',
+                'Diff-Expires',
+            ]);
+            assetCacheSetDetails(assetKey, metadata);
+        }
+
         registerAssetSource(assetKey, { birthtime: undefined, error: undefined });
         return reportBack(result.content);
     }
@@ -1029,8 +1115,7 @@ assets.metadata = async function() {
         if ( cacheEntry ) {
             assetEntry.cached = true;
             assetEntry.writeTime = cacheEntry.writeTime;
-            const obsoleteAfter =
+            const obsoleteAfter = cacheEntry.writeTime + getUpdateAfterTime(assetKey);
-                cacheEntry.writeTime + assetEntry.updateAfter * 86400000;
             assetEntry.obsolete = obsoleteAfter < now;
             assetEntry.remoteURL = cacheEntry.remoteURL;
         } else if (
@@ -1074,7 +1159,7 @@ assets.getUpdateAges = async function(conditions = {}) {
         out.push({
             assetKey,
             age,
-            ageNormalized: age / (asset.updateAfter * 86400000),
+            ageNormalized: age / getUpdateAfterTime(assetKey),
         });
     }
     return out;
@@ -1091,33 +1176,122 @@ let updaterStatus;
 let updaterAssetDelay = updaterAssetDelayDefault;
 let updaterAuto = false;
 
-const updateFirst = function() {
+const getAssetDiffDetails = assetKey => {
+    const out = { name: assetKey };
+    const cacheEntry = assetCacheRegistry[assetKey];
+    if ( cacheEntry === undefined ) { return; }
+    if ( cacheEntry.diffPath === undefined ) { return; }
+    if ( cacheEntry.diffName === undefined ) { return; }
+    out.diffName = cacheEntry.diffName;
+    out.patchPath = cacheEntry.diffPath;
+    const assetEntry = assetSourceRegistry[assetKey];
+    if ( assetEntry === undefined ) { return; }
+    if ( Array.isArray(assetEntry.cdnURLs) === false ) { return; }
+    out.cdnURLs = assetEntry.cdnURLs.slice();
+    return out;
+};
+
+async function diffUpdater() {
+    const toUpdate = await getUpdateCandidates(true);
+    const now = Date.now();
+    const toHardUpdate = [];
+    const toSoftUpdate = [];
+    while ( toUpdate.length !== 0 ) {
+        const assetKey = toUpdate.shift();
+        const assetDetails = getAssetDiffDetails(assetKey);
+        if ( assetDetails === undefined ) { continue; }
+        if ( assetDetails.patchPath === undefined ) { continue; }
+        if ( assetDetails.diffName === undefined ) { continue; }
+        assetDetails.what = 'update';
+        if ( (getWriteTime(assetKey) + getUpdateAfterTime(assetKey, true)) > now ) {
+            toSoftUpdate.push(assetDetails);
+        } else {
+            toHardUpdate.push(assetDetails);
+        }
+    }
+    if ( toHardUpdate.length === 0 ) { return; }
+    ubolog('Diff updater: cycle start');
+    return new Promise(resolve => {
+        let pendingOps = 0;
+        const bc = new globalThis.BroadcastChannel('diffUpdater');
+        bc.onmessage = ev => {
+            const data = ev.data;
+            if ( data.what === 'ready' ) {
+                ubolog('Diff updater: hard updating', toHardUpdate.map(v => v.name).join());
+                while ( toHardUpdate.length !== 0 ) {
+                    const assetDetails = toHardUpdate.shift();
+                    assetDetails.fetch = true;
+                    bc.postMessage(assetDetails);
+                    pendingOps += 1;
+                }
+                return;
+            }
+            if ( data.status === 'needtext' ) {
+                ubolog('Diff updater: need text for', data.name);
+                assetCacheRead(data.name).then(result => {
+                    data.text = result.content;
+                    data.status = undefined;
+                    bc.postMessage(data);
+                });
+                return;
+            }
+            if ( data.status === 'updated' ) {
+                ubolog(`Diff updater: successfully patched ${data.name} using ${data.diffURL}`);
+                const metadata = extractMetadataFromList(data.text, [
+                    'Last-Modified',
+                    'Expires',
+                    'Diff-Name',
+                    'Diff-Path',
+                    'Diff-Expires',
+                ]);
+                assetCacheWrite(data.name, {
+                    content: data.text,
+                    resourceTime: metadata.lastModified || 0,
+                });
+                assetCacheSetDetails(data.name, metadata);
+            } else if ( data.error ) {
+                ubolog(`Diff updater: failed to diff-update ${data.name}, reason: ${data.error}`);
+            }
+            pendingOps -= 1;
+            if ( pendingOps === 0 && toSoftUpdate.length !== 0 ) {
+                ubolog('Diff updater: soft updating', toSoftUpdate.map(v => v.name).join());
+                while ( toSoftUpdate.length !== 0 ) {
+                    bc.postMessage(toSoftUpdate.shift());
+                    pendingOps += 1;
+                }
+            }
+            if ( pendingOps !== 0 ) { return; }
+            ubolog('Diff updater: cycle complete');
+            worker.terminate();
+            bc.close();
+            resolve();
+        };
+        const worker = new Worker('js/diff-updater.js');
+    });
+}
+
+function updateFirst() {
     updaterStatus = 'updating';
     updaterFetched.clear();
     updaterUpdated.length = 0;
-    fireNotification('before-assets-updated');
+    diffUpdater().catch(reason => {
+        ubolog(reason);
+    }).finally(( ) => {
         updateNext();
-};
+    });
+}
 
-const updateNext = async function() {
+async function getUpdateCandidates() {
     const [ assetDict, cacheDict ] = await Promise.all([
         getAssetSourceRegistry(),
         getAssetCacheRegistry(),
     ]);
-
-    const now = Date.now();
     const toUpdate = [];
     for ( const assetKey in assetDict ) {
         const assetEntry = assetDict[assetKey];
         if ( assetEntry.hasRemoteURL !== true ) { continue; }
         if ( updaterFetched.has(assetKey) ) { continue; }
         const cacheEntry = cacheDict[assetKey];
-        if (
-            (cacheEntry instanceof Object) &&
-            (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now
-        ) {
-            continue;
-        }
         if (
             fireNotification('before-asset-updated', {
                 assetKey,
@@ -1132,9 +1306,6 @@ const updateNext = async function() {
             assetCacheRemove(assetKey);
         }
     }
-    if ( toUpdate.length === 0 ) {
-        return updateDone();
-    }
     // https://github.com/uBlockOrigin/uBlock-issues/issues/1165
     //   Update most obsolete asset first.
     toUpdate.sort((a, b) => {
@@ -1142,17 +1313,34 @@ const updateNext = async function() {
         const tb = cacheDict[b] !== undefined ? cacheDict[b].writeTime : 0;
         return ta - tb;
     });
-    updaterFetched.add(toUpdate[0]);
+    return toUpdate;
+}
+
+async function updateNext() {
+    const toUpdate = await getUpdateCandidates();
+    const now = Date.now();
+    const toHardUpdate = [];
+
+    while ( toUpdate.length !== 0 ) {
+        const assetKey = toUpdate.shift();
+        const writeTime = getWriteTime(assetKey);
+        const updateDelay = getUpdateAfterTime(assetKey);
+        if ( (writeTime + updateDelay) > now ) { continue; }
+        toHardUpdate.push(assetKey);
+    }
+    if ( toHardUpdate.length === 0 ) {
+        return updateDone();
+    }
+
+    const assetKey = toHardUpdate.pop();
+    updaterFetched.add(assetKey);
 
     // In auto-update context, be gentle on remote servers.
     remoteServerFriendly = updaterAuto;
 
     let result;
-    if (
+    if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) {
-        toUpdate[0] !== 'assets.json' ||
+        result = await getRemote(assetKey);
-        µb.hiddenSettings.debugAssetsJson !== true
-    ) {
-        result = await getRemote(toUpdate[0]);
     } else {
         result = await assets.fetchText(µb.assetsJsonPath);
         result.assetKey = 'assets.json';
@@ -1161,8 +1349,10 @@ const updateNext = async function() {
     remoteServerFriendly = false;
 
     if ( result.error ) {
+        ubolog(`Full updater: failed to update ${assetKey}`);
         fireNotification('asset-update-failed', { assetKey: result.assetKey });
     } else {
+        ubolog(`Full updater: successfully updated ${assetKey}`);
         updaterUpdated.push(result.assetKey);
         if ( result.assetKey === 'assets.json' && result.content !== '' ) {
             updateAssetSourceRegistry(result.content);
@@ -1170,18 +1360,18 @@ const updateNext = async function() {
     }
 
     updaterTimer.on(updaterAssetDelay);
-};
+}
 
 const updaterTimer = vAPI.defer.create(updateNext);
 
-const updateDone = function() {
+function updateDone() {
     const assetKeys = updaterUpdated.slice(0);
     updaterFetched.clear();
     updaterUpdated.length = 0;
     updaterStatus = undefined;
     updaterAssetDelay = updaterAssetDelayDefault;
     fireNotification('after-assets-updated', { assetKeys });
-};
+}
 
 assets.updateStart = function(details) {
     const oldUpdateDelay = updaterAssetDelay;

src/js/diff-updater.js

@@ -0,0 +1,243 @@
+/*******************************************************************************
+
+    uBlock Origin - a browser extension to block requests.
+    Copyright (C) 2014-present Raymond Hill
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see {http://www.gnu.org/licenses/}.
+
+    Home: https://github.com/gorhill/uBlock
+*/
+
+'use strict';
+
+// This module can be dynamically loaded or spun off as a worker.
+
+/******************************************************************************/
+
+const patches = new Map();
+const encoder = new TextEncoder();
+const reFileName = /[^\/]+$/;
+const EMPTYLINE = '';
+
+/******************************************************************************/
+
+const suffleArray = arr => {
+    const out = arr.slice();
+    for ( let i = 0, n = out.length; i < n; i++ ) {
+        const j = Math.floor(Math.random() * n);
+        if ( j === i ) { continue; }
+        [ out[j], out[i] ] = [ out[i], out[j] ];
+    }
+    return out;
+};
+
+const basename = url => {
+    const match = reFileName.exec(url);
+    return match && match[0] || '';
+};
+
+const resolveURL = (path, url) => {
+    try {
+        const urlAfter = new URL(path, url);
+        return urlAfter.href;
+    }
+    catch(_) {
+    }
+};
+
+function parsePatch(patch) {
+    const patchDetails = new Map();
+    const diffLines = patch.split('\n');
+    let i = 0, n = diffLines.length;
+    while ( i < n ) {
+        const line = diffLines[i++];
+        if ( line.startsWith('diff ') === false ) { continue; }
+        const fields = line.split(/\s+/);
+        const diffBlock = {};
+        for ( let j = 0; j < fields.length; j++ ) {
+            const field = fields[j];
+            const pos = field.indexOf(':');
+            if ( pos === -1 ) { continue; }
+            const name = field.slice(0, pos);
+            if ( name === '' ) { continue; }
+            const value = field.slice(pos+1);
+            switch ( name ) {
+            case 'name':
+            case 'checksum':
+                diffBlock[name] = value;
+                break;
+            case 'lines':
+                diffBlock.lines = parseInt(value, 10);
+                break;
+            default:
+                break;
+            }
+        }
+        if ( diffBlock.name === undefined ) { return; }
+        if ( isNaN(diffBlock.lines) || diffBlock.lines <= 0 ) { return; }
+        if ( diffBlock.checksum === undefined ) { return; }
+        patchDetails.set(diffBlock.name, diffBlock);
+        diffBlock.diff = diffLines.slice(i, i + diffBlock.lines).join('\n');
+        i += diffBlock.lines;
+    }
+    if ( patchDetails.size === 0 ) { return; }
+    return patchDetails;
+}
+
+function applyPatch(text, diff) {
+    // Inspired from (Perl) "sub _patch" at:
+    // https://twiki.org/p/pub/Codev/RcsLite/RcsLite.pm
+    // Apparently authored by John Talintyre in Jan. 2002
+    // https://twiki.org/cgi-bin/view/Codev/RcsLite
+    const lines = text.split('\n');
+    const diffLines = diff.split('\n');
+    let iAdjust = 0;
+    let iDiff = 0, nDiff = diffLines.length;
+    while ( iDiff < nDiff ) {
+        const diffParsed = /^([ad])(\d+) (\d+)$/.exec(diffLines[iDiff++]);
+        if ( diffParsed === null ) { return; }
+        const op = diffParsed[1];
+        const iOp = parseInt(diffParsed[2], 10);
+        const nOp = parseInt(diffParsed[3], 10);
+        const iOpAdj = iOp + iAdjust;
+        if ( iOpAdj > lines.length ) { return; }
+        // Delete lines
+        if ( op === 'd' ) {
+            lines.splice(iOpAdj-1, nOp);
+            iAdjust -= nOp;
+            continue;
+        }
+        // Add lines: Don't use splice() to avoid stack limit issues
+        for ( let i = 0; i < nOp; i++ ) {
+            lines.push(EMPTYLINE);
+        }
+        lines.copyWithin(iOpAdj+nOp, iOpAdj);
+        for ( let i = 0; i < nOp; i++ ) {
+            lines[iOpAdj+i] = diffLines[iDiff+i];
+        }
+        iAdjust += nOp;
+        iDiff += nOp;
+    }
+    return lines.join('\n');
+}
+
+/******************************************************************************/
+
+// Async
+
+async function applyPatchAndValidate(assetDetails, diffDetails) {
+    const { text } = assetDetails;
+    const { diff, checksum } = diffDetails;
+    const textAfter = applyPatch(text, diff);
+    if ( typeof textAfter !== 'string' ) {
+        assetDetails.error = 'baddiff';
+        return false;
+    }
+    const crypto = globalThis.crypto;
+    if ( typeof crypto !== 'object' ) {
+        assetDetails.error = 'nocrypto';
+        return false;
+    }
+    const arrayin = encoder.encode(textAfter);
+    const arraybuffer = await crypto.subtle.digest('SHA-1', arrayin);
+    const arrayout = new Uint8Array(arraybuffer);
+    const sha1Full = Array.from(arrayout).map(i =>
+        i.toString(16).padStart(2, '0')
+    ).join('');
+    if ( sha1Full.startsWith(checksum) === false ) {
+        assetDetails.error = 'badchecksum';
+        return false;
+    }
+    assetDetails.text = textAfter;
+    const head = textAfter.slice(0, 1024);
+    let match = /^! Diff-Path: (\S+)/m.exec(head);
+    assetDetails.patchPath = match ? match[1] : undefined;
+    match = /^! Diff-Name: (\S+)/m.exec(head);
+    assetDetails.diffName = match ? match[1] : undefined;
+    return true;
+}
+
+async function fetchPatchDetailsFromCDNs(assetDetails) {
+    const { patchPath, cdnURLs } = assetDetails;
+    if ( Array.isArray(cdnURLs) === false ) { return null; }
+    if ( cdnURLs.length === 0 ) { return null; }
+    for ( const cdnURL of suffleArray(cdnURLs) ) {
+        const diffURL = resolveURL(patchPath, cdnURL);
+        if ( diffURL === undefined ) { continue; }
+        const response = await fetch(diffURL).catch(reason => {
+            console.error(reason);
+        });
+        if ( response === undefined ) { continue; }
+        if ( response.ok !== true ) { continue; }
+        const patchText = await response.text();
+        const patchDetails = parsePatch(patchText);
+        if ( patchDetails === undefined ) { continue; }
+        return { diffURL, patchDetails };
+    }
+    return null;
+}
+
+async function fetchPatchDetails(assetDetails) {
+    const { patchPath } = assetDetails;
+    const patchFile = basename(patchPath);
+    if ( patchFile === '' ) { return null; }
+    if ( patches.has(patchFile) ) {
+        return patches.get(patchFile);
+    }
+    if ( assetDetails.fetch === false ) { return null; }
+    const patchDetailsPromise = fetchPatchDetailsFromCDNs(assetDetails);
+    patches.set(patchFile, patchDetailsPromise);
+    return patchDetailsPromise;
+}
+
+async function fetchAndApplyAllPatches(assetDetails) {
+    const { diffURL, patchDetails } = await fetchPatchDetails(assetDetails);
+    if ( patchDetails === null ) {
+        assetDetails.error = 'nopatch';
+        return assetDetails;
+    }
+    const diffDetails = patchDetails.get(assetDetails.diffName);
+    if ( diffDetails === undefined ) {
+        assetDetails.error = 'nodiff';
+        return assetDetails;
+    }
+    if ( assetDetails.text === undefined ) {
+        assetDetails.status = 'needtext';
+        return assetDetails;
+    }
+    const outcome = await applyPatchAndValidate(assetDetails, diffDetails);
+    if ( outcome !== true ) { return assetDetails; }
+    assetDetails.status = 'updated';
+    assetDetails.diffURL = diffURL;
+    return assetDetails;
+}
+
+/******************************************************************************/
+
+const bc = new globalThis.BroadcastChannel('diffUpdater');
+
+bc.onmessage = ev => {
+    const message = ev.data;
+    switch ( message.what ) {
+    case 'update':
+        fetchAndApplyAllPatches(message).then(response => {
+            bc.postMessage(response);
+        });
+        break;
+    }
+};
+
+bc.postMessage({ what: 'ready' });
+
+/******************************************************************************/

src/js/storage.js

@@ -1009,39 +1009,20 @@ import {
 µb.extractFilterListMetadata = function(assetKey, raw) {
     const listEntry = this.availableFilterLists[assetKey];
     if ( listEntry === undefined ) { return; }
-    // Metadata expected to be found at the top of content.
-    const head = raw.slice(0, 1024);
     // https://github.com/gorhill/uBlock/issues/313
     // Always try to fetch the name if this is an external filter list.
-    if ( listEntry.group === 'custom' ) {
+    if ( listEntry.group !== 'custom' ) { return; }
-        let matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Title[\t ]*:([^\n]+)/i);
+    const data = io.extractMetadataFromList(raw, [ 'Title', 'Homepage' ]);
-        const title = matches && matches[1].trim() || '';
+    const props = {};
-        if ( title !== '' && title !== listEntry.title ) {
+    if ( data.title && data.title !== listEntry.title ) {
-            listEntry.title = orphanizeString(title);
+        props.title = listEntry.title = orphanizeString(data.title);
-            io.registerAssetSource(assetKey, { title });
-        }
-        matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Homepage[\t ]*:[\t ]*(https?:\/\/\S+)\s/i);
-        const supportURL = matches && matches[1] || '';
-        if ( supportURL !== '' && supportURL !== listEntry.supportURL ) {
-            listEntry.supportURL = orphanizeString(supportURL);
-            io.registerAssetSource(assetKey, { supportURL });
-        }
-    }
-    // Extract update frequency information
-    const matches = head.match(/(?:^|\n)(?:!|# )[\t ]*Expires[\t ]*:[\t ]*(\d+)[\t ]*(h)?/i);
-    if ( matches !== null ) {
-        let updateAfter = parseInt(matches[1], 10);
-        if ( isNaN(updateAfter) === false ) {
-            if ( matches[2] !== undefined ) {
-                updateAfter = Math.ceil(updateAfter / 12) / 2;
-            }
-            updateAfter = Math.max(updateAfter, 0.5);
-            if ( updateAfter !== listEntry.updateAfter ) {
-                listEntry.updateAfter = updateAfter;
-                io.registerAssetSource(assetKey, { updateAfter });
     }
+    if ( data.homepage && /^https?:\/\/\S+/.test(data.homepage) ) {
+        if ( data.homepage !== listEntry.supportURL ) {
+            props.supportURL = listEntry.supportURL = orphanizeString(data.homepage);
         }
     }
+    io.registerAssetSource(assetKey, props);
 };
 
 /******************************************************************************/