From 2cd062898c0f6007ea9b44b36472c210baa1b0e4 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Thu, 23 Mar 2023 13:40:51 -0400 Subject: [PATCH] Properly handle default list status changes in assets.json This commit fix properly handling toggling off the default status of a list such that the list will be automatically turned off when its status change from default to non-default. Additionally, imported lists which become stock lists will be properly migrated from imported lists section. --- Makefile | 2 +- src/js/assets.js | 54 ++++++----- src/js/background.js | 1 + src/js/storage.js | 208 ++++++++++++++++++++++++++----------------- 4 files changed, 162 insertions(+), 103 deletions(-) diff --git a/Makefile b/Makefile index 1201e4a46..f7ce6b215 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ run_options := $(filter-out $@,$(MAKECMDGOALS)) .PHONY: all clean test lint chromium opera firefox npm dig mv3 mv3-quick \ compare maxcost medcost mincost modifiers record wasm -sources := $(wildcard assets/resources/* dist/version src/* src/*/* src/*/*/* src/*/*/*/*) +sources := $(wildcard assets/* assets/*/* dist/version src/* src/*/* src/*/*/* src/*/*/*/*) platform := $(wildcard platform/* platform/*/* platform/*/*/* platform/*/*/*/*) assets := dist/build/uAssets diff --git a/src/js/assets.js b/src/js/assets.js index b1723c9bf..2872fe5ac 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -379,22 +379,21 @@ const getAssetSourceRegistry = function() { return assetSourceRegistryPromise; }; -const registerAssetSource = function(assetKey, dict) { - const entry = assetSourceRegistry[assetKey] || {}; - for ( const prop in dict ) { - if ( dict.hasOwnProperty(prop) === false ) { continue; } - if ( dict[prop] === undefined ) { - delete entry[prop]; +const registerAssetSource = function(assetKey, newDict) { + const currentDict = assetSourceRegistry[assetKey] || {}; + for ( const [ k, v ] of Object.entries(newDict) ) { + if ( v === undefined || v === null ) { + delete currentDict[k]; } else { - entry[prop] = dict[prop]; + currentDict[k] = newDict[k]; } } - let contentURL = dict.contentURL; + let contentURL = newDict.contentURL; if ( contentURL !== undefined ) { if ( typeof contentURL === 'string' ) { - contentURL = entry.contentURL = [ contentURL ]; + contentURL = currentDict.contentURL = [ contentURL ]; } else if ( Array.isArray(contentURL) === false ) { - contentURL = entry.contentURL = []; + contentURL = currentDict.contentURL = []; } let remoteURLCount = 0; for ( let i = 0; i < contentURL.length; i++ ) { @@ -402,18 +401,18 @@ const registerAssetSource = function(assetKey, dict) { remoteURLCount += 1; } } - entry.hasLocalURL = remoteURLCount !== contentURL.length; - entry.hasRemoteURL = remoteURLCount !== 0; - } else if ( entry.contentURL === undefined ) { - entry.contentURL = []; + currentDict.hasLocalURL = remoteURLCount !== contentURL.length; + currentDict.hasRemoteURL = remoteURLCount !== 0; + } else if ( currentDict.contentURL === undefined ) { + currentDict.contentURL = []; } - if ( typeof entry.updateAfter !== 'number' ) { - entry.updateAfter = 5; + if ( typeof currentDict.updateAfter !== 'number' ) { + currentDict.updateAfter = 5; } - if ( entry.submitter ) { - entry.submitTime = Date.now(); // To detect stale entries + if ( currentDict.submitter ) { + currentDict.submitTime = Date.now(); // To detect stale entries } - assetSourceRegistry[assetKey] = entry; + assetSourceRegistry[assetKey] = currentDict; }; const unregisterAssetSource = function(assetKey) { @@ -443,12 +442,18 @@ const updateAssetSourceRegistry = function(json, silent = false) { let newDict; try { newDict = JSON.parse(json); + newDict['assets.json'].defaultListset = + Array.from(Object.entries(newDict)) + .filter(a => a[1].content === 'filters' && a[1].off === undefined) + .map(a => a[0]); } catch (ex) { } if ( newDict instanceof Object === false ) { return; } const oldDict = assetSourceRegistry; + fireNotification('assets.json-updated', { newDict, oldDict }); + // Remove obsolete entries (only those which were built-in). for ( const assetKey in oldDict ) { if ( @@ -1005,7 +1010,16 @@ const updateNext = async function() { // In auto-update context, be gentle on remote servers. remoteServerFriendly = updaterAuto; - const result = await getRemote(toUpdate[0]); + let result; + if ( + toUpdate[0] !== 'assets.json' || + µb.hiddenSettings.debugAssetsJson !== true + ) { + result = await getRemote(toUpdate[0]); + } else { + result = await assets.fetchText('/assets/assets.json'); + result.assetKey = 'assets.json'; + } remoteServerFriendly = false; diff --git a/src/js/background.js b/src/js/background.js index 099d75886..d3fcae4a1 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -65,6 +65,7 @@ const hiddenSettingsDefault = { cnameReplayFullURL: false, cnameUncloakProxied: false, consoleLogLevel: 'unset', + debugAssetsJson: false, debugScriptlets: false, debugScriptletInjector: false, disableWebAssembly: false, diff --git a/src/js/storage.js b/src/js/storage.js index 1bea75287..09a8c7e95 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -649,22 +649,21 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ µb.getAvailableLists = async function() { - let oldAvailableLists = {}, - newAvailableLists = {}; + const newAvailableLists = {}; - // User filter list. + // User filter list newAvailableLists[this.userFiltersPath] = { content: 'filters', group: 'user', title: i18n$('1pPageName'), }; - // Custom filter lists. - const importedListKeys = this.listKeysFromCustomFilterLists( - this.userSettings.importedLists + // Custom filter lists + const importedListKeys = new Set( + this.listKeysFromCustomFilterLists(this.userSettings.importedLists) ); for ( const listKey of importedListKeys ) { - const entry = { + const asset = { content: 'filters', contentURL: listKey, external: true, @@ -672,45 +671,17 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { submitter: 'user', title: '', }; - newAvailableLists[listKey] = entry; - io.registerAssetSource(listKey, entry); + newAvailableLists[listKey] = asset; + io.registerAssetSource(listKey, asset); } - // Convert a no longer existing stock list into an imported list, except - // when the removed stock list is deemed a "bad list". - const customListFromStockList = assetKey => { - const oldEntry = oldAvailableLists[assetKey]; - if ( oldEntry === undefined || oldEntry.off === true ) { return; } - let listURL = oldEntry.contentURL; - if ( Array.isArray(listURL) ) { - listURL = listURL[0]; - } - if ( this.badLists.has(listURL) ) { return; } - const newEntry = { - content: 'filters', - contentURL: listURL, - external: true, - group: 'custom', - submitter: 'user', - title: oldEntry.title || '' - }; - newAvailableLists[listURL] = newEntry; - io.registerAssetSource(listURL, newEntry); - importedListKeys.push(listURL); - this.userSettings.importedLists.push(listURL.trim()); - this.saveUserSettings(); - this.saveSelectedFilterLists([ listURL ], true); - }; - - const promises = [ + // Load previously saved available lists -- these contains data + // computed at run-time, we will reuse this data if possible + const [ bin, registeredAssets, badlists ] = await Promise.all([ vAPI.storage.get('availableFilterLists'), io.metadata(), this.badLists.size === 0 ? io.get('ublock-badlists') : false, - ]; - - // Load previously saved available lists -- these contains data - // computed at run-time, we will reuse this data if possible. - const [ bin, entries, badlists ] = await Promise.all(promises); + ]); if ( badlists instanceof Object ) { for ( const line of badlists.content.split(/\s*[\n\r]+\s*/) ) { @@ -721,51 +692,97 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { } } - oldAvailableLists = bin && bin.availableFilterLists || {}; + const oldAvailableLists = bin && bin.availableFilterLists || {}; - for ( const assetKey in entries ) { - if ( entries.hasOwnProperty(assetKey) === false ) { continue; } - const entry = entries[assetKey]; - if ( entry.content !== 'filters' ) { continue; } - newAvailableLists[assetKey] = Object.assign({}, entry); + for ( const [ assetKey, asset ] of Object.entries(registeredAssets) ) { + if ( asset.content !== 'filters' ) { continue; } + newAvailableLists[assetKey] = Object.assign({}, asset); } - // Load set of currently selected filter lists. - const listKeySet = new Set(this.selectedFilterLists); - for ( const listKey in newAvailableLists ) { - if ( newAvailableLists.hasOwnProperty(listKey) ) { - newAvailableLists[listKey].off = !listKeySet.has(listKey); - } - } + // Load set of currently selected filter lists + const selectedListset = new Set(this.selectedFilterLists); - //finalize(); - // Final steps: - // - reuse existing list metadata if any; - // - unregister unreferenced imported filter lists if any. - // Reuse existing metadata. - for ( const assetKey in oldAvailableLists ) { - const oldEntry = oldAvailableLists[assetKey]; - const newEntry = newAvailableLists[assetKey]; - // List no longer exists. If a stock list, try to convert to - // imported list if it was selected. - if ( newEntry === undefined ) { - this.removeFilterList(assetKey); - if ( assetKey.indexOf('://') === -1 ) { - customListFromStockList(assetKey); + // Remove imported filter lists which are already present in stock lists + for ( const [ stockAssetKey, stockEntry ] of Object.entries(newAvailableLists) ) { + if ( stockEntry.content !== 'filters' ) { continue; } + if ( stockEntry.group === 'user' ) { continue; } + if ( stockEntry.submitter === 'user' ) { continue; } + if ( stockAssetKey.includes('://') ) { continue; } + const contentURLs = Array.isArray(stockEntry.contentURL) + ? stockEntry.contentURL + : [ stockEntry.contentURL ]; + for ( const importedAssetKey of contentURLs ) { + const importedEntry = newAvailableLists[importedAssetKey]; + if ( importedEntry === undefined ) { continue; } + delete newAvailableLists[importedAssetKey]; + io.unregisterAssetSource(importedAssetKey); + this.removeFilterList(importedAssetKey); + if ( selectedListset.has(importedAssetKey) ) { + selectedListset.add(stockAssetKey); + selectedListset.delete(importedAssetKey); } - continue; + importedListKeys.delete(importedAssetKey); + break; } + } + + // Unregister lists in old listset not present in new listset. + // Convert a no longer existing stock list into an imported list, except + // when the removed stock list is deemed a "bad list". + for ( const [ assetKey, oldEntry ] of Object.entries(oldAvailableLists) ) { + if ( newAvailableLists[assetKey] !== undefined ) { continue; } + const on = selectedListset.delete(assetKey); + this.removeFilterList(assetKey); + io.unregisterAssetSource(assetKey); + if ( assetKey.includes('://') ) { continue; } + if ( on === false ) { continue; } + const listURL = Array.isArray(oldEntry.contentURL) + ? oldEntry.contentURL[0] + : oldEntry.contentURL; + if ( this.badLists.has(listURL) ) { continue; } + const newEntry = { + content: 'filters', + contentURL: listURL, + external: true, + group: 'custom', + submitter: 'user', + title: oldEntry.title || '' + }; + newAvailableLists[listURL] = newEntry; + io.registerAssetSource(listURL, newEntry); + importedListKeys.add(listURL); + selectedListset.add(listURL); + } + + // Remove unreferenced imported filter lists + for ( const [ assetKey, asset ] of Object.entries(newAvailableLists) ) { + if ( asset.submitter !== 'user' ) { continue; } + if ( importedListKeys.has(assetKey) ) { continue; } + selectedListset.delete(assetKey); + delete newAvailableLists[assetKey]; + this.removeFilterList(assetKey); + io.unregisterAssetSource(assetKey); + } + + // Mark lists as disabled/enabled according to selected listset + for ( const [ assetKey, asset ] of Object.entries(newAvailableLists) ) { + asset.off = selectedListset.has(assetKey) === false; + } + + // Reuse existing metadata + for ( const [ assetKey, oldEntry ] of Object.entries(oldAvailableLists) ) { + const newEntry = newAvailableLists[assetKey]; + if ( newEntry === undefined ) { continue; } if ( oldEntry.entryCount !== undefined ) { newEntry.entryCount = oldEntry.entryCount; } if ( oldEntry.entryUsedCount !== undefined ) { newEntry.entryUsedCount = oldEntry.entryUsedCount; } - // This may happen if the list name was pulled from the list - // content. + // This may happen if the list name was pulled from the list content // https://github.com/chrisaljoudi/uBlock/issues/982 - // There is no guarantee the title was successfully extracted from - // the list content. + // There is no guarantee the title was successfully extracted from + // the list content if ( newEntry.title === '' && typeof oldEntry.title === 'string' && @@ -775,14 +792,13 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { } } - // Remove unreferenced imported filter lists. - for ( const assetKey in newAvailableLists ) { - const newEntry = newAvailableLists[assetKey]; - if ( newEntry.submitter !== 'user' ) { continue; } - if ( importedListKeys.indexOf(assetKey) !== -1 ) { continue; } - delete newAvailableLists[assetKey]; - io.unregisterAssetSource(assetKey); - this.removeFilterList(assetKey); + if ( Array.from(importedListKeys).join('\n') !== this.userSettings.importedLists.join('\n') ) { + this.userSettings.importedLists = Array.from(importedListKeys); + this.saveUserSettings(); + } + + if ( Array.from(selectedListset).join() !== this.selectedFilterLists.join() ) { + this.saveSelectedFilterLists(Array.from(selectedListset)); } return newAvailableLists; @@ -1580,7 +1596,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { if ( topic === 'builtin-asset-source-added' ) { if ( details.entry.content === 'filters' ) { if ( - details.entry.off !== true || + details.entry.off === true && this.listMatchesEnvironment(details.entry) ) { this.saveSelectedFilterLists([ details.assetKey ], true); @@ -1588,4 +1604,32 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { } return; } + + if ( topic === 'assets.json-updated' ) { + const { newDict, oldDict } = details; + const newDefaultListset = new Set(newDict['assets.json'].defaultListset || []); + const oldDefaultListset = new Set(oldDict['assets.json'].defaultListset || []); + if ( oldDefaultListset.size === 0 ) { + Array.from(Object.entries(newDict)) + .filter(a => a[1].content === 'filters' && a[1].off === undefined) + .map(a => a[0]) + .forEach(a => oldDefaultListset.add(a)); + } + const selectedListset = new Set(this.selectedFilterLists); + let selectedListModified = false; + for ( const assetKey of oldDefaultListset ) { + if ( newDefaultListset.has(assetKey) ) { continue; } + selectedListset.delete(assetKey); + selectedListModified = true; + } + for ( const assetKey of newDefaultListset ) { + if ( oldDefaultListset.has(assetKey) ) { continue; } + selectedListset.add(assetKey); + selectedListModified = true; + } + if ( selectedListModified ) { + this.saveSelectedFilterLists(Array.from(selectedListset)); + } + return; + } };