From 4687c60bf9cec5b68e5a007d740af93f4119a79e Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Wed, 8 Apr 2020 09:57:55 -0400 Subject: [PATCH] Support fetching assets from CDNs when auto-updating This commit add the ability to fetch from CDN servers when an asset is fetched as a result of auto-update. If an asset has a `cdnURLs` entry in `assets.json`, the asset will be auto-updated using one of those CDN URLs. When many CDN URLs are specified, those URLs will be shuffled in order to spread the bandwidth across all specified CDN servers. If all specified CDN servers fail to respond, uBO will fall back to usual `contentURLs` entry. The `cdnURLs` are used only when an asset is auto-updated, this ensures a user will get the more recent available version of an asset when manually updating. The motivation of this new feature is to relieve GitHub from acting as a CDN (which it is not) for uBO -- an increasing concern with the growing adoption of uBO along with the growing size of key uBO assets. --- src/js/assets.js | 72 +++++++++++++++++++++++++++++++---------------- src/js/storage.js | 7 +++-- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index cf1eb98cc..3b0ef17c7 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -33,6 +33,10 @@ const errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); const api = {}; +// A hint for various pieces of code to take measures if possible to save +// bandwidth of remote servers. +let remoteServerFriendly = false; + /******************************************************************************/ const observers = []; @@ -157,14 +161,15 @@ api.fetchText = async function(url) { // https://github.com/gorhill/uBlock/issues/2592 // Force browser cache to be bypassed, but only for resources which have // been fetched more than one hour ago. - // // https://github.com/uBlockOrigin/uBlock-issues/issues/682#issuecomment-515197130 // Provide filter list authors a way to completely bypass // the browser cache. // https://github.com/gorhill/uBlock/commit/048bfd251c9b#r37972005 // Use modulo prime numbers to avoid generating the same token at the // same time across different days. - if ( isExternal ) { + // Do not bypass browser cache if we are asked to be gentle on remote + // servers. + if ( isExternal && remoteServerFriendly !== true ) { const cacheBypassToken = µBlock.hiddenSettings.updateAssetBypassBrowserCache ? Math.floor(Date.now() / 1000) % 86413 @@ -743,6 +748,19 @@ const getRemote = async function(assetKey) { contentURLs = assetDetails.contentURL.slice(0); } + // If asked to be gentle on remote servers, favour using dedicated CDN + // servers. If more than one CDN server is present, randomly shuffle the + // set of servers so as to spread the bandwidth burden. + if ( remoteServerFriendly && Array.isArray(assetDetails.cdnURLs) ) { + const cdnURLs = assetDetails.cdnURLs.slice(); + for ( let i = 0, n = cdnURLs.length; i < n; i++ ) { + const j = Math.floor(Math.random() * n); + if ( j === i ) { continue; } + [ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ]; + } + contentURLs.unshift(...cdnURLs); + } + for ( const contentURL of contentURLs ) { if ( reIsExternalPath.test(contentURL) === false ) { continue; } @@ -756,18 +774,17 @@ const getRemote = async function(assetKey) { if ( result.statusCode === 0 ) { error = 'network error'; } - registerAssetSource( - assetKey, - { error: { time: Date.now(), error } } - ); + registerAssetSource(assetKey, { + error: { time: Date.now(), error } + }); continue; } // Success - assetCacheWrite( - assetKey, - { content: result.content, url: contentURL } - ); + assetCacheWrite(assetKey, { + content: result.content, + url: contentURL + }); registerAssetSource(assetKey, { error: undefined }); return reportBack(result.content); } @@ -835,9 +852,10 @@ const updaterAssetDelayDefault = 120000; const updaterUpdated = []; const updaterFetched = new Set(); -let updaterStatus, - updaterTimer, - updaterAssetDelay = updaterAssetDelayDefault; +let updaterStatus; +let updaterTimer; +let updaterAssetDelay = updaterAssetDelayDefault; +let updaterAuto = false; const updateFirst = function() { updaterStatus = 'updating'; @@ -861,25 +879,22 @@ const updateNext = async function() { if ( updaterFetched.has(assetKey) ) { continue; } const cacheEntry = cacheDict[assetKey]; if ( - cacheEntry && + (cacheEntry instanceof Object) && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { continue; } if ( - fireNotification( - 'before-asset-updated', - { assetKey: assetKey, type: assetEntry.content } - ) === true + fireNotification('before-asset-updated', { + assetKey, + type: assetEntry.content + }) === true ) { assetKeyToUpdate = assetKey; break; } // This will remove a cached asset when it's no longer in use. - if ( - cacheEntry && - cacheEntry.readTime < assetCacheRegistryStartTime - ) { + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { assetCacheRemove(assetKey); } } @@ -888,7 +903,13 @@ const updateNext = async function() { } updaterFetched.add(assetKeyToUpdate); + // In auto-update context, be gentle on remote servers. + remoteServerFriendly = updaterAuto; + const result = await getRemote(assetKeyToUpdate); + + remoteServerFriendly = false; + if ( result.content !== '' ) { updaterUpdated.push(result.assetKey); if ( result.assetKey === 'assets.json' ) { @@ -912,10 +933,11 @@ const updateDone = function() { api.updateStart = function(details) { const oldUpdateDelay = updaterAssetDelay; - const newUpdateDelay = typeof details.delay === 'number' ? - details.delay : - updaterAssetDelayDefault; + const newUpdateDelay = typeof details.delay === 'number' + ? details.delay + : updaterAssetDelayDefault; updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); + updaterAuto = details.auto === true; if ( updaterStatus !== undefined ) { if ( newUpdateDelay < oldUpdateDelay ) { clearTimeout(updaterTimer); diff --git a/src/js/storage.js b/src/js/storage.js index 48d767648..9672f38f0 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1144,6 +1144,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { const json = await vAPI.adminStorage.getItem('adminSettings'); if ( typeof json === 'string' && json !== '' ) { data = JSON.parse(json); + } else if ( json instanceof Object ) { + data = json; } } catch (ex) { console.error(ex); @@ -1247,7 +1249,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { /******************************************************************************/ -µBlock.scheduleAssetUpdater = (function() { +µBlock.scheduleAssetUpdater = (( ) => { let timer, next = 0; return function(updateDelay) { @@ -1271,7 +1273,8 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { next = 0; this.assets.updateStart({ delay: this.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || - 120000 + 120000, + auto: true, }); }, updateDelay); };