From 3b9fd49c508812dfe2545836c63bd5d8cfd2ead4 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Wed, 18 Jan 2017 13:17:47 -0500 Subject: [PATCH] Assets management refactored (#2314) * refactoring assets management code * finalizing refactoring of assets management * various code review of new assets management code * fix #2281 * fix #1961 * fix #1293 * fix #1275 * fix update scheduler timing logic * forward compatibility (to be removed once 1.11+ is widespread) * more codereview; give admins ability to specify own assets.json * "assetKey" is more accurate than "path" * fix group count update when building dom incrementally * reorganize content (order, added URLs, etc.) * ability to customize updater through advanced settings * better spinner icon --- assets/assets.json | 585 ++++++++ src/3p-filters.html | 16 +- src/_locales/en/messages.json | 4 +- src/background.html | 1 - src/css/3p-filters.css | 91 +- src/js/3p-filters.js | 439 +++--- src/js/assets.js | 2258 +++++++++++-------------------- src/js/background.js | 103 +- src/js/logger.js | 4 +- src/js/messaging.js | 89 +- src/js/redirect-engine.js | 24 +- src/js/reverselookup-worker.js | 18 +- src/js/reverselookup.js | 38 +- src/js/scriptlets/subscriber.js | 9 +- src/js/settings.js | 5 +- src/js/start.js | 27 +- src/js/storage.js | 967 ++++++------- src/js/ublock.js | 8 +- tools/make-assets.sh | 4 +- tools/make-chromium.sh | 6 +- 20 files changed, 2171 insertions(+), 2525 deletions(-) create mode 100644 assets/assets.json diff --git a/assets/assets.json b/assets/assets.json new file mode 100644 index 000000000..883b87ba0 --- /dev/null +++ b/assets/assets.json @@ -0,0 +1,585 @@ +{ + "assets.json": { + "content": "internal", + "updateAfter": 13, + "contentURL": [ + "https://raw.githubusercontent.com/gorhill/uBlock/master/assets/assets.json", + "assets/assets.json" + ] + }, + "public_suffix_list.dat": { + "content": "internal", + "updateAfter": 19, + "contentURL": [ + "https://publicsuffix.org/list/public_suffix_list.dat", + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat" + ] + }, + "ublock-resources": { + "content": "internal", + "updateAfter": 7, + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resources.txt", + "assets/ublock/resources.txt" + ] + }, + "ublock-filters": { + "content": "filters", + "group": "default", + "title": "uBlock filters", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", + "assets/ublock/filters.txt" + ] + }, + "ublock-badware": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Badware risks", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt", + "assets/ublock/badware.txt" + ], + "supportURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks", + "instructionURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks" + }, + "ublock-experimental": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Experimental", + "off": true, + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/experimental.txt", + "assets/ublock/experimental.txt" + ], + "supportURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters", + "instructionURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters" + }, + "ublock-privacy": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Privacy", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt", + "assets/ublock/privacy.txt" + ] + }, + "ublock-unbreak": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Unbreak", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt", + "assets/ublock/unbreak.txt" + ] + }, + "awrl-0": { + "content": "filters", + "group": "ads", + "off": true, + "title": "Adblock Warning Removal List", + "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt", + "supportURL": "https://forums.lanik.us/" + }, + "reek-0": { + "content": "filters", + "group": "ads", + "off": true, + "title": "Anti-Adblock Killer | Reek", + "contentURL": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", + "supportURL": "https://github.com/reek/anti-adblock-killer", + "instructionURL": "https://github.com/reek/anti-adblock-killer#instruction" + }, + "easylist": { + "content": "filters", + "group": "ads", + "title": "EasyList", + "contentURL": [ + "https://easylist.to/easylist/easylist.txt", + "https://easylist-downloads.adblockplus.org/easylist.txt", + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easylist.txt", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "easylist-nocosmetic": { + "content": "filters", + "group": "ads", + "off": true, + "title": "EasyList without element hiding rules", + "contentURL": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", + "supportURL": "https://forums.lanik.us/" + }, + "disconnect-tracking": { + "content": "filters", + "group": "privacy", + "off": true, + "title": "Basic tracking list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" + }, + "easyprivacy": { + "content": "filters", + "group": "privacy", + "title": "EasyPrivacy", + "contentURL": [ + "https://easylist.to/easylist/easyprivacy.txt", + "https://easylist-downloads.adblockplus.org/easyprivacy.txt", + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-enhanced": { + "content": "filters", + "group": "privacy", + "off": true, + "title": "Fanboy’s Enhanced Tracking List", + "contentURL": "https://www.fanboy.co.nz/enhancedstats.txt", + "supportURL": "https://forums.lanik.us/" + }, + "disconnect-malvertising": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Malvertising filter list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt" + }, + "malware-0": { + "content": "filters", + "group": "malware", + "title": "Malware Domain List", + "contentURL": [ + "https://www.malwaredomainlist.com/hostslist/hosts.txt", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt" + ] + }, + "malware-1": { + "content": "filters", + "group": "malware", + "title": "Malware domains", + "contentURL": [ + "https://mirror.cedia.org.ec/malwaredomains/justdomains", + "https://mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" + }, + "malware-2": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Malware domains (long-lived)", + "contentURL": [ + "https://mirror1.malwaredomains.com/files/immortal_domains.txt", + "https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" + }, + "disconnect-malware": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Malware filter list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt" + }, + "spam404-0": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Spam404", + "contentURL": "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt", + "supportURL": "http://www.spam404.com/" + }, + "fanboy-thirdparty_social": { + "content": "filters", + "group": "social", + "off": true, + "title": "Anti-ThirdpartySocial (see warning inside list)", + "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-annoyance": { + "content": "filters", + "group": "social", + "off": true, + "title": "Fanboy’s Annoyance List", + "contentURL": [ + "https://easylist.to/easylist/fanboy-annoyance.txt", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-social": { + "content": "filters", + "group": "social", + "off": true, + "title": "Fanboy’s Social Blocking List", + "contentURL": [ + "https://easylist.to/easylist/fanboy-social.txt", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "dpollock-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "off": true, + "title": "Dan Pollock’s hosts file", + "contentURL": "http://someonewhocares.org/hosts/hosts", + "supportURL": "http://someonewhocares.org/hosts/" + }, + "fanboy-ultimate": { + "content": "filters", + "group": "multipurpose", + "off": true, + "title": "Fanboy+Easylist-Merged Ultimate List", + "contentURL": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", + "supportURL": "https://forums.lanik.us/" + }, + "hphosts": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "off": true, + "title": "hpHosts’ Ad and tracking servers", + "contentURL": "https://hosts-file.net/.%5Cad_servers.txt", + "supportURL": "https://hosts-file.net/" + }, + "mvps-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "off": true, + "title": "MVPS HOSTS", + "contentURL": "http://winhelp2002.mvps.org/hosts.txt", + "supportURL": "http://winhelp2002.mvps.org/" + }, + "plowe-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 13, + "title": "Peter Lowe’s Ad and tracking server list", + "contentURL": [ + "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", + "assets/thirdparties/pgl.yoyo.org/as/serverlist", + "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt" + ], + "supportURL": "https://pgl.yoyo.org/adservers/" + }, + "ara-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ara: Liste AR", + "lang": "ar", + "contentURL": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=98" + }, + "BGR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "BGR: Bulgarian Adblock list", + "lang": "bg", + "contentURL": "https://stanev.org/abp/adblock_bg.txt", + "supportURL": "https://stanev.org/abp/" + }, + "CHN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CHN: EasyList China (中文)", + "lang": "zh", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistchina.txt", + "supportURL": "http://abpchina.org/forum/forum.php" + }, + "CHN-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CHN: CJX's EasyList Lite (main focus on Chinese sites)", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" + }, + "CHN-2": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CHN: CJX's Annoyance List", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" + }, + "CZE-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CZE, SVK: EasyList Czech and Slovak", + "lang": "cs", + "contentURL": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", + "supportURL": "https://github.com/tomasko126/easylistczechandslovak" + }, + "DEU-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "DEU: EasyList Germany", + "lang": "de", + "contentURL": [ + "https://easylist.to/easylistgermany/easylistgermany.txt", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt" + ], + "supportURL": "https://forums.lanik.us/viewforum.php?f=90" + }, + "DNK-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "DNK: Schacks Adblock Plus liste", + "lang": "da", + "contentURL": "https://adblock.dk/block.csv", + "supportURL": "https://henrik.schack.dk/adblock/" + }, + "EST-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "EST: Eesti saitidele kohandatud filter", + "lang": "et", + "contentURL": "http://adblock.ee/list.php", + "supportURL": "http://adblock.ee/" + }, + "EU-prebake": { + "content": "filters", + "group": "regions", + "off": true, + "title": "EU: Prebake - Filter Obtrusive Cookie Notices", + "contentURL": "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt", + "supportURL": "https://github.com/liamja/Prebake" + }, + "FIN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "FIN: Finnish Addition to Easylist", + "lang": "fi", + "contentURL": "http://adb.juvander.net/Finland_adb.txt", + "supportURL": "http://www.juvander.fi/AdblockFinland" + }, + "FRA-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "FRA: EasyList Liste FR", + "lang": "fr", + "contentURL": "https://easylist-downloads.adblockplus.org/liste_fr.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=91" + }, + "GRC-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "GRC: Greek AdBlock Filter", + "lang": "el", + "contentURL": "https://www.void.gr/kargig/void-gr-filters.txt", + "supportURL": "https://github.com/kargig/greek-adblockplus-filter" + }, + "HUN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "HUN: hufilter", + "lang": "hu", + "contentURL": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", + "supportURL": "https://github.com/szpeter80/hufilter" + }, + "IDN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "IDN: ABPindo", + "lang": "id", + "contentURL": [ + "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt" + ], + "supportURL": "https://github.com/ABPindo/indonesianadblockrules" + }, + "ISL-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ISL: Icelandic ABP List", + "lang": "is", + "contentURL": "http://adblock.gardar.net/is.abp.txt", + "supportURL": "http://adblock.gardar.net/" + }, + "ISR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ISR: EasyList Hebrew", + "lang": "he", + "contentURL": "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt", + "supportURL": "https://github.com/easylist/EasyListHebrew" + }, + "ITA-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ITA: EasyList Italy", + "lang": "it", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=96" + }, + "ITA-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ITA: ABP X Files", + "contentURL": "https://raw.githubusercontent.com/gioxx/xfiles/master/filtri.txt", + "supportURL": "http://noads.it/" + }, + "JPN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "JPN: ABP Japanese filters (日本用フィルタ)", + "lang": "ja", + "contentURL": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", + "supportURL": "https://github.com/k2jp/abp-japanese-filters/wiki/Support_Policy" + }, + "KOR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "KOR: Korean Adblock List", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt", + "supportURL": "https://github.com/gfmaster/adblock-korea-contrib" + }, + "KOR-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "KOR: YousList", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt", + "supportURL": "https://github.com/yous/YousList" + }, + "KOR-2": { + "content": "filters", + "group": "regions", + "off": true, + "title": "KOR: Fanboy's Korean", + "contentURL": "https://www.fanboy.co.nz/fanboy-korean.txt", + "supportURL": "https://forums.lanik.us/" + }, + "LTU-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "LTU: Adblock Plus Lithuania", + "lang": "lt", + "contentURL": "http://margevicius.lt/easylistlithuania.txt", + "supportURL": "http://margevicius.lt/easylist_lithuania/" + }, + "LVA-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "LVA: Latvian List", + "lang": "lv", + "contentURL": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", + "supportURL": "https://notabug.org/latvian-list/adblock-latvian" + }, + "NLD-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "NLD: EasyList Dutch", + "lang": "nl", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=100" + }, + "POL-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "POL: polskie filtry do Adblocka i uBlocka", + "lang": "pl", + "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", + "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" + }, + "RUS-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: RU AdList (Дополнительная региональная подписка)", + "lang": "ru", + "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: BitBlock List (Дополнительная подписка фильтров)", + "contentURL": "https://easylist-downloads.adblockplus.org/bitblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-2": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: Adguard Russian Filter", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/1.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?69-%D0%A4%D0%B8%D0%BB%D1%8C%D1%82%D1%80%D1%8B-Adguard" + }, + "spa-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "spa: EasyList Spanish", + "lang": "es", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=103" + }, + "SVN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "SVN: Slovenian List", + "lang": "sl", + "contentURL": "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt", + "supportURL": "https://github.com/betterwebleon/slovenian-list" + }, + "SWE-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "SWE: Fanboy's Swedish", + "lang": "sv", + "contentURL": "https://www.fanboy.co.nz/fanboy-swedish.txt", + "supportURL": "https://forums.lanik.us/" + }, + "TUR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "TUR: Adguard Turkish Filter", + "lang": "tr", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/13.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?51-Filter-Rules" + }, + "VIE-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "VIE: Fanboy's Vietnamese", + "lang": "vi", + "contentURL": "https://www.fanboy.co.nz/fanboy-vietnam.txt", + "supportURL": "https://forums.lanik.us/" + } +} diff --git a/src/3p-filters.html b/src/3p-filters.html index 8bce9d01a..48086f71f 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -40,16 +40,10 @@

-
-
- -
-
- diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index b31059c90..653cf75c4 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -580,8 +580,8 @@ "description": "Message asking user to confirm reset" }, "errorCantConnectTo":{ - "message":"Unable to connect to {{url}}", - "description":"English: Network error: unable to connect to {{url}}" + "message":"Network error: {{msg}}", + "description":"English: Network error: {{msg}}" }, "subscriberConfirm":{ "message":"uBlock₀: Add the following URL to your custom filter lists?\n\nTitle: \"{{title}}\"\nURL: {{url}}", diff --git a/src/background.html b/src/background.html index a8463068e..0d56942fc 100644 --- a/src/background.html +++ b/src/background.html @@ -8,7 +8,6 @@ - diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index 99f87cfd4..8aa6760f9 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -1,3 +1,7 @@ +@keyframes spin { + 100% { transform: rotate(360deg); -webkit-transform: rotate(360deg); } + } + ul { padding: 0; list-style-type: none; @@ -88,7 +92,7 @@ body[dir=rtl] #buttonApply { span.status { border: 1px solid transparent; color: #444; - display: inline-block; + display: none; font-size: smaller; line-height: 1; margin: 0 0 0 0.5em; @@ -99,6 +103,16 @@ span.unsecure { background-color: hsl(0, 100%, 88%); border-color: hsl(0, 100%, 83%); } +li.listEntry.unsecure span.unsecure { + display: inline; + } +span.obsolete { + background-color: hsl(36, 100%, 80%); + border-color: hsl(36, 100%, 75%); + } +li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { + display: inline; + } span.purge { border-color: #ddd; background-color: #eee; @@ -107,10 +121,16 @@ span.purge { span.purge:hover { opacity: 1; } -span.obsolete, -span.new { - background-color: hsl(36, 100%, 80%); - border-color: hsl(36, 100%, 75%); +li.listEntry.cached span.purge { + display: inline; + } +span.updating { + border: none; + padding: 0; + } +li.listEntry.updating span.updating { + animation: spin 2s linear infinite; + display: inline-block; } #externalListsDiv { margin: 2em auto 0 2em; @@ -125,64 +145,3 @@ body[dir=rtl] #externalListsDiv { width: 100%; word-wrap: normal; } -body #busyOverlay { - background-color: transparent; - bottom: 0; - cursor: wait; - display: none; - left: 0; - position: fixed; - right: 0; - top: 0; - z-index: 1000; - } -body.busy #busyOverlay { - display: block; - } -#busyOverlay > div:nth-of-type(1) { - background-color: white; - bottom: 0; - left: 0; - opacity: 0.75; - position: absolute; - right: 0; - top: 0; - } -#busyOverlay > div:nth-of-type(2) { - background-color: #eee; - border: 1px solid transparent; - border-color: #80b3ff #80b3ff hsl(216, 100%, 75%); - border-radius: 3px; - box-sizing: border-box; - height: 3em; - left: 10%; - position: absolute; - bottom: 75%; - width: 80%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(1) { - background-color: hsl(216, 100%, 75%); - background-image: linear-gradient(#a8cbff, #80b3ff); - background-repeat: repeat-x; - border: 0; - box-sizing: border-box; - color: #222; - height: 100%; - left: 0; - padding: 0; - position: absolute; - width: 25%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(2) { - background-color: transparent; - border: 0; - box-sizing: border-box; - height: 100%; - left: 0; - line-height: 3em; - overflow: hidden; - position: absolute; - text-align: center; - top: 0; - width: 100%; - } diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 0b960e81c..4b7dfa1b9 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,38 +21,30 @@ /* global uDom */ -/******************************************************************************/ - -(function() { - 'use strict'; /******************************************************************************/ -var userListName = vAPI.i18n('1pPageName'); +(function() { + +/******************************************************************************/ + var listDetails = {}; var parseCosmeticFilters = true; var ignoreGenericCosmeticFilters = false; +var selectedListsHashBefore = ''; var externalLists = ''; -var cacheWasPurged = false; -var needUpdate = false; -var hasCachedContent = false; /******************************************************************************/ var onMessage = function(msg) { switch ( msg.what ) { + case 'assetUpdated': + updateAssetStatus(msg); + break; case 'staticFilteringDataChanged': renderFilterLists(); break; - - case 'forceUpdateAssetsProgress': - renderBusyOverlay(true, msg.progress); - if ( msg.done ) { - messaging.send('dashboard', { what: 'reloadAllFilters' }); - } - break; - default: break; } @@ -69,20 +61,15 @@ var renderNumber = function(value) { /******************************************************************************/ -// TODO: get rid of background page dependencies - var renderFilterLists = function() { - var listGroupTemplate = uDom('#templates .groupEntry'); - var listEntryTemplate = uDom('#templates .listEntry'); - var listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'); - var renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString; - var lastUpdateString = vAPI.i18n('3pLastUpdate'); + var listGroupTemplate = uDom('#templates .groupEntry'), + listEntryTemplate = uDom('#templates .listEntry'), + listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'), + renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString, + lastUpdateString = vAPI.i18n('3pLastUpdate'); - // Assemble a pretty blacklist name if possible + // Assemble a pretty list name if possible var listNameFromListKey = function(listKey) { - if ( listKey === listDetails.userFiltersPath ) { - return userListName; - } var list = listDetails.current[listKey] || listDetails.available[listKey]; var listTitle = list ? list.title : ''; if ( listTitle === '' ) { @@ -91,73 +78,68 @@ var renderFilterLists = function() { return listTitle; }; - var liFromListEntry = function(listKey) { + var liFromListEntry = function(listKey, li) { var entry = listDetails.available[listKey]; - var li = listEntryTemplate.clone(); - + li = li ? li : listEntryTemplate.clone().nodeAt(0); + li.setAttribute('data-listkey', listKey); + var elem = li.querySelector('input[type="checkbox"]'); if ( entry.off !== true ) { - li.descendants('input').attr('checked', ''); + elem.setAttribute('checked', ''); + } else { + elem.removeAttribute('checked'); } - - var elem = li.descendants('a:nth-of-type(1)'); - elem.attr('href', 'asset-viewer.html?url=' + encodeURI(listKey)); - elem.attr('type', 'text/html'); - elem.attr('data-listkey', listKey); - elem.text(listNameFromListKey(listKey) + '\u200E'); - + elem = li.querySelector('a:nth-of-type(1)'); + elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); + elem.setAttribute('type', 'text/html'); + elem.textContent = listNameFromListKey(listKey) + '\u200E'; + elem = li.querySelector('a:nth-of-type(2)'); if ( entry.instructionURL ) { - elem = li.descendants('a:nth-of-type(2)'); - elem.attr('href', entry.instructionURL); - elem.css('display', ''); + elem.setAttribute('href', entry.instructionURL); + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); } - + elem = li.querySelector('a:nth-of-type(3)'); if ( entry.supportName ) { - elem = li.descendants('a:nth-of-type(3)'); - elem.attr('href', entry.supportURL); - elem.text('(' + entry.supportName + ')'); - elem.css('display', ''); + elem.setAttribute('href', entry.supportURL); + elem.textContent = '(' + entry.supportName + ')'; + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); } - - elem = li.descendants('span.counts'); + elem = li.querySelector('span.counts'); var text = listStatsTemplate .replace('{{used}}', renderNumber(!entry.off && !isNaN(+entry.entryUsedCount) ? entry.entryUsedCount : 0)) .replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?'); - elem.text(text); - - // https://github.com/gorhill/uBlock/issues/78 - // Badge for non-secure connection - var remoteURL = listKey; - if ( remoteURL.lastIndexOf('http:', 0) !== 0 ) { - remoteURL = entry.homeURL || ''; - } - if ( remoteURL.lastIndexOf('http:', 0) === 0 ) { - li.descendants('span.status.unsecure').css('display', ''); - } + elem.textContent = text; // https://github.com/chrisaljoudi/uBlock/issues/104 var asset = listDetails.cache[listKey] || {}; + // https://github.com/gorhill/uBlock/issues/78 + // Badge for non-secure connection + var remoteURL = asset.remoteURL; + li.classList.toggle( + 'unsecure', + typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 + ); // Badge for update status - if ( entry.off !== true ) { - if ( asset.repoObsolete ) { - li.descendants('span.status.new').css('display', ''); - needUpdate = true; - } else if ( asset.cacheObsolete ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } else if ( entry.external && !asset.cached ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } - } - - // In cache + li.classList.toggle( + 'obsolete', + entry.off !== true && asset.obsolete === true + ); + // Badge for cache status + li.classList.toggle( + 'cached', + asset.cached === true && asset.writeTime > 0 + ); if ( asset.cached ) { - elem = li.descendants('span.status.purge'); - elem.css('display', ''); - elem.attr('title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.lastModified))); - hasCachedContent = true; + li.querySelector('.status.purge').setAttribute( + 'title', + lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) + ); } + li.classList.remove('discard'); return li; }; @@ -176,27 +158,31 @@ var renderFilterLists = function() { }; var liFromListGroup = function(groupKey, listKeys) { - var liGroup = listGroupTemplate.clone(); - var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); - if ( groupName !== '' ) { - liGroup.descendants('span.geName').text(groupName); - liGroup.descendants('span.geCount').text(listEntryCountFromGroup(listKeys)); + var liGroup = document.querySelector('#lists > .groupEntry[data-groupkey="' + groupKey + '"]'); + if ( liGroup === null ) { + liGroup = listGroupTemplate.clone().nodeAt(0); + var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); + if ( groupName !== '' ) { + liGroup.querySelector('.geName').textContent = groupName; + } } - var ulGroup = liGroup.descendants('ul'); - if ( !listKeys ) { - return liGroup; + if ( liGroup.querySelector('.geName:empty') === null ) { + liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); } + var ulGroup = liGroup.querySelector('.listEntries'); + if ( !listKeys ) { return liGroup; } listKeys.sort(function(a, b) { return (listDetails.available[a].title || '').localeCompare(listDetails.available[b].title || ''); }); for ( var i = 0; i < listKeys.length; i++ ) { - ulGroup.append(liFromListEntry(listKeys[i])); + var liEntry = liFromListEntry(listKeys[i], ulGroup.children[i]); + if ( liEntry.parentElement === null ) { + ulGroup.appendChild(liEntry); + } } return liGroup; }; - // https://www.youtube.com/watch?v=unCVi4hYRlY#t=30m18s - var groupsFromLists = function(lists) { var groups = {}; var listKeys = Object.keys(lists); @@ -219,14 +205,16 @@ var renderFilterLists = function() { listDetails = details; parseCosmeticFilters = details.parseCosmeticFilters; ignoreGenericCosmeticFilters = details.ignoreGenericCosmeticFilters; - needUpdate = false; - hasCachedContent = false; + + // Incremental rendering: this will allow us to easily discard unused + // DOM list entries. + uDom('#lists .listEntries .listEntry').addClass('discard'); // Visually split the filter lists in purpose-based groups - var ulLists = uDom('#lists').empty(), liGroup; - var groups = groupsFromLists(details.available); - var groupKey, i; - var groupKeys = [ + var ulLists = document.querySelector('#lists'), + groups = groupsFromLists(details.available), + liGroup, i, groupKey, + groupKeys = [ 'default', 'ads', 'privacy', @@ -239,31 +227,44 @@ var renderFilterLists = function() { for ( i = 0; i < groupKeys.length; i++ ) { groupKey = groupKeys[i]; liGroup = liFromListGroup(groupKey, groups[groupKey]); - liGroup.toggleClass( + liGroup.setAttribute('data-groupkey', groupKey); + liGroup.classList.toggle( 'collapsed', vAPI.localStorage.getItem('collapseGroup' + (i + 1)) === 'y' ); - ulLists.append(liGroup); + if ( liGroup.parentElement === null ) { + ulLists.appendChild(liGroup); + } delete groups[groupKey]; } // For all groups not covered above (if any left) groupKeys = Object.keys(groups); for ( i = 0; i < groupKeys.length; i++ ) { groupKey = groupKeys[i]; - ulLists.append(liFromListGroup(groupKey, groups[groupKey])); + ulLists.appendChild(liFromListGroup(groupKey, groups[groupKey])); } + uDom('#lists .listEntries .listEntry.discard').remove(); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); + uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); + uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); + uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); uDom('#listsOfBlockedHostsPrompt').text( vAPI.i18n('3pListsOfBlockedHostsPrompt') .replace('{{netFilterCount}}', renderNumber(details.netFilterCount)) .replace('{{cosmeticFilterCount}}', renderNumber(details.cosmeticFilterCount)) ); - uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); - uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); - uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); + + // Compute a hash of the lists currently enabled in memory. + var selectedListsBefore = []; + for ( var key in listDetails.current ) { + if ( listDetails.current[key].off !== true ) { + selectedListsBefore.push(key); + } + } + selectedListsHashBefore = selectedListsBefore.sort().join(); renderWidgets(); - renderBusyOverlay(details.manualUpdate, details.manualUpdateProgress); }; messaging.send('dashboard', { what: 'getLists' }, onListsReceived); @@ -271,33 +272,22 @@ var renderFilterLists = function() { /******************************************************************************/ -// Progress must be normalized to [0, 1], or can be undefined. - -var renderBusyOverlay = function(state, progress) { - progress = progress || {}; - var showProgress = typeof progress.value === 'number'; - if ( showProgress ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:first-child').css( - 'width', - (progress.value * 100).toFixed(1) + '%' - ); - var text = progress.text || ''; - if ( text !== '' ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:last-child').text(text); - } - } - uDom('#busyOverlay > div:nth-of-type(2)').css('display', showProgress ? '' : 'none'); - uDom('body').toggleClass('busy', !!state); -}; - -/******************************************************************************/ - // This is to give a visual hint that the selection of blacklists has changed. var renderWidgets = function() { uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); - uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged()); - uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent); + uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); +}; + +/******************************************************************************/ + +var updateAssetStatus = function(details) { + var li = uDom('#lists .listEntry[data-listkey="' + details.key + '"]'); + li.toggleClass('obsolete', !details.cached); + li.toggleClass('cached', details.cached); + li.removeClass('updating'); + renderWidgets(); }; /******************************************************************************/ @@ -307,98 +297,49 @@ var renderWidgets = function() { var listsSelectionChanged = function() { if ( listDetails.parseCosmeticFilters !== parseCosmeticFilters || - listDetails.parseCosmeticFilters && listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters + listDetails.parseCosmeticFilters && + listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters ) { return true; } - - if ( cacheWasPurged ) { - return true; + var selectedListsAfter = [], + listEntries = uDom('#lists .listEntry[data-listkey] > input[type="checkbox"]:checked'); + for ( var i = 0, n = listEntries.length; i < n; i++ ) { + selectedListsAfter.push(listEntries.at(i).ancestors('.listEntry[data-listkey]').attr('data-listkey')); } - var availableLists = listDetails.available; - var currentLists = listDetails.current; - var location, availableOff, currentOff; - - // This check existing entries - for ( location in availableLists ) { - if ( availableLists.hasOwnProperty(location) === false ) { - continue; - } - availableOff = availableLists[location].off === true; - currentOff = currentLists[location] === undefined || currentLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } - } - - // This check removed entries - for ( location in currentLists ) { - if ( currentLists.hasOwnProperty(location) === false ) { - continue; - } - currentOff = currentLists[location].off === true; - availableOff = availableLists[location] === undefined || availableLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } - } - - return false; -}; - -/******************************************************************************/ - -// Return whether content need update. - -var listsContentChanged = function() { - return needUpdate; + return selectedListsHashBefore !== selectedListsAfter.sort().join(); }; /******************************************************************************/ var onListCheckboxChanged = function() { - var href = uDom(this).parent().descendants('a').first().attr('data-listkey'); - if ( typeof href !== 'string' ) { - return; - } - if ( listDetails.available[href] === undefined ) { - return; - } - listDetails.available[href].off = !this.checked; renderWidgets(); }; /******************************************************************************/ var onPurgeClicked = function() { - var button = uDom(this); - var li = button.parent(); - var href = li.descendants('a').first().attr('data-listkey'); - if ( !href ) { - return; - } + var button = uDom(this), + liEntry = button.ancestors('[data-listkey]'), + listKey = liEntry.attr('data-listkey'); + if ( !listKey ) { return; } - messaging.send('dashboard', { what: 'purgeCache', path: href }); - button.remove(); + messaging.send('dashboard', { what: 'purgeCache', assetKey: listKey }); // If the cached version is purged, the installed version must be assumed // to be obsolete. // https://github.com/gorhill/uBlock/issues/1733 // An external filter list must not be marked as obsolete, they will always // be fetched anyways if there is no cached copy. - var entry = listDetails.current && listDetails.current[href]; - if ( entry && entry.off !== true && /^[a-z]+:\/\//.test(href) === false ) { - if ( typeof entry.homeURL !== 'string' || entry.homeURL === '' ) { - li.descendants('span.status.new').css('display', ''); - } else { - li.descendants('span.status.obsolete').css('display', ''); - } - needUpdate = true; + var entry = listDetails.current && listDetails.current[listKey]; + if ( entry && entry.off !== true ) { + liEntry.addClass('obsolete'); + uDom('#buttonUpdate').removeClass('disabled'); } + liEntry.removeClass('cached'); - if ( li.descendants('input').first().prop('checked') ) { - cacheWasPurged = true; + if ( liEntry.descendants('input').first().prop('checked') ) { renderWidgets(); } }; @@ -419,22 +360,21 @@ var selectFilterLists = function(callback) { }); // Filter lists - var switches = []; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + var listKeys = [], + liEntries = uDom('#lists .listEntry'), liEntry, + i = liEntries.length; while ( i-- ) { - li = lis.at(i); - switches.push({ - location: li.descendants('a').attr('data-listkey'), - off: li.descendants('input').prop('checked') === false - }); + liEntry = liEntries.at(i); + if ( liEntry.descendants('input').first().prop('checked') ) { + listKeys.push(liEntry.attr('data-listkey')); + } } messaging.send( 'dashboard', { what: 'selectFilterLists', - switches: switches + keys: listKeys }, callback ); @@ -444,49 +384,34 @@ var selectFilterLists = function(callback) { var buttonApplyHandler = function() { uDom('#buttonApply').removeClass('enabled'); - - renderBusyOverlay(true); - var onSelectionDone = function() { messaging.send('dashboard', { what: 'reloadAllFilters' }); }; - selectFilterLists(onSelectionDone); - - cacheWasPurged = false; }; /******************************************************************************/ var buttonUpdateHandler = function() { - uDom('#buttonUpdate').removeClass('enabled'); - - if ( needUpdate ) { - renderBusyOverlay(true); - - var onSelectionDone = function() { - messaging.send('dashboard', { what: 'forceUpdateAssets' }); - }; - - selectFilterLists(onSelectionDone); - - cacheWasPurged = false; - } + var onSelectionDone = function() { + uDom('#lists .listEntry.obsolete').addClass('updating'); + messaging.send('dashboard', { what: 'forceUpdateAssets' }); + }; + selectFilterLists(onSelectionDone); }; /******************************************************************************/ -var buttonPurgeAllHandler = function() { +var buttonPurgeAllHandler = function(ev) { uDom('#buttonPurgeAll').removeClass('enabled'); - - renderBusyOverlay(true); - - var onCompleted = function() { - cacheWasPurged = true; - renderFilterLists(); - }; - - messaging.send('dashboard', { what: 'purgeAllCaches' }, onCompleted); + messaging.send( + 'dashboard', + { + what: 'purgeAllCaches', + hard: ev.ctrlKey && ev.shiftKey + }, + renderFilterLists + ); }; /******************************************************************************/ @@ -562,7 +487,7 @@ var groupEntryClickHandler = function() { /******************************************************************************/ -var getCloudData = function() { +var toCloudData = function() { var bin = { parseCosmeticFilters: uDom.nodeFromId('parseCosmeticFilters').checked, ignoreGenericCosmeticFilters: uDom.nodeFromId('ignoreGenericCosmeticFilters').checked, @@ -570,24 +495,22 @@ var getCloudData = function() { externalLists: externalLists }; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + var liEntries = uDom('#lists .listEntry'), liEntry; + var i = liEntries.length; while ( i-- ) { - li = lis.at(i); - if ( li.descendants('input').prop('checked') ) { - bin.selectedLists.push(li.descendants('a').attr('data-listkey')); + liEntry = liEntries.at(i); + if ( liEntry.descendants('input').prop('checked') ) { + bin.selectedLists.push(liEntry.attr('data-listkey')); } } return bin; }; -var setCloudData = function(data, append) { - if ( typeof data !== 'object' || data === null ) { - return; - } +var fromCloudData = function(data, append) { + if ( typeof data !== 'object' || data === null ) { return; } - var elem, checked; + var elem, checked, i, n; elem = uDom.nodeFromId('parseCosmeticFilters'); checked = data.parseCosmeticFilters === true || append && elem.checked; @@ -597,30 +520,34 @@ var setCloudData = function(data, append) { checked = data.ignoreGenericCosmeticFilters === true || append && elem.checked; elem.checked = listDetails.ignoreGenericCosmeticFilters = checked; - var lis = uDom('#lists .listEntry'), li, listKey; - var i = lis.length; - while ( i-- ) { - li = lis.at(i); - elem = li.descendants('input'); - listKey = li.descendants('a').attr('data-listkey'); - checked = data.selectedLists.indexOf(listKey) !== -1 || - append && elem.prop('checked'); - elem.prop('checked', checked); - listDetails.available[listKey].off = !checked; + var listKey; + for ( i = 0, n = data.selectedLists.length; i < n; i++ ) { + listKey = data.selectedLists[i]; + if ( listDetails.aliases[listKey] ) { + data.selectedLists[i] = listDetails.aliases[listKey]; + } + } + var selectedSet = new Set(data.selectedLists), + listEntries = uDom('#lists .listEntry'), + listEntry, input; + for ( i = 0, n = listEntries.length; i < n; i++ ) { + listEntry = listEntries.at(i); + listKey = listEntry.attr('data-listkey'); + input = listEntry.descendants('input').first(); + if ( append && input.prop('checked') ) { continue; } + input.prop('checked', selectedSet.has(listKey) ); } elem = uDom.nodeFromId('externalLists'); - if ( !append ) { - elem.value = ''; - } + if ( !append ) { elem.value = ''; } elem.value += data.externalLists || ''; renderWidgets(); externalListsChangeHandler(); }; -self.cloud.onPush = getCloudData; -self.cloud.onPull = setCloudData; +self.cloud.onPush = toCloudData; +self.cloud.onPull = fromCloudData; /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index 88909948b..f4daf8f28 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,311 +19,54 @@ Home: https://github.com/gorhill/uBlock */ -/* global YaMD5 */ - 'use strict'; -/******************************************************************************* - -File system structure: - assets - ublock - ... - thirdparties - ... - user - filters.txt - ... - -*/ - /******************************************************************************/ -// Low-level asset files manager - µBlock.assets = (function() { /******************************************************************************/ -var oneSecond = 1000; -var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -var oneDay = 24 * oneHour; +var reIsExternalPath = /^(?:[a-z-]+):\/\//, + reIsUserAsset = /^user-/, + errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); -/******************************************************************************/ - -var projectRepositoryRoot = µBlock.projectServerRoot; -var assetsRepositoryRoot = 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/'; -var nullFunc = function() {}; -var reIsExternalPath = /^(file|ftps?|https?|resource):\/\//; -var reIsUserPath = /^assets\/user\//; -var reIsCachePath = /^cache:\/\//; -var lastRepoMetaTimestamp = 0; -var lastRepoMetaIsRemote = false; -var refreshRepoMetaPeriod = 5 * oneHour; -var errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); -var xhrTimeout = vAPI.localStorage.getItem('xhrTimeout') || 30000; -var onAssetRemovedListener = null; - -var exports = { - autoUpdate: true, - autoUpdateDelay: 4 * oneDay, - - // https://github.com/chrisaljoudi/uBlock/issues/426 - remoteFetchBarrier: 0 +var api = { }; /******************************************************************************/ -var AssetEntry = function() { - this.localChecksum = ''; - this.repoChecksum = ''; - this.expireTimestamp = 0; +var observers = []; + +api.addObserver = function(observer) { + if ( observers.indexOf(observer) === -1 ) { + observers.push(observer); + } }; -var RepoMetadata = function() { - this.entries = {}; - this.waiting = []; +api.removeObserver = function(observer) { + var pos; + while ( (pos = observers.indexOf(observer)) !== -1 ) { + observers.splice(pos, 1); + } }; -var repoMetadata = null; - -// We need these to persist beyond repoMetaData -var homeURLs = {}; - -/******************************************************************************/ - -var stringIsNotEmpty = function(s) { - return typeof s === 'string' && s !== ''; -}; - -/******************************************************************************/ - -var cacheIsObsolete = function(t) { - return typeof t !== 'number' || (Date.now() - t) >= exports.autoUpdateDelay; -}; - -/******************************************************************************/ - -var cachedAssetsManager = (function() { - var exports = {}; - var entries = null; - var cachedAssetPathPrefix = 'cached_asset_content://'; - - var getEntries = function(callback) { - if ( entries !== null ) { - callback(entries); - return; +var fireNotification = function(topic, details) { + var result; + for ( var i = 0; i < observers.length; i++ ) { + if ( observers[i](topic, details) === false ) { + result = false; } - // Flush cached non-user assets if these are from a prior version. - // https://github.com/gorhill/httpswitchboard/issues/212 - var onLastVersionRead = function(store) { - var currentVersion = vAPI.app.version; - var lastVersion = store.extensionLastVersion || '0.0.0.0'; - if ( currentVersion !== lastVersion ) { - vAPI.cacheStorage.set({ 'extensionLastVersion': currentVersion }); - } - callback(entries); - }; - var onLoaded = function(bin) { - // https://github.com/gorhill/httpswitchboard/issues/381 - // Maybe the index was requested multiple times and already - // fetched by one of the occurrences. - if ( entries === null ) { - var lastError = vAPI.lastError(); - if ( lastError ) { - console.error( - 'µBlock> cachedAssetsManager> getEntries():', - lastError.message - ); - } - entries = bin.cached_asset_entries || {}; - } - vAPI.cacheStorage.get('extensionLastVersion', onLastVersionRead); - }; - vAPI.cacheStorage.get('cached_asset_entries', onLoaded); - }; - exports.entries = getEntries; - - exports.load = function(path, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - 'path': path, - 'content': '' - }; - var cachedContentPath = cachedAssetPathPrefix + path; - var onLoaded = function(bin) { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µBlock> cachedAssetsManager.load():', details.error); - cbError(details); - return; - } - // Not sure how this can happen, but I've seen it happen. It could - // be because the save occurred while I was stepping in the code - // though, which means it would not occur during normal operation. - // Still, just to be safe. - if ( stringIsNotEmpty(bin[cachedContentPath]) === false ) { - exports.remove(path); - details.error = 'Error: not found'; - cbError(details); - return; - } - details.content = bin[cachedContentPath]; - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries[path] === undefined ) { - details.error = 'Error: not found'; - cbError(details); - return; - } - vAPI.cacheStorage.get(cachedContentPath, onLoaded); - }; - getEntries(onEntries); - }; - - exports.save = function(path, content, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - path: path, - content: content - }; - if ( content === '' ) { - exports.remove(path); - cbSuccess(details); - return; - } - var cachedContentPath = cachedAssetPathPrefix + path; - var bin = {}; - bin[cachedContentPath] = content; - var removedItems = []; - var onSaved = function() { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µBlock> cachedAssetsManager.save():', details.error); - cbError(details); - return; - } - // Saving over an existing item must be seen as removing an - // existing item and adding a new one. - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener(removedItems); - } - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries.hasOwnProperty(path) ) { - removedItems.push(path); - } - entries[path] = Date.now(); - bin.cached_asset_entries = entries; - vAPI.cacheStorage.set(bin, onSaved); - }; - getEntries(onEntries); - }; - - exports.remove = function(pattern, before) { - var onEntries = function(entries) { - var keystoRemove = []; - var removedItems = []; - var paths = Object.keys(entries); - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( typeof pattern === 'string' && path !== pattern ) { - continue; - } - if ( pattern instanceof RegExp && !pattern.test(path) ) { - continue; - } - if ( typeof before === 'number' && entries[path] >= before ) { - continue; - } - removedItems.push(path); - keystoRemove.push(cachedAssetPathPrefix + path); - delete entries[path]; - } - if ( keystoRemove.length ) { - vAPI.cacheStorage.remove(keystoRemove); - vAPI.cacheStorage.set({ 'cached_asset_entries': entries }); - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener(removedItems); - } - } - }; - getEntries(onEntries); - }; - - exports.removeAll = function(callback) { - var onEntries = function() { - // Careful! do not remove 'assets/user/' - exports.remove(/^https?:\/\/[a-z0-9]+/); - exports.remove(/^assets\/(ublock|thirdparties)\//); - exports.remove(/^cache:\/\//); - exports.remove('assets/checksums.txt'); - if ( typeof callback === 'function' ) { - callback(null); - } - }; - getEntries(onEntries); - }; - - exports.rmrf = function() { - exports.remove(/./); - }; - - exports.exists = function(path) { - return entries !== null && entries.hasOwnProperty(path); - }; - - getEntries(function(){}); - - return exports; -})(); - -/******************************************************************************/ - -var toRepoURL = function(path) { - if ( path.startsWith('assets/ublock/filter-lists.json') ) { - return projectRepositoryRoot + path; } - - if ( path.startsWith('assets/checksums.txt') ) { - return path.replace( - /^assets\/checksums.txt/, - assetsRepositoryRoot + 'checksums/ublock0.txt' - ); - } - - if ( path.startsWith('assets/thirdparties/') ) { - return path.replace( - /^assets\/thirdparties\//, - assetsRepositoryRoot + 'thirdparties/' - ); - } - - if ( path.startsWith('assets/ublock/') ) { - return path.replace( - /^assets\/ublock\//, - assetsRepositoryRoot + 'filters/' - ); - } - - // At this point, `path` is assumed to point to a resource specific to - // this project. - return projectRepositoryRoot + path; + return result; }; /******************************************************************************/ var getTextFileFromURL = function(url, onLoad, onError) { - // console.log('µBlock.assets/getTextFileFromURL("%s"):', url); + if ( reIsExternalPath.test(url) === false ) { + url = vAPI.getURL(url); + } if ( typeof onError !== 'function' ) { onError = onLoad; @@ -353,6 +96,7 @@ var getTextFileFromURL = function(url, onLoad, onError) { var onErrorReceived = function() { this.onload = this.onerror = this.ontimeout = null; + µBlock.logger.writeOne('', 'error', errorCantConnectTo.replace('{{msg}}', '')); onError.call(this); }; @@ -362,7 +106,7 @@ var getTextFileFromURL = function(url, onLoad, onError) { var xhr = new XMLHttpRequest(); try { xhr.open('get', url, true); - xhr.timeout = xhrTimeout; + xhr.timeout = µBlock.hiddenSettings.assetFetchTimeout * 1000 || 30000; xhr.onload = onResponseReceived; xhr.onerror = onErrorReceived; xhr.ontimeout = onErrorReceived; @@ -373,686 +117,578 @@ var getTextFileFromURL = function(url, onLoad, onError) { } }; -/******************************************************************************/ +/******************************************************************************* -var updateLocalChecksums = function() { - var localChecksums = []; - var entries = repoMetadata.entries; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entry = entries[path]; - if ( entry.localChecksum !== '' ) { - localChecksums.push(entry.localChecksum + ' ' + path); - } - } - cachedAssetsManager.save('assets/checksums.txt', localChecksums.join('\n')); + TODO(seamless migration): + This block of code will be removed when I am confident all users have + moved to a version of uBO which does not require the old way of caching + assets. + + api.listKeyAliases: a map of old asset keys to new asset keys. + + migrate(): to seamlessly migrate the old cache manager to the new one: + - attempt to preserve and move content of cached assets to new locations; + - removes all traces of now obsolete cache manager entries in cacheStorage. + + This code will typically execute only once, when the newer version of uBO + is first installed and executed. + +**/ + +api.listKeyAliases = { + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", + "assets/user/filters.txt": "user-filters", + "assets/ublock/resources.txt": "ublock-resources", + "assets/ublock/filters.txt": "ublock-filters", + "assets/ublock/privacy.txt": "ublock-privacy", + "assets/ublock/unbreak.txt": "ublock-unbreak", + "assets/ublock/badware.txt": "ublock-badware", + "assets/ublock/experimental.txt": "ublock-experimental", + "https://easylist-downloads.adblockplus.org/easylistchina.txt": "CHN-0", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "CHN-1", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt": "CHN-2", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt": "DEU-0", + "https://adblock.dk/block.csv": "DNK-0", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt": "easylist", + "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "easylist-nocosmetic", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt": "easyprivacy", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "fanboy-annoyance", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt": "fanboy-social", + "https://easylist-downloads.adblockplus.org/liste_fr.txt": "FRA-0", + "http://adblock.gardar.net/is.abp.txt": "ISL-0", + "https://easylist-downloads.adblockplus.org/easylistitaly.txt": "ITA-0", + "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "ITA-1", + "https://easylist-downloads.adblockplus.org/advblock.txt": "RUS-0", + "https://easylist-downloads.adblockplus.org/bitblock.txt": "RUS-1", + "https://filters.adtidy.org/extension/chromium/filters/1.txt": "RUS-2", + "https://adguard.com/en/filter-rules.html?id=1": "RUS-2", + "https://easylist-downloads.adblockplus.org/easylistdutch.txt": "NLD-0", + "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt": "LVA-0", + "http://hosts-file.net/.%5Cad_servers.txt": "hphosts", + "http://adblock.ee/list.php": "EST-0", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt": "disconnect-malvertising", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt": "disconnect-malware", + "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt": "disconnect-tracking", + "https://www.certyficate.it/adblock/adblock.txt": "POL-0", + "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt": "awrl-0", + "http://adb.juvander.net/Finland_adb.txt": "FIN-0", + "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt": "KOR-0", + "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt": "KOR-1", + "https://www.fanboy.co.nz/fanboy-korean.txt": "KOR-2", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0", + "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt": "JPN-0", + "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake", + "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0", + "http://margevicius.lt/easylistlithuania.txt": "LTU-0", + "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-0", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-1", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-2", + "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", + "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0", + "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0", + "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "HUN-0", + "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "CZE-0", + "http://someonewhocares.org/hosts/hosts": "dpollock-0", + "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt": "spam404-0", + "http://stanev.org/abp/adblock_bg.txt": "BGR-0", + "http://winhelp2002.mvps.org/hosts.txt": "mvps-0", + "https://www.fanboy.co.nz/enhancedstats.txt": "fanboy-enhanced", + "https://www.fanboy.co.nz/fanboy-antifacebook.txt": "fanboy-thirdparty_social", + "https://easylist-downloads.adblockplus.org/easylistspanish.txt": "spa-0", + "https://www.fanboy.co.nz/fanboy-swedish.txt": "SWE-0", + "https://www.fanboy.co.nz/r/fanboy-ultimate.txt": "fanboy-ultimate", + "https://filters.adtidy.org/extension/chromium/filters/13.txt": "TUR-0", + "https://adguard.com/filter-rules.html?id=13": "TUR-0", + "https://www.fanboy.co.nz/fanboy-vietnam.txt": "VIE-0", + "https://www.void.gr/kargig/void-gr-filters.txt": "GRC-0", + "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt": "SVN-0" }; -/******************************************************************************/ +var migrate = function(callback) { + var entries, + moveCount = 0, + toRemove = []; -// Gather meta data of all assets. + var countdown = function(change) { + moveCount -= (change || 0); + if ( moveCount !== 0 ) { return; } + vAPI.cacheStorage.remove(toRemove); + saveAssetCacheRegistry(); + callback(); + }; -var getRepoMetadata = function(callback) { - callback = callback || nullFunc; - - // https://github.com/chrisaljoudi/uBlock/issues/515 - // Handle re-entrancy here, i.e. we MUST NOT tamper with the waiting list - // of callers, if any, except to add one at the end of the list. - if ( repoMetadata !== null && repoMetadata.waiting.length !== 0 ) { - repoMetadata.waiting.push(callback); - return; - } - - if ( exports.remoteFetchBarrier === 0 && lastRepoMetaIsRemote === false ) { - lastRepoMetaTimestamp = 0; - } - if ( (Date.now() - lastRepoMetaTimestamp) >= refreshRepoMetaPeriod ) { - repoMetadata = null; - } - if ( repoMetadata !== null ) { - callback(repoMetadata); - return; - } - - lastRepoMetaTimestamp = Date.now(); - lastRepoMetaIsRemote = exports.remoteFetchBarrier === 0; - - var defaultChecksums; - var localChecksums; - var repoChecksums; - - var checksumsReceived = function() { - if ( - defaultChecksums === undefined || - localChecksums === undefined || - repoChecksums === undefined - ) { - return; - } - // Remove from cache assets which no longer exist in the repo - var entries = repoMetadata.entries; - var checksumsChanged = false; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; + var onContentRead = function(oldKey, newKey, bin) { + var content = bin && bin['cached_asset_content://' + oldKey] || undefined; + if ( content ) { + assetCacheRegistry[newKey] = { + readTime: Date.now(), + writeTime: entries[oldKey] + }; + if ( reIsExternalPath.test(oldKey) ) { + assetCacheRegistry[newKey].remoteURL = oldKey; } - entry = entries[path]; - // https://github.com/gorhill/uBlock/issues/760 - // If the resource does not have a cached instance, we must reset - // the checksum to its value at install time. - if ( - stringIsNotEmpty(defaultChecksums[path]) && - entry.localChecksum !== defaultChecksums[path] && - cachedAssetsManager.exists(path) === false - ) { - entry.localChecksum = defaultChecksums[path]; - checksumsChanged = true; + bin = {}; + bin['cache/' + newKey] = content; + vAPI.cacheStorage.set(bin); + } + countdown(1); + }; + + var onEntries = function(bin) { + entries = bin && bin['cached_asset_entries']; + if ( !entries ) { return callback(); } + if ( bin && bin['assetCacheRegistry'] ) { + assetCacheRegistry = bin['assetCacheRegistry']; + } + var aliases = api.listKeyAliases; + for ( var oldKey in entries ) { + if ( oldKey.endsWith('assets/user/filters.txt') ) { continue; } + var newKey = aliases[oldKey]; + if ( !newKey && /^https?:\/\//.test(oldKey) ) { + newKey = oldKey; } - // If repo checksums could not be fetched, assume no change. - // https://github.com/gorhill/uBlock/issues/602 - // Added: if repo checksum is that of the empty string, - // assume no change - if ( - repoChecksums === '' || - entry.repoChecksum === 'd41d8cd98f00b204e9800998ecf8427e' - ) { - entry.repoChecksum = entry.localChecksum; + if ( newKey ) { + vAPI.cacheStorage.get( + 'cached_asset_content://' + oldKey, + onContentRead.bind(null, oldKey, newKey) + ); + moveCount += 1; } - if ( entry.repoChecksum !== '' || entry.localChecksum === '' ) { - continue; - } - checksumsChanged = true; - cachedAssetsManager.remove(path); - entry.localChecksum = ''; + toRemove.push('cached_asset_content://' + oldKey); } - if ( checksumsChanged ) { - updateLocalChecksums(); - } - // Notify all waiting callers - // https://github.com/chrisaljoudi/uBlock/issues/515 - // VERY IMPORTANT: because of re-entrancy, we MUST: - // - process the waiting callers in a FIFO manner - // - not cache repoMetadata.waiting.length, we MUST use the live - // value, because it can change while looping - // - not change the waiting list until they are all processed - for ( var i = 0; i < repoMetadata.waiting.length; i++ ) { - repoMetadata.waiting[i](repoMetadata); - } - repoMetadata.waiting.length = 0; + toRemove.push('cached_asset_entries', 'extensionLastVersion'); + countdown(); }; - var validateChecksums = function(details) { - if ( details.error || details.content === '' ) { - return ''; - } - if ( /^(?:[0-9a-f]{32}\s+\S+(?:\s+|$))+/.test(details.content) === false ) { - return ''; - } - // https://github.com/gorhill/uBlock/issues/602 - // External filter lists are not meant to appear in checksums.txt. - // TODO: remove this code once v1.1.0.0 is everywhere. - var out = []; - var listMap = µBlock.oldListToNewListMap; - var lines = details.content.split(/\s*\n\s*/); - var line, matches; - for ( var i = 0; i < lines.length; i++ ) { - line = lines[i]; - matches = line.match(/^[0-9a-f]+ (.+)$/); - if ( matches === null || listMap.hasOwnProperty(matches[1]) ) { - continue; - } - out.push(line); - } - return out.join('\n'); - }; - - var parseChecksums = function(text, eachFn) { - var lines = text.split(/\n+/); - var i = lines.length; - var fields; - while ( i-- ) { - fields = lines[i].trim().split(/\s+/); - if ( fields.length !== 2 ) { - continue; - } - eachFn(fields[1], fields[0]); - } - }; - - var onLocalChecksumsLoaded = function(details) { - var entries = repoMetadata.entries; - var processChecksum = function(path, checksum) { - if ( entries.hasOwnProperty(path) === false ) { - entries[path] = new AssetEntry(); - } - entries[path].localChecksum = checksum; - }; - if ( (localChecksums = validateChecksums(details)) ) { - parseChecksums(localChecksums, processChecksum); - } - checksumsReceived(); - }; - - var onRepoChecksumsLoaded = function(details) { - var entries = repoMetadata.entries; - var processChecksum = function(path, checksum) { - if ( entries.hasOwnProperty(path) === false ) { - entries[path] = new AssetEntry(); - } - entries[path].repoChecksum = checksum; - }; - if ( (repoChecksums = validateChecksums(details)) ) { - parseChecksums(repoChecksums, processChecksum); - } - checksumsReceived(); - }; - - // https://github.com/gorhill/uBlock/issues/760 - // We need the checksum values at install time, because some resources - // may have been purged, in which case the checksum must be reset to the - // value at install time. - var onDefaultChecksumsLoaded = function() { - defaultChecksums = Object.create(null); - var processChecksum = function(path, checksum) { - defaultChecksums[path] = checksum; - }; - parseChecksums(this.responseText || '', processChecksum); - checksumsReceived(); - }; - - repoMetadata = new RepoMetadata(); - repoMetadata.waiting.push(callback); - readRepoFile('assets/checksums.txt', onRepoChecksumsLoaded); - getTextFileFromURL(vAPI.getURL('assets/checksums.txt'), onDefaultChecksumsLoaded); - readLocalFile('assets/checksums.txt', onLocalChecksumsLoaded); -}; - -// https://www.youtube.com/watch?v=-t3WYfgM4x8 - -/******************************************************************************/ - -exports.setHomeURL = function(path, homeURL) { - if ( typeof homeURL !== 'string' || homeURL === '' ) { - return; - } - homeURLs[path] = homeURL; -}; - -/******************************************************************************/ - -// Get a local asset, do not look-up repo or remote location if local asset -// is not found. - -var readLocalFile = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; - - var onInstallFileLoaded = function() { - //console.log('µBlock> readLocalFile("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µBlock> readLocalFile("%s") / onInstallFileError()', path); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readLocalFile("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function(details) { - //console.error('µBlock> readLocalFile("%s") / onCachedContentError()', path); - if ( reIsExternalPath.test(path) ) { - reportBack('', 'Error: asset not found'); - return; - } - // It's ok for user data to not be found - if ( reIsUserPath.test(path) ) { - reportBack(''); - return; - } - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; - - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); -}; - -// https://www.youtube.com/watch?v=r9KVpuFPtHc - -/******************************************************************************/ - -// Get the repository copy of a built-in asset. - -var readRepoFile = function(path, callback) { - // https://github.com/chrisaljoudi/uBlock/issues/426 - if ( exports.remoteFetchBarrier !== 0 ) { - readLocalFile(path, callback); - return; - } - - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content, - 'error': err - }; - callback(details); - }; - - var repositoryURL = toRepoURL(path); - - var onRepoFileLoaded = function() { - //console.log('µBlock> readRepoFile("%s") / onRepoFileLoaded()', path); - // https://github.com/gorhill/httpswitchboard/issues/263 - if ( this.status === 200 ) { - reportBack(this.responseText); - } else { - reportBack('', 'Error: ' + this.statusText); - } - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - reportBack('', 'Error'); - }; - - // '_=...' is to skip browser cache - getTextFileFromURL( - repositoryURL + '?_=' + Date.now(), - onRepoFileLoaded, - onRepoFileError + vAPI.cacheStorage.get( + [ 'cached_asset_entries', 'assetCacheRegistry' ], + onEntries ); }; -/******************************************************************************/ +/******************************************************************************* -// An asset from an external source with a copy shipped with the extension: -// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL -// External --> -// Repository --> has checksum (to detect need for update only) -// Cache --> has expiration timestamp (in cache) -// Local --> install time version + The purpose of the asset source registry is to keep key detail information + about an asset: + - Where to load it from: this may consist of one or more URLs, either local + or remote. + - After how many days an asset should be deemed obsolete -- i.e. in need of + an update. + - The origin and type of an asset. + - The last time an asset was registered. -var readRepoCopyAsset = function(path, callback) { - var assetEntry; - var homeURL = homeURLs[path]; +**/ - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; +var assetSourceRegistryStatus, + assetSourceRegistry = Object.create(null); - var updateChecksum = function() { - if ( assetEntry !== undefined && assetEntry.repoChecksum !== assetEntry.localChecksum ) { - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); - } - }; - - var onInstallFileLoaded = function() { - //console.log('µBlock> readRepoCopyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µBlock> readRepoCopyAsset("%s") / onInstallFileError():', path, this.statusText); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function(details) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileError()', path); - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; - - var repositoryURL = toRepoURL(path); - var repositoryURLSkipCache = repositoryURL + '?_=' + Date.now(); - - var onRepoFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s"): error', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onHomeFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s"): no response', path, homeURL); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - } - return; - } - //console.log('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s")', path, homeURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onHomeFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', homeURL)); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); +var registerAssetSource = function(assetKey, dict) { + var entry = assetSourceRegistry[assetKey] || {}; + for ( var prop in dict ) { + if ( dict.hasOwnProperty(prop) === false ) { continue; } + if ( dict[prop] === undefined ) { + delete entry[prop]; } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + entry[prop] = dict[prop]; } - }; - - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Auto-update enabled AND (not in cache OR in cache but obsolete) - var timestamp = entries[path]; - var inCache = typeof timestamp === 'number'; - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && stringIsNotEmpty(homeURL) - ) { - if ( inCache === false || cacheIsObsolete(timestamp) ) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheMetaReady(): not cached or obsolete', path); - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - return; + } + var contentURL = dict.contentURL; + if ( contentURL !== undefined ) { + if ( typeof contentURL === 'string' ) { + contentURL = entry.contentURL = [ contentURL ]; + } else if ( Array.isArray(contentURL) === false ) { + contentURL = entry.contentURL = []; + } + var remoteURLCount = 0; + for ( var i = 0; i < contentURL.length; i++ ) { + if ( reIsExternalPath.test(contentURL[i]) ) { + remoteURLCount += 1; } } + entry.hasLocalURL = remoteURLCount !== contentURL.length; + entry.hasRemoteURL = remoteURLCount !== 0; + } else if ( entry.contentURL === undefined ) { + entry.contentURL = []; + } + if ( typeof entry.updateAfter !== 'number' ) { + entry.updateAfter = 5; + } + if ( entry.submitter ) { + entry.submitTime = Date.now(); // To detect stale entries + } + assetSourceRegistry[assetKey] = entry; +}; - // In cache - if ( inCache ) { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } +var unregisterAssetSource = function(assetKey) { + assetCacheRemove(assetKey); + delete assetSourceRegistry[assetKey]; +}; - // Not in cache - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); +var saveAssetSourceRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry }); }; - - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; + return function(lazily) { + if ( timer !== undefined ) { + clearTimeout(timer); } + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; +})(); - // Repo copy changed: fetch from home URL - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µBlock> readRepoCopyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - if ( stringIsNotEmpty(homeURL) ) { - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - } else { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); +var updateAssetSourceRegistry = function(json) { + var newDict; + try { + newDict = JSON.parse(json); + } catch (ex) { + } + if ( newDict instanceof Object === false ) { return; } + + getAssetSourceRegistry(function(oldDict) { + var assetKey; + // Remove obsolete entries + for ( assetKey in oldDict ) { + if ( newDict[assetKey] === undefined ) { + unregisterAssetSource(assetKey); } - return; } - - // Load from cache - cachedAssetsManager.entries(onCacheMetaReady); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -// https://www.youtube.com/watch?v=uvUW4ozs7pY - -/******************************************************************************/ - -// An important asset shipped with the extension -- typically small, or -// doesn't change often: -// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL -// Repository --> has checksum (to detect need for update and corruption) -// Cache --> whatever from above -// Local --> install time version - -var readRepoOnlyAsset = function(path, callback) { - - var assetEntry; - - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; + // Add/update existing entries + for ( assetKey in newDict ) { + registerAssetSource(assetKey, newDict[assetKey]); } - callback(details); - }; - - var onInstallFileLoaded = function() { - //console.log('µBlock> readRepoOnlyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µBlock> readRepoOnlyAsset("%s") / onInstallFileError()', path); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentError()', path); - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); - }; - - var repositoryURL = toRepoURL(path + '?_=' + Date.now()); - - var onRepoFileLoaded = function() { - if ( typeof this.responseText !== 'string' ) { - console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): no response', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - if ( YaMD5.hashStr(this.responseText) !== assetEntry.repoChecksum ) { - console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): bad md5 checksum', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } - - // Asset added or changed: load from repo URL and then cache result - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - getTextFileFromURL(repositoryURL, onRepoFileLoaded, onRepoFileError); - return; - } - - // Load from cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -/******************************************************************************/ - -// Asset doesn't exist. Just for symmetry purpose. - -var readNilAsset = function(path, callback) { - callback({ - 'path': path, - 'content': '', - 'error': 'Error: asset not found' + saveAssetSourceRegistry(); }); }; -/******************************************************************************/ +var getAssetSourceRegistry = function(callback) { + // Already loaded. + if ( assetSourceRegistryStatus === 'ready' ) { + callback(assetSourceRegistry); + return; + } -// An external asset: -// Path --> starts with 'http' -// External --> https://..., http://... -// Cache --> has expiration timestamp (in cache) + // Being loaded. + if ( Array.isArray(assetSourceRegistryStatus) ) { + assetSourceRegistryStatus.push(callback); + return; + } -var readExternalAsset = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; + // Not loaded: load it. + assetSourceRegistryStatus = [ callback ]; + + var registryReady = function() { + var callers = assetSourceRegistryStatus; + assetSourceRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetSourceRegistry); } + }; + + // First-install case. + var createRegistry = function() { + getTextFileFromURL( + µBlock.assetsBootstrapLocation || 'assets/assets.json', + function() { + updateAssetSourceRegistry(this.responseText); + registryReady(); + } + ); + }; + + vAPI.cacheStorage.get('assetSourceRegistry', function(bin) { + if ( !bin || !bin.assetSourceRegistry ) { + createRegistry(); + return; + } + assetSourceRegistry = bin.assetSourceRegistry; + registryReady(); + }); +}; + +api.registerAssetSource = function(assetKey, details) { + getAssetSourceRegistry(function() { + registerAssetSource(assetKey, details); + saveAssetSourceRegistry(true); + }); +}; + +api.unregisterAssetSource = function(assetKey) { + getAssetSourceRegistry(function() { + unregisterAssetSource(assetKey); + saveAssetSourceRegistry(true); + }); +}; + +/******************************************************************************* + + The purpose of the asset cache registry is to keep track of all assets + which have been persisted into the local cache. + +**/ + +var assetCacheRegistryStatus, + assetCacheRegistryStartTime = Date.now(), + assetCacheRegistry = {}; + +var getAssetCacheRegistry = function(callback) { + // Already loaded. + if ( assetCacheRegistryStatus === 'ready' ) { + callback(assetCacheRegistry); + return; + } + + // Being loaded. + if ( Array.isArray(assetCacheRegistryStatus) ) { + assetCacheRegistryStatus.push(callback); + return; + } + + // Not loaded: load it. + assetCacheRegistryStatus = [ callback ]; + + var registryReady = function() { + var callers = assetCacheRegistryStatus; + assetCacheRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetCacheRegistry); + } + }; + + var migrationDone = function() { + vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { + if ( bin && bin.assetCacheRegistry ) { + assetCacheRegistry = bin.assetCacheRegistry; + } + registryReady(); + }); + }; + + migrate(migrationDone); +}; + +var saveAssetCacheRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry }); + }; + return function(lazily) { + if ( timer !== undefined ) { clearTimeout(timer); } + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; +})(); + +var assetCacheRead = function(assetKey, callback) { + var internalKey = 'cache/' + assetKey; + + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { details.error = err; } callback(details); }; - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readExternalAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - console.error('µBlock> readExternalAsset("%s") / onCachedContentError()', path); - reportBack('', 'Error'); - }; - - var onExternalFileLoaded = function() { - // https://github.com/chrisaljoudi/uBlock/issues/708 - // A successful download should never return an empty file: turn this - // into an error condition. - if ( stringIsNotEmpty(this.responseText) === false ) { - onExternalFileError(); - return; + var onAssetRead = function(bin) { + if ( !bin || !bin[internalKey] ) { + return reportBack('', 'E_NOTFOUND'); } - //console.log('µBlock> readExternalAsset("%s") / onExternalFileLoaded1()', path); - cachedAssetsManager.save(path, this.responseText); - reportBack(this.responseText); - }; - - var onExternalFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', path)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Not in cache OR - // - // - Auto-update enabled AND in cache but obsolete - var timestamp = entries[path]; - var notInCache = typeof timestamp !== 'number'; - var updateCache = exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - cacheIsObsolete(timestamp); - if ( notInCache || updateCache ) { - getTextFileFromURL(path, onExternalFileLoaded, onExternalFileError); - return; + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + return reportBack('', 'E_NOTFOUND'); } - - // In cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + entry.readTime = Date.now(); + saveAssetCacheRegistry(true); + reportBack(bin[internalKey]); }; - cachedAssetsManager.entries(onCacheMetaReady); + var onReady = function() { + vAPI.cacheStorage.get(internalKey, onAssetRead); + }; + + getAssetCacheRegistry(onReady); +}; + +var assetCacheWrite = function(assetKey, details, callback) { + var internalKey = 'cache/' + assetKey; + var content = ''; + if ( typeof details === 'string' ) { + content = details; + } else if ( details instanceof Object ) { + content = details.content || ''; + } + + if ( content === '' ) { + return assetCacheRemove(assetKey, callback); + } + + var reportBack = function(content) { + var details = { assetKey: assetKey, content: content }; + if ( typeof callback === 'function' ) { + callback(details); + } + fireNotification('after-asset-updated', details); + }; + + var onReady = function() { + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + entry = assetCacheRegistry[assetKey] = {}; + } + entry.writeTime = entry.readTime = Date.now(); + if ( details instanceof Object && typeof details.url === 'string' ) { + entry.remoteURL = details.url; + } + var bin = { assetCacheRegistry: assetCacheRegistry }; + bin[internalKey] = content; + vAPI.cacheStorage.set(bin); + reportBack(content); + }; + getAssetCacheRegistry(onReady); +}; + +var assetCacheRemove = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + removedEntries = [], + removedContent = []; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; + } + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + removedEntries.push(assetKey); + removedContent.push('cache/' + assetKey); + delete cacheDict[assetKey]; + } + if ( removedContent.length !== 0 ) { + vAPI.cacheStorage.remove(removedContent); + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); + } + for ( var i = 0; i < removedEntries.length; i++ ) { + fireNotification('after-asset-updated', { assetKey: removedEntries[i] }); + } + }; + + getAssetCacheRegistry(onReady); +}; + +var assetCacheMarkAsDirty = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + cacheEntry, + mustSave = false; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; + } + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + cacheEntry = cacheDict[assetKey]; + if ( !cacheEntry.writeTime ) { continue; } + cacheDict[assetKey].writeTime = 0; + mustSave = true; + } + if ( mustSave ) { + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); + } + }; + + getAssetCacheRegistry(onReady); }; /******************************************************************************/ -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved +var stringIsNotEmpty = function(s) { + return typeof s === 'string' && s !== ''; +}; -var readUserAsset = function(path, callback) { - // TODO: remove when confident all users no longer have their custom - // filters saved into vAPI.cacheStorage. - var onCachedContentLoaded = function(details) { - saveUserAsset(path, details.content); - //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; +/******************************************************************************* - var onCachedContentError = function() { - saveUserAsset(path, ''); - //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); + User assets are NOT persisted in the cache storage. User assets are + recognized by the asset key which always starts with 'user-'. + + TODO(seamless migration): + Can remove instances of old user asset keys when I am confident all users + are using uBO v1.11 and beyond. + +**/ + +var readUserAsset = function(assetKey, callback) { + var reportBack = function(content) { + callback({ assetKey: assetKey, content: content }); }; var onLoaded = function(bin) { - var content = bin && bin[path]; - if ( typeof content === 'string' ) { - callback({ 'path': path, 'content': content }); - return; + if ( !bin ) { return reportBack(''); } + var content = ''; + if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) { + content = bin['cached_asset_content://assets/user/filters.txt']; + vAPI.cacheStorage.remove('cached_asset_content://assets/user/filters.txt'); } - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + if ( typeof bin['assets/user/filters.txt'] === 'string' ) { + content = bin['assets/user/filters.txt']; + // TODO(seamless migration): + // Uncomment once all moved to v1.11+. + //vAPI.storage.remove('assets/user/filters.txt'); + } + if ( typeof bin[assetKey] === 'string' ) { + // TODO(seamless migration): + // Replace conditional with assignment once all moved to v1.11+ + if ( content !== bin[assetKey] ) { + saveUserAsset(assetKey, content); + } + } else if ( content !== '' ) { + saveUserAsset(assetKey, content); + } + return reportBack(content); }; - - vAPI.storage.get(path, onLoaded); + var toRead = assetKey; + if ( assetKey === µBlock.userFiltersPath ) { + toRead = [ + assetKey, + 'assets/user/filters.txt', + 'cached_asset_content://assets/user/filters.txt' + ]; + } + vAPI.storage.get(toRead, onLoaded); }; -var saveUserAsset = function(path, content, callback) { +var saveUserAsset = function(assetKey, content, callback) { var bin = {}; - bin[path] = content; + bin[assetKey] = content; + // TODO(seamless migration): + // This is for forward compatibility. Only for a limited time. Remove when + // everybody moved to 1.11.0 and beyond. + // >>>>>>>> + if ( assetKey === µBlock.userFiltersPath ) { + bin['assets/user/filters.txt'] = content; + } + // <<<<<<<< var onSaved = function() { - // Saving over an existing asset must be seen as removing an - // existing asset and adding a new one. - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener([ path ]); - } if ( callback instanceof Function ) { - callback({ path: path, content: content }); + callback({ assetKey: assetKey, content: content }); } }; vAPI.storage.set(bin, onSaved); @@ -1060,602 +696,318 @@ var saveUserAsset = function(path, content, callback) { /******************************************************************************/ -// Asset available only from the cache. -// Cache data: -// Path --> starts with 'cache://' -// Cache --> whatever +api.get = function(assetKey, callback) { + if ( assetKey === µBlock.userFiltersPath ) { + readUserAsset(assetKey, callback); + return; + } -var readCacheAsset = function(path, callback) { - var onCachedContentLoaded = function(details) { - //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); + var assetDetails = {}, + contentURLs, + contentURL; + + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; + } + callback(details); }; - var onCachedContentError = function() { - //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); + var onContentNotLoaded = function() { + var isExternal; + while ( (contentURL = contentURLs.shift()) ) { + isExternal = reIsExternalPath.test(contentURL); + if ( isExternal === false || assetDetails.hasLocalURL !== true ) { + break; + } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + getTextFileFromURL(contentURL, onContentLoaded, onContentNotLoaded); }; - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); -}; - -/******************************************************************************/ - -// Assets -// -// A copy of an asset from an external source shipped with the extension: -// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL -// External --> -// Repository --> has checksum (to detect obsolescence) -// Cache --> has expiration timestamp (to detect obsolescence) -// Local --> install time version -// -// An important asset shipped with the extension (usually small, or doesn't -// change often): -// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL -// Repository --> has checksum (to detect obsolescence or data corruption) -// Cache --> whatever from above -// Local --> install time version -// -// An external filter list: -// Path --> starts with 'http' -// External --> -// Cache --> has expiration timestamp (to detect obsolescence) -// -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved -// -// When a checksum is present, it is used to determine whether the asset -// needs to be updated. -// When an expiration timestamp is present, it is used to determine whether -// the asset needs to be updated. -// -// If no update required, an asset if first fetched from the cache. If the -// asset is not cached it is fetched from the closest location: local for -// an asset shipped with the extension, external for an asset not shipped -// with the extension. - -exports.get = function(path, callback) { - - if ( reIsUserPath.test(path) ) { - readUserAsset(path, callback); - return; - } - - if ( reIsCachePath.test(path) ) { - readCacheAsset(path, callback); - return; - } - - if ( reIsExternalPath.test(path) ) { - readExternalAsset(path, callback); - return; - } - - var onRepoMetaReady = function(meta) { - var assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - readNilAsset(path, callback); + var onContentLoaded = function() { + if ( stringIsNotEmpty(this.responseText) === false ) { + onContentNotLoaded(); return; } - - // Asset is repo copy of external content - if ( stringIsNotEmpty(homeURLs[path]) ) { - readRepoCopyAsset(path, callback); - return; - } - - // Asset is repo only - readRepoOnlyAsset(path, callback); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -// https://www.youtube.com/watch?v=98y0Q7nLGWk - -/******************************************************************************/ - -exports.getLocal = readLocalFile; - -/******************************************************************************/ - -exports.put = function(path, content, callback) { - if ( reIsUserPath.test(path) ) { - saveUserAsset(path, content, callback); - return; - } - - cachedAssetsManager.save(path, content, callback); -}; - -/******************************************************************************/ - -exports.rmrf = function() { - cachedAssetsManager.rmrf(); -}; - -/******************************************************************************/ - -exports.rename = function(from, to, callback) { - var done = function() { - if ( typeof callback === 'function' ) { - callback(); - } - }; - - var fromLoaded = function(details) { - cachedAssetsManager.remove(from); - cachedAssetsManager.save(to, details.content, callback); - done(); - }; - - var toLoaded = function(details) { - // `to` already exists: do nothing - if ( details.content !== '' ) { - return done(); - } - cachedAssetsManager.load(from, fromLoaded); - }; - - // If `to` content already exists, do nothing. - cachedAssetsManager.load(to, toLoaded); -}; - -/******************************************************************************/ - -exports.metadata = function(callback) { - var out = {}; - - // https://github.com/chrisaljoudi/uBlock/issues/186 - // We need to check cache obsolescence when both cache and repo meta data - // has been gathered. - var checkCacheObsolescence = function() { - var entry, homeURL; - for ( var path in out ) { - if ( out.hasOwnProperty(path) === false ) { - continue; - } - entry = out[path]; - // https://github.com/gorhill/uBlock/issues/528 - // Not having a homeURL property does not mean the filter list - // is not external. - homeURL = reIsExternalPath.test(path) ? path : homeURLs[path]; - entry.cacheObsolete = stringIsNotEmpty(homeURL) && - cacheIsObsolete(entry.lastModified); - } - callback(out); - }; - - var onRepoMetaReady = function(meta) { - var entries = meta.entries; - var entryRepo, entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryRepo = entries[path]; - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; - } - entryOut.localChecksum = entryRepo.localChecksum; - entryOut.repoChecksum = entryRepo.repoChecksum; - entryOut.homeURL = homeURLs[path] || ''; - entryOut.supportURL = entryRepo.supportURL || ''; - entryOut.repoObsolete = entryOut.localChecksum !== entryOut.repoChecksum; - } - checkCacheObsolescence(); - }; - - var onCacheMetaReady = function(entries) { - var entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; - } - entryOut.lastModified = entries[path]; - // User data is not literally cache data - if ( reIsUserPath.test(path) ) { - continue; - } - entryOut.cached = true; - if ( reIsExternalPath.test(path) ) { - entryOut.homeURL = path; - } - } - getRepoMetadata(onRepoMetaReady); - }; - - cachedAssetsManager.entries(onCacheMetaReady); -}; - -/******************************************************************************/ - -exports.purge = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); -}; - -exports.purgeCacheableAsset = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); - lastRepoMetaTimestamp = 0; -}; - -exports.purgeAll = function(callback) { - cachedAssetsManager.removeAll(callback); - lastRepoMetaTimestamp = 0; -}; - -/******************************************************************************/ - -exports.onAssetRemoved = { - addListener: function(callback) { - onAssetRemovedListener = callback instanceof Function ? callback : null; - } -}; - -/******************************************************************************/ - -return exports; - -})(); - -/******************************************************************************/ -/******************************************************************************/ - -µBlock.assetUpdater = (function() { - -/******************************************************************************/ - -var µb = µBlock; - -var updateDaemonTimer = null; -var autoUpdateDaemonTimerPeriod = 11 * 60 * 1000; // 11 minutes -var manualUpdateDaemonTimerPeriod = 5 * 1000; // 5 seconds - -var updateCycleFirstPeriod = 7 * 60 * 1000; // 7 minutes -var updateCycleNextPeriod = 11 * 60 * 60 * 1000; // 11 hours -var updateCycleTime = 0; - -var toUpdate = {}; -var toUpdateCount = 0; -var updated = {}; -var updatedCount = 0; -var metadata = null; - -var onStartListener = null; -var onCompletedListener = null; -var onAssetUpdatedListener = null; - -var exports = { - manualUpdate: false, - manualUpdateProgress: { - value: 0, - text: null - } -}; - -/******************************************************************************/ - -var onOneUpdated = function(details) { - // Resource fetched, we can safely restart the daemon. - scheduleUpdateDaemon(); - - var path = details.path; - if ( details.error ) { - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount)); - //console.debug('µBlock.assetUpdater/onOneUpdated: "%s" failed', path); - return; - } - - //console.debug('µBlock.assetUpdater/onOneUpdated: "%s"', path); - updated[path] = true; - updatedCount += 1; - - if ( typeof onAssetUpdatedListener === 'function' ) { - onAssetUpdatedListener(details); - } - - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount + 1)); -}; - -/******************************************************************************/ - -var updateOne = function() { - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var metaEntry; - var updatingCount = 0; - var updatingText = null; - - for ( var path in toUpdate ) { - if ( toUpdate.hasOwnProperty(path) === false ) { - continue; - } - if ( toUpdate[path] !== true ) { - continue; - } - toUpdate[path] = false; - toUpdateCount -= 1; - if ( metadata.hasOwnProperty(path) === false ) { - continue; - } - metaEntry = metadata[path]; - if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) { - continue; - } - - // Will restart the update daemon once the resource is received: the - // fetching of a resource may take some time, possibly beyond the - // next scheduled daemon cycle, so this ensure the daemon won't do - // anything else before the resource is fetched (or times out). - suspendUpdateDaemon(); - - //console.debug('µBlock.assetUpdater/updateOne: assets.get("%s")', path); - µb.assets.get(path, onOneUpdated); - updatingCount = 1; - updatingText = metaEntry.homeURL || path; - break; - } - - manualUpdateNotify( - false, - (updatedCount + updatingCount/2) / (updatedCount + toUpdateCount + updatingCount + 1), - updatingText - ); -}; - -/******************************************************************************/ - -// Update one asset, fetch metadata if not done yet. - -var safeUpdateOne = function() { - if ( metadata !== null ) { - updateOne(); - return; - } - - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var onMetadataReady = function(response) { - scheduleUpdateDaemon(); - metadata = response; - updateOne(); - }; - - suspendUpdateDaemon(); - µb.assets.metadata(onMetadataReady); -}; - -/******************************************************************************/ - -var safeStartListener = function(callback) { - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var onStartListenerDone = function(assets) { - scheduleUpdateDaemon(); - assets = assets || {}; - for ( var path in assets ) { - if ( assets.hasOwnProperty(path) === false ) { - continue; - } - if ( toUpdate.hasOwnProperty(path) ) { - continue; - } - //console.debug('assets.js > µBlock.assetUpdater/safeStartListener: "%s"', path); - toUpdate[path] = true; - toUpdateCount += 1; - } - if ( typeof callback === 'function' ) { - callback(); - } - }; - - if ( typeof onStartListener === 'function' ) { - suspendUpdateDaemon(); - onStartListener(onStartListenerDone); - } else { - onStartListenerDone(null); - } -}; - -/******************************************************************************/ - -var updateDaemon = function() { - updateDaemonTimer = null; - scheduleUpdateDaemon(); - - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - if ( µb.assets.autoUpdate !== true ) { - return; - } - - // Start an update cycle? - if ( updateCycleTime !== 0 ) { - if ( Date.now() >= updateCycleTime ) { - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle started'); - reset(); - safeStartListener(); - } - return; - } - - // Any asset to update? - if ( toUpdateCount !== 0 ) { - safeUpdateOne(); - return; - } - // Nothing left to update - - // In case of manual update, fire progress notifications - manualUpdateNotify(true, 1, ''); - - // If anything was updated, notify listener - if ( updatedCount !== 0 ) { - if ( typeof onCompletedListener === 'function' ) { - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle completed'); - onCompletedListener({ - updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy - updatedCount: updatedCount + if ( reIsExternalPath.test(contentURL) ) { + assetCacheWrite(assetKey, { + content: this.responseText, + url: contentURL }); } - } + reportBack(this.responseText); + }; - // Schedule next update cycle - if ( updateCycleTime === 0 ) { - reset(); - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle re-scheduled'); - updateCycleTime = Date.now() + updateCycleNextPeriod; - } + var onCachedContentLoaded = function(details) { + if ( details.content !== '' ) { + return reportBack(details.content); + } + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + onContentNotLoaded(); + }); + }; + + assetCacheRead(assetKey, onCachedContentLoaded); }; /******************************************************************************/ -var scheduleUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - } - updateDaemonTimer = vAPI.setTimeout( - updateDaemon, - exports.manualUpdate ? manualUpdateDaemonTimerPeriod : autoUpdateDaemonTimerPeriod - ); -}; +var getRemote = function(assetKey, callback) { + var assetDetails = {}, + contentURLs, + contentURL; -var suspendUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - updateDaemonTimer = null; - } -}; + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; + } + callback(details); + }; -scheduleUpdateDaemon(); + var onRemoteContentLoaded = function() { + if ( stringIsNotEmpty(this.responseText) === false ) { + registerAssetSource(assetKey, { error: { time: Date.now(), error: 'No content' } }); + tryLoading(); + return; + } + assetCacheWrite(assetKey, { + content: this.responseText, + url: contentURL + }); + registerAssetSource(assetKey, { error: undefined }); + reportBack(this.responseText); + }; -/******************************************************************************/ + var onRemoteContentError = function() { + registerAssetSource(assetKey, { error: { time: Date.now(), error: this.statusText } }); + tryLoading(); + }; -var reset = function() { - toUpdate = {}; - toUpdateCount = 0; - updated = {}; - updatedCount = 0; - updateCycleTime = 0; - metadata = null; + var tryLoading = function() { + while ( (contentURL = contentURLs.shift()) ) { + if ( reIsExternalPath.test(contentURL) ) { break; } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + getTextFileFromURL(contentURL, onRemoteContentLoaded, onRemoteContentError); + }; + + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + tryLoading(); + }); }; /******************************************************************************/ -var manualUpdateNotify = function(done, value, text) { - if ( exports.manualUpdate === false ) { - return; +api.put = function(assetKey, content, callback) { + if ( reIsUserAsset.test(assetKey) ) { + return saveUserAsset(assetKey, content, callback); } + assetCacheWrite(assetKey, content, callback); +}; - exports.manualUpdate = !done; - exports.manualUpdateProgress.value = value || 0; - if ( typeof text === 'string' ) { - exports.manualUpdateProgress.text = text; - } +/******************************************************************************/ - vAPI.messaging.broadcast({ - what: 'forceUpdateAssetsProgress', - done: !exports.manualUpdate, - progress: exports.manualUpdateProgress, - updatedCount: updatedCount +api.metadata = function(callback) { + var assetRegistryReady = false, + cacheRegistryReady = false; + + var onReady = function() { + var assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)), + cacheDict = assetCacheRegistry, + assetEntry, cacheEntry, + now = Date.now(), obsoleteAfter; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry ) { + assetEntry.cached = true; + assetEntry.writeTime = cacheEntry.writeTime; + obsoleteAfter = cacheEntry.writeTime + assetEntry.updateAfter * 86400000; + assetEntry.obsolete = obsoleteAfter < now; + assetEntry.remoteURL = cacheEntry.remoteURL; + } else { + assetEntry.writeTime = 0; + obsoleteAfter = 0; + assetEntry.obsolete = true; + } + } + callback(assetDict); + }; + + getAssetSourceRegistry(function() { + assetRegistryReady = true; + if ( cacheRegistryReady ) { onReady(); } }); - // When manually updating, whatever launched the manual update is - // responsible to launch a reload of the filter lists. - if ( exports.manualUpdate !== true ) { - reset(); - } + getAssetCacheRegistry(function() { + cacheRegistryReady = assetCacheRegistry; + if ( assetRegistryReady ) { onReady(); } + }); }; /******************************************************************************/ -// Manual update: just a matter of forcing the update daemon to work on a -// tighter schedule. +api.purge = function(pattern, callback) { + assetCacheMarkAsDirty(pattern, callback); +}; -exports.force = function() { - if ( exports.manualUpdate ) { - return; - } +api.remove = function(pattern, callback) { + assetCacheRemove(pattern, callback); +}; - reset(); +api.rmrf = function() { + assetCacheRemove(/./); +}; - exports.manualUpdate = true; +/******************************************************************************/ - var onStartListenerDone = function() { - if ( toUpdateCount === 0 ) { - updateCycleTime = Date.now() + updateCycleNextPeriod; - manualUpdateNotify(true, 1); - } else { - manualUpdateNotify(false, 0); - safeUpdateOne(); +// Asset updater area. +var updaterStatus, + updaterTimer, + updaterAssetDelayDefault = 120000, + updaterAssetDelay = updaterAssetDelayDefault, + updaterUpdated = [], + updaterFetched = new Set(); + +var updateFirst = function() { + updaterStatus = 'updating'; + updaterFetched.clear(); + updaterUpdated = []; + fireNotification('before-assets-updated'); + updateNext(); +}; + +var updateNext = function() { + var assetDict, cacheDict; + + // This will remove a cached asset when it's no longer in use. + var garbageCollectOne = function(assetKey) { + var cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { + assetCacheRemove(assetKey); } }; - safeStartListener(onStartListenerDone); -}; - -/******************************************************************************/ - -exports.onStart = { - addEventListener: function(callback) { - onStartListener = callback || null; - if ( typeof onStartListener === 'function' ) { - updateCycleTime = Date.now() + updateCycleFirstPeriod; + var findOne = function() { + var now = Date.now(), + assetEntry, cacheEntry; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + if ( assetEntry.hasRemoteURL !== true ) { continue; } + if ( updaterFetched.has(assetKey) ) { continue; } + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { + continue; + } + if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { + return assetKey; + } + garbageCollectOne(assetKey); } + }; + + var updatedOne = function(details) { + if ( details.content !== '' ) { + updaterUpdated.push(details.assetKey); + if ( details.assetKey === 'assets.json' ) { + updateAssetSourceRegistry(details.content); + } + } + if ( findOne() !== undefined ) { + vAPI.setTimeout(updateNext, updaterAssetDelay); + } else { + updateDone(); + } + }; + + var updateOne = function() { + var assetKey = findOne(); + if ( assetKey === undefined ) { + return updateDone(); + } + updaterFetched.add(assetKey); + getRemote(assetKey, updatedOne); + }; + + getAssetSourceRegistry(function(dict) { + assetDict = dict; + if ( !cacheDict ) { return; } + updateOne(); + }); + + getAssetCacheRegistry(function(dict) { + cacheDict = dict; + if ( !assetDict ) { return; } + updateOne(); + }); +}; + +var updateDone = function() { + var assetKeys = updaterUpdated.slice(0); + updaterFetched.clear(); + updaterUpdated = []; + updaterStatus = undefined; + updaterAssetDelay = updaterAssetDelayDefault; + fireNotification('after-assets-updated', { assetKeys: assetKeys }); +}; + +api.updateStart = function(details) { + var oldUpdateDelay = updaterAssetDelay, + newUpdateDelay = details.delay || updaterAssetDelayDefault; + updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); + if ( updaterStatus !== undefined ) { + if ( newUpdateDelay < oldUpdateDelay ) { + clearTimeout(updaterTimer); + updaterTimer = vAPI.setTimeout(updateNext, updaterAssetDelay); + } + return; + } + updateFirst(); +}; + +api.updateStop = function() { + if ( updaterTimer ) { + clearTimeout(updaterTimer); + updaterTimer = undefined; + } + if ( updaterStatus !== undefined ) { + updateDone(); } }; /******************************************************************************/ -exports.onAssetUpdated = { - addEventListener: function(callback) { - onAssetUpdatedListener = callback || null; - } -}; +return api; /******************************************************************************/ -exports.onCompleted = { - addEventListener: function(callback) { - onCompletedListener = callback || null; - } -}; - -/******************************************************************************/ - -// Typically called when an update has been forced. - -exports.restart = function() { - reset(); - updateCycleTime = Date.now() + updateCycleNextPeriod; -}; - -/******************************************************************************/ - -// Call when disabling uBlock, to ensure it doesn't stick around as a detached -// window object in Firefox. - -exports.shutdown = function() { - suspendUpdateDaemon(); - reset(); -}; - -/******************************************************************************/ - -return exports; - })(); /******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js index 00b82a1c6..791e56be9 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,20 +19,16 @@ Home: https://github.com/gorhill/uBlock */ -/* exported µBlock */ - 'use strict'; /******************************************************************************/ -var µBlock = (function() { +var µBlock = (function() { // jshint ignore:line /******************************************************************************/ var oneSecond = 1000; var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -// var oneDay = 24 * oneHour; /******************************************************************************/ @@ -71,8 +67,12 @@ return { }, hiddenSettingsDefault: { + assetFetchTimeout: 30, + autoUpdateAssetFetchPeriod: 120, + autoUpdatePeriod: 7, ignoreRedirectFilters: false, ignoreScriptInjectFilters: false, + manualUpdateAssetFetchPeriod: 2000, popupFontSize: 'unset', suspendTabsUntilReady: false }, @@ -119,92 +119,15 @@ return { lastBackupTime: 0 }, - // EasyList, EasyPrivacy and many others have an 4-day update period, - // as per list headers. - updateAssetsEvery: 97 * oneHour, - projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/', - userFiltersPath: 'assets/user/filters.txt', - pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', + // Allows to fully customize uBO's assets, typically set through admin + // settings. The content of 'assets.json' will also tell which filter + // lists to enable by default when uBO is first installed. + assetsBootstrapLocation: 'assets/assets.json', - // permanent lists - permanentLists: { - // User - 'assets/user/filters.txt': { - group: 'default' - }, - // uBlock - 'assets/ublock/filters.txt': { - title: 'uBlock filters', - group: 'default' - }, - 'assets/ublock/privacy.txt': { - title: 'uBlock filters – Privacy', - group: 'default' - }, - 'assets/ublock/unbreak.txt': { - title: 'uBlock filters – Unbreak', - group: 'default' - }, - 'assets/ublock/badware.txt': { - title: 'uBlock filters – Badware risks', - group: 'default', - supportURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks', - instructionURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks' - }, - 'assets/ublock/experimental.txt': { - title: 'uBlock filters – Experimental', - group: 'default', - off: true, - supportURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters', - instructionURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters' - } - }, + userFiltersPath: 'user-filters', + pslAssetKey: 'public_suffix_list.dat', - // current lists - remoteBlacklists: {}, - oldListToNewListMap: { - "assets/thirdparties/adblock.gardar.net/is.abp.txt": "http://adblock.gardar.net/is.abp.txt", - "assets/thirdparties/adblock.schack.dk/block.txt": "https://adblock.dk/block.csv", - "https://adblock.schack.dk/block.txt": "https://adblock.dk/block.csv", - "assets/thirdparties/dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/advblock.txt": "https://easylist-downloads.adblockplus.org/advblock.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/bitblock.txt": "https://easylist-downloads.adblockplus.org/bitblock.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistchina.txt": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistdutch.txt": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistgermany.txt": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistitaly.txt": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-social.txt": "https://easylist-downloads.adblockplus.org/fanboy-social.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/liste_fr.txt": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "assets/thirdparties/gitorious.org/adblock-latvian/adblock-latvian/raw/master_lists/latvian-list.txt": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", - "assets/thirdparties/home.fredfiber.no/langsholt/adblock.txt": "http://home.fredfiber.no/langsholt/adblock.txt", - "assets/thirdparties/hosts-file.net/ad-servers": "http://hosts-file.net/.%5Cad_servers.txt", - "assets/thirdparties/http://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "assets/thirdparties/liste-ar-adblock.googlecode.com/hg/Liste_AR.txt": "https://liste-ar-adblock.googlecode.com/hg/Liste_AR.txt", - "assets/thirdparties/margevicius.lt/easylistlithuania.txt": "http://margevicius.lt/easylistlithuania.txt", - "assets/thirdparties/mirror1.malwaredomains.com/files/immortal_domains.txt": "http://malwaredomains.lehigh.edu/files/immortal_domains.txt", - "assets/thirdparties/raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt": "https://raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt", - "assets/thirdparties/raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", - "assets/thirdparties/raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", - "assets/thirdparties/raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", - "assets/thirdparties/raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", - "assets/thirdparties/someonewhocares.org/hosts/hosts": "http://someonewhocares.org/hosts/hosts", - "assets/thirdparties/spam404bl.com/spam404scamlist.txt": "https://spam404bl.com/spam404scamlist.txt", - "assets/thirdparties/stanev.org/abp/adblock_bg.txt": "http://stanev.org/abp/adblock_bg.txt", - "assets/thirdparties/winhelp2002.mvps.org/hosts.txt": "http://winhelp2002.mvps.org/hosts.txt", - "assets/thirdparties/www.fanboy.co.nz/enhancedstats.txt": "https://www.fanboy.co.nz/enhancedstats.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-antifacebook.txt": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-korean.txt": "https://www.fanboy.co.nz/fanboy-korean.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-swedish.txt": "https://www.fanboy.co.nz/fanboy-swedish.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-ultimate.txt": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-vietnam.txt": "https://www.fanboy.co.nz/fanboy-vietnam.txt", - "assets/thirdparties/www.void.gr/kargig/void-gr-filters.txt": "https://www.void.gr/kargig/void-gr-filters.txt", - "assets/thirdparties/www.zoso.ro/pages/rolist.txt": "", - "https://iadb.azurewebsites.net/Finland_adb.txt": "http://adb.juvander.net/Finland_adb.txt", - "https://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt" - }, + availableFilterLists: {}, selfieAfter: 23 * oneMinute, diff --git a/src/js/logger.js b/src/js/logger.js index b9a7bee32..700521ab5 100644 --- a/src/js/logger.js +++ b/src/js/logger.js @@ -19,15 +19,13 @@ Home: https://github.com/gorhill/uBlock */ -/* global µBlock */ +'use strict'; /******************************************************************************/ /******************************************************************************/ µBlock.logger = (function() { -'use strict'; - /******************************************************************************/ /******************************************************************************/ diff --git a/src/js/messaging.js b/src/js/messaging.js index 360fc7e0f..4cc466f7c 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -77,7 +77,7 @@ var onMessage = function(request, sender, callback) { return; case 'reloadAllFilters': - µb.reloadAllFilters(callback); + µb.loadFilterLists(); return; case 'scriptlet': @@ -121,7 +121,8 @@ var onMessage = function(request, sender, callback) { break; case 'forceUpdateAssets': - µb.assetUpdater.force(); + µb.scheduleAssetUpdater(0); + µb.assets.updateStart({ delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod || 2000 }); break; case 'getAppData': @@ -160,7 +161,7 @@ var onMessage = function(request, sender, callback) { break; case 'selectFilterLists': - µb.selectFilterLists(request.switches); + µb.saveSelectedFilterLists(request.keys, request.append); break; case 'setWhitelist': @@ -753,7 +754,7 @@ var backupUserData = function(callback) { timeStamp: Date.now(), version: vAPI.app.version, userSettings: µb.userSettings, - filterLists: {}, + selectedFilterLists: [], hiddenSettingsString: µb.stringFromHiddenSettings(), netWhitelist: µb.stringFromWhitelist(µb.netWhitelist), dynamicFilteringString: µb.permanentFirewall.toString(), @@ -762,8 +763,17 @@ var backupUserData = function(callback) { userFilters: '' }; - var onSelectedListsReady = function(filterLists) { - userData.filterLists = filterLists; + var onSelectedListsReady = function(selectedFilterLists) { + userData.selectedFilterLists = selectedFilterLists; + + // TODO(seamless migration): + // The following is strictly for convenience, to be minimally + // forward-compatible. This will definitely be removed in the + // short term, as I do not expect the need to install an older + // version of uBO to ever be needed beyond the short term. + // >>>>>>>> + userData.filterLists = µb.oldDataFromNewListKeys(selectedFilterLists); + // <<<<<<<< var filename = vAPI.i18n('aboutBackupFilename') .replace('{{datetime}}', µb.dateNowToSensibleString()) @@ -773,17 +783,15 @@ var backupUserData = function(callback) { 'url': 'data:text/plain;charset=utf-8,' + encodeURIComponent(JSON.stringify(userData, null, ' ')), 'filename': filename }); - µb.restoreBackupSettings.lastBackupFile = filename; µb.restoreBackupSettings.lastBackupTime = Date.now(); vAPI.storage.set(µb.restoreBackupSettings); - getLocalData(callback); }; var onUserFiltersReady = function(details) { userData.userFilters = details.content; - µb.extractSelectedFilterLists(onSelectedListsReady); + µb.loadSelectedFilterLists(onSelectedListsReady); }; µb.assets.get(µb.userFiltersPath, onUserFiltersReady); @@ -791,32 +799,32 @@ var backupUserData = function(callback) { var restoreUserData = function(request) { var userData = request.userData; - var countdown = 8; - var onCountdown = function() { - countdown -= 1; - if ( countdown === 0 ) { - vAPI.app.restart(); - } - }; var onAllRemoved = function() { - // Be sure to adjust `countdown` if adding/removing anything below - µb.keyvalSetOne('version', userData.version); µBlock.saveLocalSettings(); - vAPI.storage.set(userData.userSettings, onCountdown); - µb.keyvalSetOne('remoteBlacklists', userData.filterLists, onCountdown); + vAPI.storage.set(userData.userSettings); µb.hiddenSettingsFromString(userData.hiddenSettingsString || ''); - µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown); - µb.keyvalSetOne('dynamicFilteringString', userData.dynamicFilteringString || '', onCountdown); - µb.keyvalSetOne('urlFilteringString', userData.urlFilteringString || '', onCountdown); - µb.keyvalSetOne('hostnameSwitchesString', userData.hostnameSwitchesString || '', onCountdown); - µb.assets.put(µb.userFiltersPath, userData.userFilters, onCountdown); vAPI.storage.set({ + netWhitelist: userData.netWhitelist || '', + dynamicFilteringString: userData.dynamicFilteringString || '', + urlFilteringString: userData.urlFilteringString || '', + hostnameSwitchesString: userData.hostnameSwitchesString || '', lastRestoreFile: request.file || '', lastRestoreTime: Date.now(), lastBackupFile: '', lastBackupTime: 0 - }, onCountdown); + }); + µb.assets.put(µb.userFiltersPath, userData.userFilters); + + // 'filterLists' is available up to uBO v1.10.4, not beyond. + // 'selectedFilterLists' is available from uBO v1.11 and beyond. + if ( Array.isArray(userData.selectedFilterLists) ) { + µb.saveSelectedFilterLists(userData.selectedFilterLists); + } else if ( userData.filterLists instanceof Object ) { + µb.saveSelectedFilterLists(µb.newListKeysFromOldData(userData.filterLists)); + } + + vAPI.app.restart(); }; // https://github.com/chrisaljoudi/uBlock/issues/1102 @@ -848,9 +856,7 @@ var prepListEntries = function(entries) { var µburi = µb.URI; var entry, hn; for ( var k in entries ) { - if ( entries.hasOwnProperty(k) === false ) { - continue; - } + if ( entries.hasOwnProperty(k) === false ) { continue; } entry = entries[k]; if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { entry.supportName = µburi.hostnameFromURI(entry.supportURL); @@ -869,16 +875,14 @@ var getLists = function(callback) { cache: null, parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters, cosmeticFilterCount: µb.cosmeticFilteringEngine.getFilterCount(), - current: µb.remoteBlacklists, + current: µb.availableFilterLists, ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, - manualUpdate: false, netFilterCount: µb.staticNetFilteringEngine.getFilterCount(), - userFiltersPath: µb.userFiltersPath + userFiltersPath: µb.userFiltersPath, + aliases: µb.assets.listKeyAliases }; var onMetadataReady = function(entries) { r.cache = entries; - r.manualUpdate = µb.assetUpdater.manualUpdate; - r.manualUpdateProgress = µb.assetUpdater.manualUpdateProgress; prepListEntries(r.cache); callback(r); }; @@ -952,9 +956,6 @@ var onMessage = function(request, sender, callback) { case 'getLocalData': return getLocalData(callback); - case 'purgeAllCaches': - return µb.assets.purgeAll(callback); - case 'readUserFilters': return µb.loadUserFilters(callback); @@ -973,8 +974,18 @@ var onMessage = function(request, sender, callback) { response = getRules(); break; + case 'purgeAllCaches': + if ( request.hard ) { + µb.assets.remove(/./); + } else { + µb.assets.remove(/compiled\//); + µb.assets.purge(/./); + } + break; + case 'purgeCache': - µb.assets.purgeCacheableAsset(request.path); + µb.assets.purge(request.assetKey); + µb.assets.remove('compiled/' + request.assetKey); break; case 'readHiddenSettings': diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 473bffa79..dfae64ffd 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -402,27 +402,15 @@ RedirectEngine.prototype.resourceContentFromName = function(name, mime) { // TODO: combine same key-redirect pairs into a single regex. RedirectEngine.prototype.resourcesFromString = function(text) { - var textEnd = text.length; - var lineBeg = 0, lineEnd; - var line, fields, encoded; - var reNonEmptyLine = /\S/; + var line, fields, encoded, + reNonEmptyLine = /\S/, + lineIter = new µBlock.LineIterator(text); this.resources = new Map(); - while ( lineBeg < textEnd ) { - lineEnd = text.indexOf('\n', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = text.indexOf('\r', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = textEnd; - } - } - line = text.slice(lineBeg, lineEnd); - lineBeg = lineEnd + 1; - - if ( line.startsWith('#') ) { - continue; - } + while ( lineIter.eot() === false ) { + line = lineIter.next(); + if ( line.startsWith('#') ) { continue; } if ( fields === undefined ) { fields = line.trim().split(/\s+/); diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js index f52002e67..17e8f9915 100644 --- a/src/js/reverselookup-worker.js +++ b/src/js/reverselookup-worker.js @@ -1,7 +1,7 @@ /******************************************************************************* - uBlock - a browser extension to block requests. - Copyright (C) 2015 Raymond Hill + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -43,8 +43,8 @@ var fromNetFilter = function(details) { var lists = []; var compiledFilter = details.compiledFilter; var entry, content, pos, c; - for ( var path in listEntries ) { - entry = listEntries[path]; + for ( var assetKey in listEntries ) { + entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } @@ -173,11 +173,11 @@ var fromCosmeticFilter = function(details) { ); } - var re, path, entry; + var re, assetKey, entry; for ( var candidate in candidates ) { re = candidates[candidate]; - for ( path in listEntries ) { - entry = listEntries[path]; + for ( assetKey in listEntries ) { + entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } @@ -206,7 +206,7 @@ var reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/; /******************************************************************************/ -onmessage = function(e) { +onmessage = function(e) { // jshint ignore:line var msg = e.data; switch ( msg.what ) { @@ -215,7 +215,7 @@ onmessage = function(e) { break; case 'setList': - listEntries[msg.details.path] = msg.details; + listEntries[msg.details.assetKey] = msg.details; break; case 'fromNetFilter': diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 79af14e7c..c18bfba66 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -1,7 +1,7 @@ /******************************************************************************* - uBlock - a browser extension to block requests. - Copyright (C) 2015 Raymond Hill + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,14 +19,12 @@ Home: https://github.com/gorhill/uBlock */ -/* global µBlock */ +'use strict'; /******************************************************************************/ µBlock.staticFilteringReverseLookup = (function() { -'use strict'; - /******************************************************************************/ var worker = null; @@ -77,16 +75,16 @@ var initWorker = function(callback) { var countdown = 0; var onListLoaded = function(details) { - var entry = entries[details.path]; + var entry = entries[details.assetKey]; // https://github.com/gorhill/uBlock/issues/536 - // Use path string when there is no filter list title. + // Use assetKey when there is no filter list title. worker.postMessage({ what: 'setList', details: { - path: details.path, - title: entry.title || details.path, + assetKey: details.assetKey, + title: entry.title || details.assetKey, supportURL: entry.supportURL, content: details.content } @@ -99,18 +97,18 @@ var initWorker = function(callback) { }; var µb = µBlock; - var path, entry; + var listKey, entry; - for ( path in µb.remoteBlacklists ) { - if ( µb.remoteBlacklists.hasOwnProperty(path) === false ) { + for ( listKey in µb.availableFilterLists ) { + if ( µb.availableFilterLists.hasOwnProperty(listKey) === false ) { continue; } - entry = µb.remoteBlacklists[path]; - if ( entry.off === true ) { - continue; - } - entries[path] = { - title: path !== µb.userFiltersPath ? entry.title : vAPI.i18n('1pPageName'), + entry = µb.availableFilterLists[listKey]; + if ( entry.off === true ) { continue; } + entries[listKey] = { + title: listKey !== µb.userFiltersPath ? + entry.title : + vAPI.i18n('1pPageName'), supportURL: entry.supportURL || '' }; countdown += 1; @@ -121,8 +119,8 @@ var initWorker = function(callback) { return; } - for ( path in entries ) { - µb.getCompiledFilterList(path, onListLoaded); + for ( listKey in entries ) { + µb.getCompiledFilterList(listKey, onListLoaded); } }; diff --git a/src/js/scriptlets/subscriber.js b/src/js/scriptlets/subscriber.js index 9c525de0d..887e95db8 100644 --- a/src/js/scriptlets/subscriber.js +++ b/src/js/scriptlets/subscriber.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2015-2016 Raymond Hill + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,6 +21,8 @@ /* global vAPI, HTMLDocument */ +'use strict'; + /******************************************************************************/ // Injected into specific web pages, those which have been pre-selected @@ -30,8 +32,6 @@ (function() { -'use strict'; - /******************************************************************************/ // https://github.com/chrisaljoudi/uBlock/issues/464 @@ -100,7 +100,8 @@ var onAbpLinkClicked = function(ev) { 'scriptlets', { what: 'selectFilterLists', - switches: [ { location: location, off: false } ] + keys: [ location ], + append: true }, onListsSelectionDone ); diff --git a/src/js/settings.js b/src/js/settings.js index 6e75aa158..165829559 100644 --- a/src/js/settings.js +++ b/src/js/settings.js @@ -56,7 +56,10 @@ var handleImportFilePicker = function() { if ( typeof userData.netWhitelist !== 'string' ) { throw 'Invalid'; } - if ( typeof userData.filterLists !== 'object' ) { + if ( + typeof userData.filterLists !== 'object' && + Array.isArray(userData.selectedFilterLists) === false + ) { throw 'Invalid'; } } diff --git a/src/js/start.js b/src/js/start.js index 12a583db4..2ccd4fc8d 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ var µb = µBlock; vAPI.app.onShutdown = function() { µb.staticFilteringReverseLookup.shutdown(); - µb.assetUpdater.shutdown(); + µb.assets.updateStop(); µb.staticNetFilteringEngine.reset(); µb.cosmeticFilteringEngine.reset(); µb.sessionFirewall.reset(); @@ -58,14 +58,8 @@ vAPI.app.onShutdown = function() { var onAllReady = function() { // https://github.com/chrisaljoudi/uBlock/issues/184 // Check for updates not too far in the future. - µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb)); - µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb)); - µb.assetUpdater.onAssetUpdated.addEventListener(µb.assetUpdatedHandler.bind(µb)); - µb.assets.onAssetRemoved.addListener(µb.assetCacheRemovedHandler.bind(µb)); - - // Important: remove barrier to remote fetching, this was useful only - // for launch time. - µb.assets.remoteFetchBarrier -= 1; + µb.assets.addObserver(µb.assetObserver.bind(µb)); + µb.scheduleAssetUpdater(µb.userSettings.autoUpdate ? 7 * 60 * 1000 : 0); // vAPI.cloud is optional. if ( µb.cloudStorageSupported ) { @@ -129,7 +123,7 @@ var onSelfieReady = function(selfie) { return false; } - µb.remoteBlacklists = selfie.filterLists; + µb.availableFilterLists = selfie.availableFilterLists; µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); µb.redirectEngine.fromSelfie(selfie.redirectEngine); µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine); @@ -157,12 +151,6 @@ var onUserSettingsReady = function(fetched) { fromFetch(userSettings, fetched); - // https://github.com/chrisaljoudi/uBlock/issues/426 - // Important: block remote fetching for when loading assets at launch - // time. - µb.assets.autoUpdate = userSettings.autoUpdate; - µb.assets.autoUpdateDelay = µb.updateAssetsEvery; - if ( µb.privacySettingsSupported ) { vAPI.browserSettings.set({ 'hyperlinkAuditing': !userSettings.hyperlinkAuditingDisabled, @@ -192,7 +180,7 @@ var onUserSettingsReady = function(fetched) { var onSystemSettingsReady = function(fetched) { var mustSaveSystemSettings = false; if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { - µb.assets.purge(/^cache:\/\/compiled-/); + µb.assets.remove(/^compiled\//); mustSaveSystemSettings = true; } if ( fetched.selfieMagic !== µb.systemSettings.selfieMagic ) { @@ -254,9 +242,6 @@ var fromFetch = function(to, fetched) { /******************************************************************************/ var onAdminSettingsRestored = function() { - // Forbid remote fetching of assets - µb.assets.remoteFetchBarrier += 1; - var fetchableProps = { 'compiledMagic': '', 'dynamicFilteringString': 'behind-the-scene * 3p noop\nbehind-the-scene * 3p-frame noop', diff --git a/src/js/storage.js b/src/js/storage.js index db079f5ac..c58791948 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uBlock */ -/* global YaMD5, objectAssign, punycode, publicSuffixList */ +/* global objectAssign, punycode, publicSuffixList */ 'use strict'; @@ -104,6 +104,12 @@ case 'string': out[name] = value; break; + case 'number': + out[name] = parseInt(value, 10); + if ( isNaN(out[name]) ) { + out[name] = this.hiddenSettingsDefault[name]; + } + break; default: break; } @@ -151,70 +157,119 @@ this.netWhitelistModifyTime = Date.now(); }; -/******************************************************************************/ +/******************************************************************************* -// This will remove all unused filter list entries from -// µBlock.remoteBlacklists`. This helps reduce the size of backup files. + TODO(seamless migration): + The code related to 'remoteBlacklist' can be removed when I am confident + all users have moved to a version of uBO which no longer depends on + the property 'remoteBlacklists, i.e. v1.11 and beyond. -µBlock.extractSelectedFilterLists = function(callback) { +**/ + +µBlock.loadSelectedFilterLists = function(callback) { var µb = this; - - var onBuiltinListsLoaded = function(details) { - var builtin; - try { - builtin = JSON.parse(details.content); - } catch (e) { - builtin = {}; + vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists' ], function(bin) { + if ( !bin || !bin.selectedFilterLists && !bin.remoteBlacklists ) { + return callback(); } - - var result = JSON.parse(JSON.stringify(µb.remoteBlacklists)); - var entry, builtinPath, defaultState; - - for ( var path in result ) { - if ( result.hasOwnProperty(path) === false ) { - continue; - } - entry = result[path]; - // https://github.com/gorhill/uBlock/issues/277 - // uBlock's filter lists are always enabled by default, so we - // have to include in backup only those which are turned off. - if ( path.startsWith('assets/ublock/') ) { - if ( entry.off !== true ) { - delete result[path]; - } - continue; - } - builtinPath = path.replace(/^assets\/thirdparties\//, ''); - defaultState = builtin.hasOwnProperty(builtinPath) === false || - builtin[builtinPath].off === true; - if ( entry.off === true && entry.off === defaultState ) { - delete result[path]; - } + var listKeys = []; + if ( bin.selectedFilterLists ) { + listKeys = bin.selectedFilterLists; } - - callback(result); - }; - - // https://github.com/gorhill/uBlock/issues/63 - // Get built-in block lists: this will help us determine whether a - // specific list must be included in the result. - this.loadAndPatchStockFilterLists(onBuiltinListsLoaded); + if ( bin.remoteBlacklists ) { + var oldListKeys = µb.newListKeysFromOldData(bin.remoteBlacklists); + if ( oldListKeys.sort().join() !== listKeys.sort().join() ) { + listKeys = oldListKeys; + µb.saveSelectedFilterLists(listKeys); + } + // TODO(seamless migration): + // Uncomment when all have moved to v1.11 and beyond. + //vAPI.storage.remove('remoteBlacklists'); + } + callback(listKeys); + }); }; +µBlock.saveSelectedFilterLists = function(listKeys, append) { + var µb = this; + var save = function(keys) { + var bin = { + selectedFilterLists: keys, + remoteBlacklists: µb.oldDataFromNewListKeys(keys) + }; + vAPI.storage.set(bin); + }; + if ( append ) { + this.loadSelectedFilterLists(function(keys) { + listKeys = listKeys.concat(keys || []); + save(listKeys); + }); + } else { + save(listKeys); + } +}; + +// TODO(seamless migration): +// Remove when all have moved to v1.11 and beyond. +// >>>>>>>> +µBlock.newListKeysFromOldData = function(oldLists) { + var aliases = this.assets.listKeyAliases, + listKeys = [], newKey; + for ( var oldKey in oldLists ) { + if ( oldLists[oldKey].off !== true ) { + newKey = aliases[oldKey]; + listKeys.push(newKey ? newKey : oldKey); + } + } + return listKeys; +}; + +µBlock.oldDataFromNewListKeys = function(selectedFilterLists) { + var µb = this, + remoteBlacklists = {}; + var reverseAliases = Object.keys(this.assets.listKeyAliases).reduce( + function(a, b) { + a[µb.assets.listKeyAliases[b]] = b; return a; + }, + {} + ); + remoteBlacklists = selectedFilterLists.reduce( + function(a, b) { + a[reverseAliases[b] || b] = { off: false }; + return a; + }, + {} + ); + remoteBlacklists = Object.keys(µb.assets.listKeyAliases).reduce( + function(a, b) { + var aliases = µb.assets.listKeyAliases; + if ( + b.startsWith('assets/') && + aliases[b] !== 'public_suffix_list.dat' && + aliases[b] !== 'ublock-resources' && + !a[b] + ) { + a[b] = { off: true }; + } + return a; + }, + remoteBlacklists + ); + return remoteBlacklists; +}; +// <<<<<<<< + /******************************************************************************/ µBlock.saveUserFilters = function(content, callback) { // https://github.com/gorhill/uBlock/issues/1022 // Be sure to end with an empty line. content = content.trim(); - if ( content !== '' ) { - content += '\n'; - } + if ( content !== '' ) { content += '\n'; } this.assets.put(this.userFiltersPath, content, callback); + this.removeCompiledFilterList(this.userFiltersPath); }; -/******************************************************************************/ - µBlock.loadUserFilters = function(callback) { return this.assets.get(this.userFiltersPath, callback); }; @@ -222,25 +277,23 @@ /******************************************************************************/ µBlock.appendUserFilters = function(filters) { - if ( filters.length === 0 ) { - return; - } + if ( filters.length === 0 ) { return; } var µb = this; var onSaved = function() { - var compiledFilters = µb.compileFilters(filters); - var snfe = µb.staticNetFilteringEngine; - var cfe = µb.cosmeticFilteringEngine; - var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; - var discardedCount = snfe.discardedCount + cfe.discardedCount; + var compiledFilters = µb.compileFilters(filters), + snfe = µb.staticNetFilteringEngine, + cfe = µb.cosmeticFilteringEngine, + acceptedCount = snfe.acceptedCount + cfe.acceptedCount, + discardedCount = snfe.discardedCount + cfe.discardedCount; µb.applyCompiledFilters(compiledFilters, true); - var entry = µb.remoteBlacklists[µb.userFiltersPath]; - var deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; - var deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); + var entry = µb.availableFilterLists[µb.userFiltersPath], + deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount, + deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); entry.entryCount += deltaEntryCount; entry.entryUsedCount += deltaEntryUsedCount; - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); µb.staticNetFilteringEngine.freeze(); µb.redirectEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); @@ -248,9 +301,7 @@ }; var onLoaded = function(details) { - if ( details.error ) { - return; - } + if ( details.error ) { return; } // https://github.com/chrisaljoudi/uBlock/issues/976 // If we reached this point, the filter quite probably needs to be // added for sure: do not try to be too smart, trying to avoid @@ -263,168 +314,196 @@ /******************************************************************************/ -µBlock.getAvailableLists = function(callback) { - var availableLists = {}; - var relocationMap = {}; +µBlock.listKeysFromCustomFilterLists = function(raw) { + var out = {}; + var reIgnore = /^[!#]|[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/, + lineIter = new this.LineIterator(raw), + location; + while ( lineIter.eot() === false ) { + location = lineIter.next().trim(); + if ( location === '' || reIgnore.test(location) ) { continue; } + out[location] = true; + } + return Object.keys(out); +}; - var fixLocation = function(location) { - // https://github.com/chrisaljoudi/uBlock/issues/418 - // We now support built-in external filter lists - if ( /^https?:/.test(location) === false ) { - location = 'assets/thirdparties/' + location; +/******************************************************************************/ + +µBlock.autoSelectRegionalFilterLists = function(lists) { + var lang = self.navigator.language.slice(0, 2), + selectedListKeys = [], + list; + for ( var key in lists ) { + if ( lists.hasOwnProperty(key) === false ) { continue; } + list = lists[key]; + if ( list.off !== true ) { + selectedListKeys.push(key); + continue; } - return location; + if ( list.lang === lang ) { + selectedListKeys.push(key); + list.off = false; + } + } + return selectedListKeys; +}; + +/******************************************************************************/ + +µBlock.changeExternalFilterLists = function(before, after) { + var µb = µBlock; + var onLoaded = function(keys) { + var fullDict = new Set(keys || []), + mustSave = false, + oldKeys = µb.listKeysFromCustomFilterLists(before), + oldDict = new Set(oldKeys), + newKeys = µb.listKeysFromCustomFilterLists(after), + newDict = new Set(newKeys), + i, key; + i = oldKeys.length; + while ( i-- ) { + key = oldKeys[i]; + if ( fullDict.has(key) && !newDict.has(key) ) { + fullDict.delete(key); + mustSave = true; + } + } + i = newKeys.length; + while ( i-- ) { + key = newKeys[i]; + if ( !fullDict.has(key) && !oldDict.has(key) ) { + fullDict.add(key); + mustSave = true; + } + } + if ( mustSave ) { + µb.saveSelectedFilterLists(µb.setToArray(fullDict)); + } + }; + this.loadSelectedFilterLists(onLoaded); +}; + +/******************************************************************************/ + +µBlock.getAvailableLists = function(callback) { + var µb = this, + oldAvailableLists = {}, + newAvailableLists = {}; + + // User filter list. + newAvailableLists[this.userFiltersPath] = { + group: 'default', + title: vAPI.i18n('1pPageName') }; - // selected lists - var onSelectedListsLoaded = function(store) { - var µb = µBlock; - var lists = store.remoteBlacklists; - var locations = Object.keys(lists); - var location, availableEntry, storedEntry; - var off; + // Custom filter lists. + var importedListKeys = this.listKeysFromCustomFilterLists(µb.userSettings.externalLists), + i = importedListKeys.length, listKey, entry; + while ( i-- ) { + listKey = importedListKeys[i]; + entry = { + content: 'filters', + contentURL: importedListKeys[i], + external: true, + group: 'custom', + submitter: 'user', + title: '' + }; + newAvailableLists[listKey] = entry; + this.assets.registerAssetSource(listKey, entry); + } - while ( (location = locations.pop()) ) { - storedEntry = lists[location]; - off = storedEntry.off === true; - // New location? - if ( relocationMap.hasOwnProperty(location) ) { - µb.purgeFilterList(location); - location = relocationMap[location]; - if ( off && lists.hasOwnProperty(location) ) { - off = lists[location].off === true; - } - } - availableEntry = availableLists[location]; - if ( availableEntry === undefined ) { - µb.purgeFilterList(location); + // Final steps: + // - reuse existing list metadata if any; + // - unregister unreferenced imported filter lists if any. + var finalize = function() { + var assetKey, newEntry, oldEntry; + + // Reuse existing metadata. + for ( assetKey in oldAvailableLists ) { + oldEntry = oldAvailableLists[assetKey]; + newEntry = newAvailableLists[assetKey]; + if ( newEntry === undefined ) { + µb.removeFilterList(assetKey); continue; } - availableEntry.off = off; - if ( typeof availableEntry.homeURL === 'string' ) { - µb.assets.setHomeURL(location, availableEntry.homeURL); + if ( oldEntry.entryCount !== undefined ) { + newEntry.entryCount = oldEntry.entryCount; } - if ( storedEntry.entryCount !== undefined ) { - availableEntry.entryCount = storedEntry.entryCount; - } - if ( storedEntry.entryUsedCount !== undefined ) { - availableEntry.entryUsedCount = storedEntry.entryUsedCount; + if ( oldEntry.entryUsedCount !== undefined ) { + newEntry.entryUsedCount = oldEntry.entryUsedCount; } // This may happen if the list name was pulled from the list // content. // https://github.com/chrisaljoudi/uBlock/issues/982 // There is no guarantee the title was successfully extracted from // the list content. - if ( availableEntry.title === '' && - typeof storedEntry.title === 'string' && - storedEntry.title !== '' + if ( + newEntry.title === '' && + typeof oldEntry.title === 'string' && + oldEntry.title !== '' ) { - availableEntry.title = storedEntry.title; + newEntry.title = oldEntry.title; } } - // https://github.com/gorhill/uBlock/issues/747 - if ( µb.firstInstall ) { - µb.autoSelectFilterLists(availableLists); + // Remove unreferenced imported filter lists. + var dict = new Set(importedListKeys); + for ( assetKey in newAvailableLists ) { + newEntry = newAvailableLists[assetKey]; + if ( newEntry.submitter !== 'user' ) { continue; } + if ( dict.has(assetKey) ) { continue; } + delete newAvailableLists[assetKey]; + µb.assets.unregisterAssetSource(assetKey); + µb.removeFilterList(assetKey); } - - callback(availableLists); }; - // built-in lists - var onBuiltinListsLoaded = function(details) { - var location, locations; - try { - locations = JSON.parse(details.content); - } catch (e) { - locations = {}; - } - var entry; - for ( location in locations ) { - if ( locations.hasOwnProperty(location) === false ) { - continue; + // Selected lists. + var onSelectedListsLoaded = function(keys) { + var listKey; + // No user lists data means use default settings. + if ( Array.isArray(keys) ) { + var listKeySet = new Set(keys); + for ( listKey in newAvailableLists ) { + if ( newAvailableLists.hasOwnProperty(listKey) ) { + newAvailableLists[listKey].off = !listKeySet.has(listKey); + } } - entry = locations[location]; - location = fixLocation(location); - // Migrate obsolete location to new location, if any - if ( typeof entry.oldLocation === 'string' ) { - entry.oldLocation = fixLocation(entry.oldLocation); - relocationMap[entry.oldLocation] = location; - } - availableLists[location] = entry; + } else if ( µb.firstInstall ) { + µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(newAvailableLists)); } - // Now get user's selection of lists - vAPI.storage.get( - { 'remoteBlacklists': availableLists }, - onSelectedListsLoaded - ); + finalize(); + callback(newAvailableLists); }; - // permanent lists - var location; - var lists = this.permanentLists; - for ( location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; + // Built-in filter lists. + var onBuiltinListsLoaded = function(entries) { + for ( var assetKey in entries ) { + if ( entries.hasOwnProperty(assetKey) === false ) { continue; } + entry = entries[assetKey]; + if ( entry.content !== 'filters' ) { continue; } + newAvailableLists[assetKey] = objectAssign({}, entry); } - availableLists[location] = lists[location]; - } - // custom lists - var c; - var locations = this.userSettings.externalLists.split('\n'); - for ( var i = 0; i < locations.length; i++ ) { - location = locations[i].trim(); - c = location.charAt(0); - if ( location === '' || c === '!' || c === '#' ) { - continue; - } - // Coarse validation - if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) { - continue; - } - availableLists[location] = { - title: '', - group: 'custom', - external: true - }; - } + // Load set of currently selected filter lists. + µb.loadSelectedFilterLists(onSelectedListsLoaded); + }; - // get built-in block lists. - this.loadAndPatchStockFilterLists(onBuiltinListsLoaded); + // Available lists previously computed. + var onOldAvailableListsLoaded = function(bin) { + oldAvailableLists = bin && bin.availableFilterLists || {}; + µb.assets.metadata(onBuiltinListsLoaded); + }; + + // Load previously saved available lists -- these contains data + // computed at run-time, we will reuse this data if possible. + vAPI.storage.get('availableFilterLists', onOldAvailableListsLoaded); }; /******************************************************************************/ -µBlock.autoSelectFilterLists = function(lists) { - var lang = self.navigator.language.slice(0, 2), - list; - for ( var path in lists ) { - if ( lists.hasOwnProperty(path) === false ) { - continue; - } - list = lists[path]; - if ( list.off !== true ) { - continue; - } - if ( list.lang === lang ) { - list.off = false; - } - } -}; - -/******************************************************************************/ - -µBlock.createShortUniqueId = function(path) { - var md5 = YaMD5.hashStr(path); - return md5.slice(0, 4) + md5.slice(-4); -}; - -µBlock.createShortUniqueId.idLength = 8; - -/******************************************************************************/ - // This is used to be re-entrancy resistant. µBlock.loadingFilterLists = false; @@ -444,18 +523,11 @@ callback = this.noopFunc; } - // Never fetch from remote servers when we load filter lists: this has to - // be as fast as possible. - µb.assets.remoteFetchBarrier += 1; - var onDone = function() { - // Remove barrier to remote fetching - µb.assets.remoteFetchBarrier -= 1; - µb.staticNetFilteringEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); µb.redirectEngine.freeze(); - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); //quickProfiler.stop(0); @@ -473,15 +545,15 @@ var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; var discardedCount = snfe.discardedCount + cfe.discardedCount; µb.applyCompiledFilters(compiled, path === µb.userFiltersPath); - if ( µb.remoteBlacklists.hasOwnProperty(path) ) { - var entry = µb.remoteBlacklists[path]; + if ( µb.availableFilterLists.hasOwnProperty(path) ) { + var entry = µb.availableFilterLists[path]; entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; entry.entryUsedCount = entry.entryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); } }; var onCompiledListLoaded = function(details) { - applyCompiledFilters(details.path, details.content); + applyCompiledFilters(details.assetKey, details.content); filterlistsCount -= 1; if ( filterlistsCount === 0 ) { onDone(); @@ -489,7 +561,7 @@ }; var onFilterListsReady = function(lists) { - µb.remoteBlacklists = lists; + µb.availableFilterLists = lists; µb.redirectEngine.reset(); µb.cosmeticFilteringEngine.reset(); @@ -502,14 +574,10 @@ // This happens for assets which do not exist, ot assets with no // content. var toLoad = []; - for ( var path in lists ) { - if ( lists.hasOwnProperty(path) === false ) { - continue; - } - if ( lists[path].off ) { - continue; - } - toLoad.push(path); + for ( var assetKey in lists ) { + if ( lists.hasOwnProperty(assetKey) === false ) { continue; } + if ( lists[assetKey].off ) { continue; } + toLoad.push(assetKey); } filterlistsCount = toLoad.length; if ( filterlistsCount === 0 ) { @@ -528,32 +596,17 @@ /******************************************************************************/ -µBlock.getCompiledFilterListPath = function(path) { - return 'cache://compiled-filter-list:' + this.createShortUniqueId(path); -}; - -/******************************************************************************/ - -µBlock.getCompiledFilterList = function(path, callback) { - var compiledPath = this.getCompiledFilterListPath(path); - var µb = this; +µBlock.getCompiledFilterList = function(assetKey, callback) { + var µb = this, + compiledPath = 'compiled/' + assetKey; var onRawListLoaded = function(details) { + details.assetKey = assetKey; if ( details.content === '' ) { callback(details); return; } - var listMeta = µb.remoteBlacklists[path]; - // https://github.com/gorhill/uBlock/issues/313 - // Always try to fetch the name if this is an external filter list. - if ( listMeta && (listMeta.title === '' || listMeta.group === 'custom') ) { - var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i); - if ( matches !== null ) { - listMeta.title = matches[1].trim(); - } - } - - //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); + µb.extractFilterListMetadata(assetKey, details.content); details.content = µb.compileFilters(details.content); µb.assets.put(compiledPath, details.content); callback(details); @@ -561,12 +614,10 @@ var onCompiledListLoaded = function(details) { if ( details.content === '' ) { - //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: no compiled version for "%s"', path); - µb.assets.get(path, onRawListLoaded); + µb.assets.get(assetKey, onRawListLoaded); return; } - //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: using compiled version for "%s"', path); - details.path = path; + details.assetKey = assetKey; callback(details); }; @@ -575,61 +626,70 @@ /******************************************************************************/ -µBlock.purgeCompiledFilterList = function(path) { - this.assets.purge(this.getCompiledFilterListPath(path)); +µBlock.extractFilterListMetadata = function(assetKey, raw) { + var listEntry = this.availableFilterLists[assetKey]; + if ( listEntry === undefined ) { return; } + // Metadata expected to be found at the top of content. + var head = raw.slice(0, 1024), + matches, v; + // https://github.com/gorhill/uBlock/issues/313 + // Always try to fetch the name if this is an external filter list. + if ( listEntry.title === '' || listEntry.group === 'custom' ) { + matches = head.match(/(?:^|\n)!\s*Title:([^\n]+)/i); + if ( matches !== null ) { + listEntry.title = matches[1].trim(); + } + } + // Extract update frequency information + matches = head.match(/(?:^|\n)![\t ]*Expires:[\t ]*([\d]+)[\t ]*days?/i); + if ( matches !== null ) { + v = Math.max(parseInt(matches[1], 10), 2); + if ( v !== listEntry.updateAfter ) { + this.assets.registerAssetSource(assetKey, { updateAfter: v }); + } + } }; /******************************************************************************/ -µBlock.purgeFilterList = function(path) { - this.purgeCompiledFilterList(path); - this.assets.purge(path); +µBlock.removeCompiledFilterList = function(assetKey) { + this.assets.remove('compiled/' + assetKey); +}; + +µBlock.removeFilterList = function(assetKey) { + this.removeCompiledFilterList(assetKey); + this.assets.remove(assetKey); }; /******************************************************************************/ µBlock.compileFilters = function(rawText) { - var rawEnd = rawText.length; var compiledFilters = []; // Useful references: // https://adblockplus.org/en/filter-cheatsheet // https://adblockplus.org/en/filters - var staticNetFilteringEngine = this.staticNetFilteringEngine; - var cosmeticFilteringEngine = this.cosmeticFilteringEngine; - var reIsWhitespaceChar = /\s/; - var reMaybeLocalIp = /^[\d:f]/; - var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/; - var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/; + var staticNetFilteringEngine = this.staticNetFilteringEngine, + cosmeticFilteringEngine = this.cosmeticFilteringEngine, + reIsWhitespaceChar = /\s/, + reMaybeLocalIp = /^[\d:f]/, + reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/, + reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/, + line, lineRaw, c, pos, + lineIter = new this.LineIterator(rawText); - var lineBeg = 0, lineEnd, currentLineBeg; - var line, lineRaw, c, pos; - - while ( lineBeg < rawEnd ) { - lineEnd = rawText.indexOf('\n', lineBeg); - if ( lineEnd === -1 ) { - lineEnd = rawText.indexOf('\r', lineBeg); - if ( lineEnd === -1 ) { - lineEnd = rawEnd; - } - } + while ( lineIter.eot() === false ) { + line = lineRaw = lineIter.next().trim(); // rhill 2014-04-18: The trim is important here, as without it there // could be a lingering `\r` which would cause problems in the // following parsing code. - line = lineRaw = rawText.slice(lineBeg, lineEnd).trim(); - currentLineBeg = lineBeg; - lineBeg = lineEnd + 1; - if ( line.length === 0 ) { - continue; - } + if ( line.length === 0 ) { continue; } // Strip comments c = line.charAt(0); - if ( c === '!' || c === '[' ) { - continue; - } + if ( c === '!' || c === '[' ) { continue; } // Parse or skip cosmetic filters // All cosmetic filters are caught here @@ -640,9 +700,7 @@ // Whatever else is next can be assumed to not be a cosmetic filter // Most comments start in first column - if ( c === '#' ) { - continue; - } + if ( c === '#' ) { continue; } // Catch comments somewhere on the line // Remove: @@ -663,15 +721,11 @@ // Ignore hosts file redirect configuration // 127.0.0.1 localhost // 255.255.255.255 broadcasthost - if ( reIsLocalhostRedirect.test(line) ) { - continue; - } + if ( reIsLocalhostRedirect.test(line) ) { continue; } line = line.replace(reLocalIp, '').trim(); } - if ( line.length === 0 ) { - continue; - } + if ( line.length === 0 ) { continue; } staticNetFilteringEngine.compile(line, compiledFilters); } @@ -699,55 +753,6 @@ /******************************************************************************/ -// `switches` contains the filter lists for which the switch must be revisited. - -µBlock.selectFilterLists = function(switches) { - switches = switches || {}; - - // Only the lists referenced by the switches are touched. - var filterLists = this.remoteBlacklists; - var entry, state, location; - var i = switches.length; - while ( i-- ) { - entry = switches[i]; - state = entry.off === true; - location = entry.location; - if ( filterLists.hasOwnProperty(location) === false ) { - if ( state !== true ) { - filterLists[location] = { off: state }; - } - continue; - } - if ( filterLists[location].off === state ) { - continue; - } - filterLists[location].off = state; - } - - vAPI.storage.set({ 'remoteBlacklists': filterLists }); -}; - -/******************************************************************************/ - -// Plain reload of all filters. - -µBlock.reloadAllFilters = function() { - var µb = this; - - // We are just reloading the filter lists: we do not want assets to update. - // TODO: probably not needed anymore, since filter lists are now always - // loaded without update => see `µb.assets.remoteFetchBarrier`. - this.assets.autoUpdate = false; - - var onFiltersReady = function() { - µb.assets.autoUpdate = µb.userSettings.autoUpdate; - }; - - this.loadFilterLists(onFiltersReady); -}; - -/******************************************************************************/ - µBlock.loadRedirectResources = function(callback) { var µb = this; @@ -762,40 +767,46 @@ callback(); }; - this.assets.get('assets/ublock/resources.txt', onResourcesLoaded); + this.assets.get('ublock-resources', onResourcesLoaded); }; /******************************************************************************/ µBlock.loadPublicSuffixList = function(callback) { - var µb = this; - var path = µb.pslPath; - var compiledPath = 'cache://compiled-publicsuffixlist'; + var µb = this, + assetKey = µb.pslAssetKey, + compiledAssetKey = 'compiled/' + assetKey; if ( typeof callback !== 'function' ) { callback = this.noopFunc; } var onRawListLoaded = function(details) { if ( details.content !== '' ) { - //console.debug('µBlock.loadPublicSuffixList/onRawListLoaded: compiling "%s"', path); - publicSuffixList.parse(details.content, punycode.toASCII); - µb.assets.put(compiledPath, JSON.stringify(publicSuffixList.toSelfie())); + µb.compilePublicSuffixList(details.content); } callback(); }; var onCompiledListLoaded = function(details) { if ( details.content === '' ) { - //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: no compiled version for "%s"', path); - µb.assets.get(path, onRawListLoaded); + µb.assets.get(assetKey, onRawListLoaded); return; } - //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: using compiled version for "%s"', path); publicSuffixList.fromSelfie(JSON.parse(details.content)); callback(); }; - this.assets.get(compiledPath, onCompiledListLoaded); + this.assets.get(compiledAssetKey, onCompiledListLoaded); +}; + +/******************************************************************************/ + +µBlock.compilePublicSuffixList = function(content) { + publicSuffixList.parse(content, punycode.toASCII); + this.assets.put( + 'compiled/' + this.pslAssetKey, + JSON.stringify(publicSuffixList.toSelfie()) + ); }; /******************************************************************************/ @@ -814,7 +825,7 @@ var selfie = { magic: µb.systemSettings.selfieMagic, publicSuffixList: publicSuffixList.toSelfie(), - filterLists: µb.remoteBlacklists, + availableFilterLists: µb.availableFilterLists, staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), redirectEngine: µb.redirectEngine.toSelfie(), cosmeticFilteringEngine: µb.cosmeticFilteringEngine.toSelfie() @@ -885,6 +896,13 @@ var bin = {}; var binNotEmpty = false; + // Allows an admin to set their own 'assets.json' file, with their own + // set of stock assets. + if ( typeof data.assetsBootstrapLocation === 'string' ) { + bin.assetsBootstrapLocation = data.assetsBootstrapLocation; + binNotEmpty = true; + } + if ( typeof data.userSettings === 'object' ) { for ( var name in µb.userSettings ) { if ( µb.userSettings.hasOwnProperty(name) === false ) { @@ -898,8 +916,13 @@ } } - if ( typeof data.filterLists === 'object' ) { - bin.remoteBlacklists = data.filterLists; + // 'selectedFilterLists' is an array of filter list tokens. Each token + // is a reference to an asset in 'assets.json'. + if ( Array.isArray(data.selectedFilterLists) ) { + bin.selectedFilterLists = data.selectedFilterLists; + binNotEmpty = true; + } else if ( typeof data.filterLists === 'object' ) { + bin.selectedFilterLists = µb.newListKeysFromOldData(data.filterLists); binNotEmpty = true; } @@ -939,203 +962,95 @@ /******************************************************************************/ -µBlock.updateStartHandler = function(callback) { - var µb = this; - var onListsReady = function(lists) { - var assets = {}; - for ( var location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - if ( lists[location].off ) { - continue; - } - assets[location] = true; +µBlock.scheduleAssetUpdater = (function() { + var timer, next = 0; + return function(updateDelay) { + if ( timer ) { + clearTimeout(timer); + timer = undefined; } - assets[µb.pslPath] = true; - assets['assets/ublock/resources.txt'] = true; - callback(assets); - }; - - this.getAvailableLists(onListsReady); -}; - -/******************************************************************************/ - -µBlock.assetUpdatedHandler = function(details) { - var path = details.path || ''; - if ( this.remoteBlacklists.hasOwnProperty(path) === false ) { - return; - } - var entry = this.remoteBlacklists[path]; - if ( entry.off ) { - return; - } - // Compile the list while we have the raw version in memory - //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); - this.assets.put( - this.getCompiledFilterListPath(path), - this.compileFilters(details.content) - ); -}; - -/******************************************************************************/ - -µBlock.updateCompleteHandler = function(details) { - var µb = this; - var updatedCount = details.updatedCount; - - // Assets are supposed to have been all updated, prevent fetching from - // remote servers. - µb.assets.remoteFetchBarrier += 1; - - var onFiltersReady = function() { - µb.assets.remoteFetchBarrier -= 1; - }; - - var onPSLReady = function() { - if ( updatedCount !== 0 ) { - //console.debug('storage.js > µBlock.updateCompleteHandler: reloading filter lists'); - µb.loadFilterLists(onFiltersReady); - } else { - onFiltersReady(); - } - }; - - if ( details.hasOwnProperty(this.pslPath) ) { - //console.debug('storage.js > µBlock.updateCompleteHandler: reloading PSL'); - this.loadPublicSuffixList(onPSLReady); - updatedCount -= 1; - } else { - onPSLReady(); - } -}; - -/******************************************************************************/ - -µBlock.assetCacheRemovedHandler = (function() { - var barrier = false; - - var handler = function(paths) { - if ( barrier ) { + if ( updateDelay === 0 ) { + next = 0; return; } - barrier = true; - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( this.remoteBlacklists.hasOwnProperty(path) ) { - //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path); - this.purgeCompiledFilterList(path); - continue; - } - if ( path === this.pslPath ) { - //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path); - this.assets.purge('cache://compiled-publicsuffixlist'); - continue; - } + var now = Date.now(); + // Use the new schedule if and only if it is earlier than the previous + // one. + if ( next !== 0 ) { + updateDelay = Math.min(updateDelay, Math.max(next - now, 0)); } - this.selfieManager.destroy(); - barrier = false; + next = now + updateDelay; + timer = vAPI.setTimeout(function() { + timer = undefined; + next = 0; + var µb = µBlock; + µb.assets.updateStart({ + delay: µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 120000 + }); + }, updateDelay); }; - - return handler; })(); /******************************************************************************/ -// https://github.com/gorhill/uBlock/issues/602 -// - Load and patch `filter-list.json` -// - Load and patch user's `remoteBlacklists` -// - Load and patch cached filter lists -// - Load and patch compiled filter lists -// -// Once enough time has passed to safely assume all uBlock Origin -// installations have been converted to the new stock filter lists, this code -// can be removed. - -µBlock.patchFilterLists = function(filterLists) { - var modified = false; - var oldListKey, newListKey, listEntry; - for ( var listKey in filterLists ) { - if ( filterLists.hasOwnProperty(listKey) === false ) { - continue; +µBlock.assetObserver = function(topic, details) { + // Do not update filter list if not in use. + if ( topic === 'before-asset-updated' ) { + if ( + this.availableFilterLists.hasOwnProperty(details.assetKey) && + this.availableFilterLists[details.assetKey].off === true + ) { + return false; } - oldListKey = listKey; - if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - oldListKey = 'assets/thirdparties/' + listKey; - if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - continue; - } - } - newListKey = this.oldListToNewListMap[oldListKey]; - // https://github.com/gorhill/uBlock/issues/668 - // https://github.com/gorhill/uBlock/issues/669 - // Beware: an entry for the new list key may already exists. If it is - // the case, leave it as is. - if ( newListKey !== '' && filterLists.hasOwnProperty(newListKey) === false ) { - listEntry = filterLists[listKey]; - listEntry.homeURL = undefined; - filterLists[newListKey] = listEntry; - } - delete filterLists[listKey]; - modified = true; + return; } - return modified; -}; -µBlock.loadAndPatchStockFilterLists = function(callback) { - var onStockListsLoaded = function(details) { - var µb = µBlock; - var stockLists; - try { - stockLists = JSON.parse(details.content); - } catch (e) { - stockLists = {}; + // Compile the list while we have the raw version in memory + if ( topic === 'after-asset-updated' ) { + var cached = typeof details.content === 'string' && details.content !== ''; + if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { + if ( cached ) { + if ( this.availableFilterLists[details.assetKey].off !== true ) { + this.extractFilterListMetadata( + details.assetKey, + details.content + ); + this.assets.put( + 'compiled/' + details.assetKey, + this.compileFilters(details.content) + ); + } + } else { + this.removeCompiledFilterList(details.assetKey); + } + } else if ( details.assetKey === this.pslAssetKey ) { + if ( cached ) { + this.compilePublicSuffixList(details.content); + } + } else if ( details.assetKey === 'ublock-resources' ) { + if ( cached ) { + this.redirectEngine.resourcesFromString(details.content); + } } - - // Migrate assets affected by the change to their new name. - var reExternalURL = /^https?:\/\//; - var newListKey; - for ( var oldListKey in stockLists ) { - if ( stockLists.hasOwnProperty(oldListKey) === false ) { - continue; - } - // https://github.com/gorhill/uBlock/issues/708 - // Support migrating external stock filter lists as well. - if ( reExternalURL.test(oldListKey) === false ) { - oldListKey = 'assets/thirdparties/' + oldListKey; - } - if ( µb.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - continue; - } - newListKey = µb.oldListToNewListMap[oldListKey]; - if ( newListKey === '' ) { - continue; - } - // Rename cached asset to preserve content -- so it does not - // need to be fetched from remote server. - µb.assets.rename(oldListKey, newListKey); - µb.assets.purge(µb.getCompiledFilterListPath(oldListKey)); - } - µb.patchFilterLists(stockLists); - - // Stock lists information cascades into - // - In-memory user's selected filter lists, so we need to patch this. - µb.patchFilterLists(µb.remoteBlacklists); - - // Stock lists information cascades into - // - In-storage user's selected filter lists, so we need to patch this. - vAPI.storage.get('remoteBlacklists', function(bin) { - var userLists = bin.remoteBlacklists || {}; - if ( µb.patchFilterLists(userLists) ) { - µb.keyvalSetOne('remoteBlacklists', userLists); - } - details.content = JSON.stringify(stockLists); - callback(details); + vAPI.messaging.broadcast({ + what: 'assetUpdated', + key: details.assetKey, + cached: cached + }); - }; + return; + } - this.assets.get('assets/ublock/filter-lists.json', onStockListsLoaded); + // Reload all filter lists if needed. + if ( topic === 'after-assets-updated' ) { + if ( details.assetKeys.length !== 0 ) { + this.loadFilterLists(); + } + if ( this.userSettings.autoUpdate ) { + this.scheduleAssetUpdater(this.hiddenSettings.assetAutoUpdatePeriod * 3600000 || 25200000); + } else { + this.scheduleAssetUpdater(0); + } + return; + } }; diff --git a/src/js/ublock.js b/src/js/ublock.js index 528edb534..f765fbd74 100644 --- a/src/js/ublock.js +++ b/src/js/ublock.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -317,6 +317,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/, // Pre-change switch ( name ) { + case 'externalLists': + this.changeExternalFilterLists(us.externalLists, value); + break; case 'largeMediaSize': if ( typeof value !== 'number' ) { value = parseInt(value, 10) || 0; @@ -340,6 +343,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/, us.dynamicFilteringEnabled = true; } break; + case 'autoUpdate': + this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0); + break; case 'collapseBlocked': if ( value === false ) { this.cosmeticFilteringEngine.removeFromSelectorCache('*', 'net'); diff --git a/tools/make-assets.sh b/tools/make-assets.sh index fade452dc..cebea3f45 100755 --- a/tools/make-assets.sh +++ b/tools/make-assets.sh @@ -24,8 +24,6 @@ cp -R ../uAssets/thirdparties/www.malwaredomainlist.com $DES/thirdparti mkdir $DES/ublock cp -R ../uAssets/filters/* $DES/ublock/ -cp -R ./assets/ublock/filter-lists.json $DES/ublock/ - -cp ../uAssets/checksums/ublock0.txt $DES/checksums.txt +cp -R ./assets/assets.json $DES/ echo "done." diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh index 8d1f58911..eda30f299 100755 --- a/tools/make-chromium.sh +++ b/tools/make-chromium.sh @@ -5,7 +5,11 @@ echo "*** uBlock0.chromium: Creating web store package" echo "*** uBlock0.chromium: Copying files" -DES=dist/build/uBlock0.chromium +if [ "$1" = experimental ]; then + DES=dist/build/experimental/uBlock0.chromium +else + DES=dist/build/uBlock0.chromium +fi rm -rf $DES mkdir -p $DES