diff --git a/js/3p-filters.js b/js/3p-filters.js index 344379ad4..e5ad3d08b 100644 --- a/js/3p-filters.js +++ b/js/3p-filters.js @@ -380,7 +380,7 @@ var reloadAll = function(update) { /******************************************************************************/ var buttonApplyHandler = function() { - reloadAll(); + reloadAll(false); uDom('#buttonApply').toggleClass('enabled', false); }; diff --git a/js/abp-filters.js b/js/abp-filters.js index 46d79e73f..0d2b927d5 100644 --- a/js/abp-filters.js +++ b/js/abp-filters.js @@ -24,7 +24,7 @@ /******************************************************************************/ -µBlock.abpFilters = (function(){ +µBlock.netFilteringEngine = (function(){ /******************************************************************************/ @@ -137,6 +137,19 @@ var histogram = function(label, categories) { console.log('\tTotal buckets count: %d', total); }; */ +/******************************************************************************/ + +// Could be replaced with encodeURIComponent/decodeURIComponent, +// which seems faster on Firefox. +var encode = JSON.stringify; +var decode = JSON.parse; + +var cachedParseInt = parseInt; + +var atoi = function(s) { + return cachedParseInt(s, 10); +}; + /******************************************************************************* Filters family tree: @@ -187,10 +200,22 @@ FilterPlain.prototype.match = function(url, tokenBeg) { return url.substr(tokenBeg - this.tokenBeg, this.s.length) === this.s; }; +FilterPlain.prototype.fid = 'a'; + FilterPlain.prototype.toString = function() { return this.s; }; +FilterPlain.prototype.toSelfie = function() { + return this.s + '\t' + + this.tokenBeg; +}; + +FilterPlain.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterPlain(s.slice(0, pos), atoi(s.slice(pos + 1))); +}; + /******************************************************************************/ var FilterPlainHostname = function(s, tokenBeg, hostname) { @@ -204,10 +229,23 @@ FilterPlainHostname.prototype.match = function(url, tokenBeg) { url.substr(tokenBeg - this.tokenBeg, this.s.length) === this.s; }; +FilterPlainHostname.prototype.fid = 'ah'; + FilterPlainHostname.prototype.toString = function() { return this.s + '$domain=' + this.hostname; }; +FilterPlainHostname.prototype.toSelfie = function() { + return this.s + '\t' + + this.tokenBeg + '\t' + + this.hostname; +}; + +FilterPlainHostname.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterPlainHostname(args[0], atoi(args[1]), args[2]); +}; + /******************************************************************************/ var FilterPlainPrefix0 = function(s) { @@ -218,10 +256,20 @@ FilterPlainPrefix0.prototype.match = function(url, tokenBeg) { return url.substr(tokenBeg, this.s.length) === this.s; }; +FilterPlainPrefix0.prototype.fid = '0a'; + FilterPlainPrefix0.prototype.toString = function() { return this.s; }; +FilterPlainPrefix0.prototype.toSelfie = function() { + return this.s; +}; + +FilterPlainPrefix0.fromSelfie = function(s) { + return new FilterPlainPrefix0(s); +}; + /******************************************************************************/ var FilterPlainPrefix0Hostname = function(s, hostname) { @@ -234,10 +282,22 @@ FilterPlainPrefix0Hostname.prototype.match = function(url, tokenBeg) { url.substr(tokenBeg, this.s.length) === this.s; }; +FilterPlainPrefix0Hostname.prototype.fid = '0ah'; + FilterPlainPrefix0Hostname.prototype.toString = function() { return this.s + '$domain=' + this.hostname; }; +FilterPlainPrefix0Hostname.prototype.toSelfie = function() { + return this.s + '\t' + + this.hostname; +}; + +FilterPlainPrefix0Hostname.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterPlainPrefix0Hostname(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ var FilterPlainPrefix1 = function(s) { @@ -248,10 +308,20 @@ FilterPlainPrefix1.prototype.match = function(url, tokenBeg) { return url.substr(tokenBeg - 1, this.s.length) === this.s; }; +FilterPlainPrefix1.prototype.fid = '1a'; + FilterPlainPrefix1.prototype.toString = function() { return this.s; }; +FilterPlainPrefix1.prototype.toSelfie = function() { + return this.s; +}; + +FilterPlainPrefix1.fromSelfie = function(s) { + return new FilterPlainPrefix1(s); +}; + /******************************************************************************/ var FilterPlainPrefix1Hostname = function(s, hostname) { @@ -264,10 +334,22 @@ FilterPlainPrefix1Hostname.prototype.match = function(url, tokenBeg) { url.substr(tokenBeg - 1, this.s.length) === this.s; }; +FilterPlainPrefix1Hostname.prototype.fid = '1ah'; + FilterPlainPrefix1Hostname.prototype.toString = function() { return this.s + '$domain=' + this.hostname; }; +FilterPlainPrefix1Hostname.prototype.toSelfie = function() { + return this.s + '\t' + + this.hostname; +}; + +FilterPlainPrefix1Hostname.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterPlainPrefix1Hostname(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ var FilterPlainLeftAnchored = function(s) { @@ -278,10 +360,20 @@ FilterPlainLeftAnchored.prototype.match = function(url) { return url.slice(0, this.s.length) === this.s; }; +FilterPlainLeftAnchored.prototype.fid = '|a'; + FilterPlainLeftAnchored.prototype.toString = function() { return '|' + this.s; }; +FilterPlainLeftAnchored.prototype.toSelfie = function() { + return this.s; +}; + +FilterPlainLeftAnchored.fromSelfie = function(s) { + return new FilterPlainLeftAnchored(s); +}; + /******************************************************************************/ var FilterPlainLeftAnchoredHostname = function(s, hostname) { @@ -294,10 +386,22 @@ FilterPlainLeftAnchoredHostname.prototype.match = function(url) { url.slice(0, this.s.length) === this.s; }; +FilterPlainLeftAnchoredHostname.prototype.fid = '|ah'; + FilterPlainLeftAnchoredHostname.prototype.toString = function() { return '|' + this.s + '$domain=' + this.hostname; }; +FilterPlainLeftAnchoredHostname.prototype.toSelfie = function() { + return this.s + '\t' + + this.hostname; +}; + +FilterPlainLeftAnchoredHostname.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterPlainLeftAnchoredHostname(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ var FilterPlainRightAnchored = function(s) { @@ -308,10 +412,20 @@ FilterPlainRightAnchored.prototype.match = function(url) { return url.slice(-this.s.length) === this.s; }; +FilterPlainRightAnchored.prototype.fid = 'a|'; + FilterPlainRightAnchored.prototype.toString = function() { return this.s + '|'; }; +FilterPlainRightAnchored.prototype.toSelfie = function() { + return this.s; +}; + +FilterPlainRightAnchored.fromSelfie = function(s) { + return new FilterPlainRightAnchored(s); +}; + /******************************************************************************/ var FilterPlainRightAnchoredHostname = function(s, hostname) { @@ -324,10 +438,22 @@ FilterPlainRightAnchoredHostname.prototype.match = function(url) { url.slice(-this.s.length) === this.s; }; +FilterPlainRightAnchoredHostname.prototype.fid = 'a|h'; + FilterPlainRightAnchoredHostname.prototype.toString = function() { return this.s + '|$domain=' + this.hostname; }; +FilterPlainRightAnchoredHostname.prototype.toSelfie = function() { + return this.s + '\t' + + this.hostname; +}; + +FilterPlainRightAnchoredHostname.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterPlainRightAnchoredHostname(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ // With a single wildcard, regex is not optimal. @@ -335,11 +461,10 @@ FilterPlainRightAnchoredHostname.prototype.toString = function() { // http://jsperf.com/regexp-vs-indexof-abp-miss/3 // http://jsperf.com/regexp-vs-indexof-abp-hit/3 -var FilterSingleWildcard = function(s, tokenBeg) { +var FilterSingleWildcard = function(lSegment, rSegment, tokenBeg) { this.tokenBeg = tokenBeg; - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); + this.lSegment = lSegment; + this.rSegment = rSegment; }; FilterSingleWildcard.prototype.match = function(url, tokenBeg) { @@ -348,17 +473,29 @@ FilterSingleWildcard.prototype.match = function(url, tokenBeg) { url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0; }; +FilterSingleWildcard.prototype.fid = '*'; + FilterSingleWildcard.prototype.toString = function() { return this.lSegment + '*' + this.rSegment; }; +FilterSingleWildcard.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment + '\t' + + this.tokenBeg; +}; + +FilterSingleWildcard.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterSingleWildcard(args[0], args[1], atoi(args[2])); +}; + /******************************************************************************/ -var FilterSingleWildcardHostname = function(s, tokenBeg, hostname) { +var FilterSingleWildcardHostname = function(lSegment, rSegment, tokenBeg, hostname) { this.tokenBeg = tokenBeg; - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); + this.lSegment = lSegment; + this.rSegment = rSegment; this.hostname = hostname; }; @@ -369,16 +506,29 @@ FilterSingleWildcardHostname.prototype.match = function(url, tokenBeg) { url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0; }; +FilterSingleWildcardHostname.prototype.fid = '*h'; + FilterSingleWildcardHostname.prototype.toString = function() { return this.lSegment + '*' + this.rSegment + '$domain=' + this.hostname; }; +FilterSingleWildcardHostname.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment + '\t' + + this.tokenBeg + '\t' + + this.hostname; +}; + +FilterSingleWildcardHostname.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterSingleWildcardHostname(args[0], args[1], atoi(args[2]), args[3]); +}; + /******************************************************************************/ -var FilterSingleWildcardPrefix0 = function(s) { - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); +var FilterSingleWildcardPrefix0 = function(lSegment, rSegment) { + this.lSegment = lSegment; + this.rSegment = rSegment; }; FilterSingleWildcardPrefix0.prototype.match = function(url, tokenBeg) { @@ -386,16 +536,27 @@ FilterSingleWildcardPrefix0.prototype.match = function(url, tokenBeg) { url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0; }; +FilterSingleWildcardPrefix0.prototype.fid = '0*'; + FilterSingleWildcardPrefix0.prototype.toString = function() { return this.lSegment + '*' + this.rSegment; }; +FilterSingleWildcardPrefix0.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment; +}; + +FilterSingleWildcardPrefix0.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterSingleWildcardPrefix0(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ -var FilterSingleWildcardPrefix0Hostname = function(s, hostname) { - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); +var FilterSingleWildcardPrefix0Hostname = function(lSegment, rSegment, hostname) { + this.lSegment = lSegment; + this.rSegment = rSegment; this.hostname = hostname; }; @@ -405,21 +566,28 @@ FilterSingleWildcardPrefix0Hostname.prototype.match = function(url, tokenBeg) { url.indexOf(this.rSegment, tokenBeg + this.lSegment.length) > 0; }; +FilterSingleWildcardPrefix0Hostname.prototype.fid = '0*h'; + FilterSingleWildcardPrefix0Hostname.prototype.toString = function() { return this.lSegment + '*' + this.rSegment + '$domain=' + this.hostname; }; +FilterSingleWildcardPrefix0Hostname.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment + '\t' + + this.hostname; +}; + +FilterSingleWildcardPrefix0Hostname.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterSingleWildcardPrefix0Hostname(args[0], args[1], args[2]); +}; + /******************************************************************************/ -// With a single wildcard, regex is not optimal. -// See: -// http://jsperf.com/regexp-vs-indexof-abp-miss/3 -// http://jsperf.com/regexp-vs-indexof-abp-hit/3 - -var FilterSingleWildcardLeftAnchored = function(s) { - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); +var FilterSingleWildcardLeftAnchored = function(lSegment, rSegment) { + this.lSegment = lSegment; + this.rSegment = rSegment; }; FilterSingleWildcardLeftAnchored.prototype.match = function(url) { @@ -427,16 +595,27 @@ FilterSingleWildcardLeftAnchored.prototype.match = function(url) { url.indexOf(this.rSegment, this.lSegment.length) > 0; }; +FilterSingleWildcardLeftAnchored.prototype.fid = '|*'; + FilterSingleWildcardLeftAnchored.prototype.toString = function() { return '|' + this.lSegment + '*' + this.rSegment; }; +FilterSingleWildcardLeftAnchored.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment; +}; + +FilterSingleWildcardLeftAnchored.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterSingleWildcardLeftAnchored(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ -var FilterSingleWildcardLeftAnchoredHostname = function(s, hostname) { - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); +var FilterSingleWildcardLeftAnchoredHostname = function(lSegment, rSegment, hostname) { + this.lSegment = lSegment; + this.rSegment = rSegment; this.hostname = hostname; }; @@ -446,21 +625,28 @@ FilterSingleWildcardLeftAnchoredHostname.prototype.match = function(url) { url.indexOf(this.rSegment, this.lSegment.length) > 0; }; +FilterSingleWildcardLeftAnchoredHostname.prototype.fid = '|*h'; + FilterSingleWildcardLeftAnchoredHostname.prototype.toString = function() { return '|' + this.lSegment + '*' + this.rSegment + '$domain=' + this.hostname; }; +FilterSingleWildcardLeftAnchoredHostname.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment + '\t' + + this.hostname; +}; + +FilterSingleWildcardLeftAnchoredHostname.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterSingleWildcardLeftAnchoredHostname(args[0], args[1], args[2]); +}; + /******************************************************************************/ -// With a single wildcard, regex is not optimal. -// See: -// http://jsperf.com/regexp-vs-indexof-abp-miss/3 -// http://jsperf.com/regexp-vs-indexof-abp-hit/3 - -var FilterSingleWildcardRightAnchored = function(s) { - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); +var FilterSingleWildcardRightAnchored = function(lSegment, rSegment) { + this.lSegment = lSegment; + this.rSegment = rSegment; }; FilterSingleWildcardRightAnchored.prototype.match = function(url) { @@ -468,16 +654,27 @@ FilterSingleWildcardRightAnchored.prototype.match = function(url) { url.lastIndexOf(this.lSegment, url.length - this.rSegment.length - this.lSegment.length) >= 0; }; +FilterSingleWildcardRightAnchored.prototype.fid = '*|'; + FilterSingleWildcardRightAnchored.prototype.toString = function() { return this.lSegment + '*' + this.rSegment + '|'; }; +FilterSingleWildcardRightAnchored.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment; +}; + +FilterSingleWildcardRightAnchored.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterSingleWildcardRightAnchored(s.slice(0, pos), s.slice(pos + 1)); +}; + /******************************************************************************/ -var FilterSingleWildcardRightAnchoredHostname = function(s, hostname) { - var wcOffset = s.indexOf('*'); - this.lSegment = s.slice(0, wcOffset); - this.rSegment = s.slice(wcOffset + 1); +var FilterSingleWildcardRightAnchoredHostname = function(lSegment, rSegment, hostname) { + this.lSegment = lSegment; + this.rSegment = rSegment; this.hostname = hostname; }; @@ -487,10 +684,23 @@ FilterSingleWildcardRightAnchoredHostname.prototype.match = function(url) { url.lastIndexOf(this.lSegment, url.length - this.rSegment.length - this.lSegment.length) >= 0; }; +FilterSingleWildcardRightAnchoredHostname.prototype.fid = '*|h'; + FilterSingleWildcardRightAnchoredHostname.prototype.toString = function() { return this.lSegment + '*' + this.rSegment + '|$domain=' + this.hostname; }; +FilterSingleWildcardRightAnchoredHostname.prototype.toSelfie = function() { + return this.lSegment + '\t' + + this.rSegment + '\t' + + this.hostname; +}; + +FilterSingleWildcardRightAnchoredHostname.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterSingleWildcardRightAnchoredHostname(args[0], args[1], args[2]); +}; + /******************************************************************************/ // With many wildcards, a regex is best. @@ -509,10 +719,22 @@ FilterManyWildcards.prototype.match = function(url, tokenBeg) { return this.re.test(url.slice(tokenBeg - this.tokenBeg)); }; +FilterManyWildcards.prototype.fid = '*+'; + FilterManyWildcards.prototype.toString = function() { return this.s; }; +FilterManyWildcards.prototype.toSelfie = function() { + return this.s + '\t' + + this.tokenBeg; +}; + +FilterManyWildcards.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterManyWildcards(s.slice(0, pos), atoi(s.slice(pos + 1))); +}; + /******************************************************************************/ var FilterManyWildcardsHostname = function(s, tokenBeg, hostname) { @@ -527,29 +749,90 @@ FilterManyWildcardsHostname.prototype.match = function(url, tokenBeg) { this.re.test(url.slice(tokenBeg - this.tokenBeg)); }; +FilterManyWildcardsHostname.prototype.fid = '*+h'; + FilterManyWildcardsHostname.prototype.toString = function() { return this.s + '$domain=' + this.hostname; }; +FilterManyWildcardsHostname.prototype.toSelfie = function() { + return this.s + '\t' + + this.tokenBeg + '\t' + + this.hostname; +}; + +FilterManyWildcardsHostname.fromSelfie = function(s) { + var args = s.split('\t'); + return new FilterManyWildcardsHostname(args[0], atoi(args[1]), args[2]); +}; + +/******************************************************************************/ + +var FilterBucket = function(a, b) { + this.f = null; + this.filters = []; + if ( a !== undefined ) { + this.filters[0] = a; + if ( b !== undefined ) { + this.filters[1] = b; + } + } +}; + +FilterBucket.prototype.add = function(a) { + this.filters.push(a); +}; + +FilterBucket.prototype.match = function(url, tokenBeg) { + var filters = this.filters; + var i = filters.length; + while ( i-- ) { + if ( filters[i].match(url, tokenBeg) !== false ) { + this.f = filters[i]; + return true; + } + } + return false; +}; + +FilterBucket.prototype.fid = '[]'; + +FilterBucket.prototype.toString = function() { + if ( this.f !== null ) { + return this.f.toString(); + } + return ''; +}; + +FilterBucket.prototype.toSelfie = function() { + return this.filters.length.toString(); +}; + +FilterBucket.fromSelfie = function() { + return new FilterBucket(); +}; + /******************************************************************************/ var makeFilter = function(details, tokenBeg) { var s = details.f; var wcOffset = s.indexOf('*'); - if ( wcOffset > 0 ) { - if ( (/\*[^*]\*/).test(s) ) { + if ( wcOffset !== -1 ) { + if ( s.indexOf('*', wcOffset + 1) !== -1 ) { return details.anchor === 0 ? new FilterManyWildcards(s, tokenBeg) : null; } + var lSegment = s.slice(0, wcOffset); + var rSegment = s.slice(wcOffset + 1); if ( details.anchor < 0 ) { - return new FilterSingleWildcardLeftAnchored(s); + return new FilterSingleWildcardLeftAnchored(lSegment, rSegment); } if ( details.anchor > 0 ) { - return new FilterSingleWildcardRightAnchored(s); + return new FilterSingleWildcardRightAnchored(lSegment, rSegment); } if ( tokenBeg === 0 ) { - return new FilterSingleWildcardPrefix0(s); + return new FilterSingleWildcardPrefix0(lSegment, rSegment); } - return new FilterSingleWildcard(s, tokenBeg); + return new FilterSingleWildcard(lSegment, rSegment, tokenBeg); } if ( details.anchor < 0 ) { return new FilterPlainLeftAnchored(s); @@ -571,20 +854,22 @@ var makeFilter = function(details, tokenBeg) { var makeHostnameFilter = function(details, tokenBeg, hostname) { var s = details.f; var wcOffset = s.indexOf('*'); - if ( wcOffset > 0 ) { - if ( (/\*[^*]\*/).test(s) ) { + if ( wcOffset !== -1 ) { + if ( s.indexOf('*', wcOffset + 1) !== -1 ) { return details.anchor === 0 ? new FilterManyWildcardsHostname(s, tokenBeg, hostname) : null; } + var lSegment = s.slice(0, wcOffset); + var rSegment = s.slice(wcOffset + 1); if ( details.anchor < 0 ) { - return new FilterSingleWildcardLeftAnchoredHostname(s, hostname); + return new FilterSingleWildcardLeftAnchoredHostname(lSegment, rSegment, hostname); } if ( details.anchor > 0 ) { - return new FilterSingleWildcardRightAnchoredHostname(s, hostname); + return new FilterSingleWildcardRightAnchoredHostname(lSegment, rSegment, hostname); } if ( tokenBeg === 0 ) { - return new FilterSingleWildcardPrefix0Hostname(s, hostname); + return new FilterSingleWildcardPrefix0Hostname(lSegment, rSegment, hostname); } - return new FilterSingleWildcardHostname(s, tokenBeg, hostname); + return new FilterSingleWildcardHostname(lSegment, rSegment, tokenBeg, hostname); } if ( details.anchor < 0 ) { return new FilterPlainLeftAnchoredHostname(s, hostname); @@ -613,6 +898,7 @@ var badTokens = { 'com': true, 'http': true, 'https': true, + 'icon': true, 'images': true, 'img': true, 'js': true, @@ -661,6 +947,7 @@ var trimChar = function(s, c) { return s; }; +/******************************************************************************/ /******************************************************************************/ var FilterParser = function() { @@ -783,6 +1070,19 @@ FilterParser.prototype.parse = function(s) { s = s.slice(2); } + // options + var pos = s.indexOf('$'); + if ( pos > 0 ) { + this.fopts = s.slice(pos + 1); + s = s.slice(0, pos); + } + + // regex? (not supported) + if ( s.charAt(0) === '/' && s.slice(-1) === '/' ) { + this.unsupported = true; + return this; + } + // hostname anchoring if ( s.slice(0, 2) === '||' ) { this.hostname = true; @@ -795,13 +1095,6 @@ FilterParser.prototype.parse = function(s) { s = s.slice(1); } - // options - var pos = s.indexOf('$'); - if ( pos > 0 ) { - this.fopts = s.slice(pos + 1); - s = s.slice(0, pos); - } - // right-anchored if ( s.slice(-1) === '|' ) { this.anchor = 1; @@ -859,37 +1152,6 @@ FilterParser.prototype.parse = function(s) { /******************************************************************************/ /******************************************************************************/ -var FilterBucket = function(a, b) { - this.filters = [a, b]; - this.f = null; -}; - -FilterBucket.prototype.add = function(a) { - this.filters.push(a); -}; - -FilterBucket.prototype.match = function(url, tokenBeg) { - var filters = this.filters; - var i = filters.length; - while ( i-- ) { - if ( filters[i].match(url, tokenBeg) !== false ) { - this.f = filters[i]; - return true; - } - } - return false; -}; - -FilterBucket.prototype.toString = function() { - if ( this.f !== null ) { - return this.f.toString(); - } - return ''; -}; - -/******************************************************************************/ -/******************************************************************************/ - var FilterContainer = function() { this.reAnyToken = /[%0-9a-z]+/g; this.buckets = new Array(8); @@ -908,6 +1170,7 @@ FilterContainer.prototype.reset = function() { this.frozen = false; this.processedFilterCount = 0; this.acceptedCount = 0; + this.rejectedCount = 0; this.allowFilterCount = 0; this.blockFilterCount = 0; this.duplicateCount = 0; @@ -931,6 +1194,136 @@ FilterContainer.prototype.freeze = function() { /******************************************************************************/ +FilterContainer.prototype.toSelfie = function() { + var categoryToSelfie = function(dict) { + var selfie = []; + var bucket, ff, n, i, f; + for ( var k in dict ) { + if ( dict.hasOwnProperty(k) === false ) { + continue; + } + // We need to encode the key because there could be a `\n` or '\t' + // character in it, which would trip the code at parse time. + selfie.push('k2\t' + encode(k)); + bucket = dict[k]; + selfie.push(bucket.fid + '\t' + bucket.toSelfie()); + if ( bucket.fid !== '[]' ) { + continue; + } + ff = bucket.filters; + n = ff.length; + for ( i = 0; i < n; i++ ) { + f = ff[i]; + selfie.push(f.fid + '\t' + f.toSelfie()); + } + } + return selfie.join('\n'); + }; + + var categoriesToSelfie = function(dict) { + var selfie = []; + for ( var k in dict ) { + if ( dict.hasOwnProperty(k) === false ) { + continue; + } + // We need to encode the key because there could be a `\n` or '\t' + // character in it, which would trip the code at parse time. + selfie.push('k1\t' + encode(k)); + selfie.push(categoryToSelfie(dict[k])); + } + return selfie.join('\n'); + }; + + return { + processedFilterCount: this.processedFilterCount, + acceptedCount: this.acceptedCount, + rejectedCount: this.rejectedCount, + allowFilterCount: this.allowFilterCount, + blockFilterCount: this.blockFilterCount, + duplicateCount: this.duplicateCount, + categories: categoriesToSelfie(this.categories), + blockedAnyPartyHostnames: this.blockedAnyPartyHostnames.toSelfie(), + blocked3rdPartyHostnames: this.blocked3rdPartyHostnames.toSelfie() + }; +}; + +/******************************************************************************/ + +FilterContainer.prototype.fromSelfie = function(selfie) { + this.frozen = true; + this.processedFilterCount = selfie.processedFilterCount; + this.acceptedCount = selfie.acceptedCount; + this.rejectedCount = selfie.rejectedCount; + this.allowFilterCount = selfie.allowFilterCount; + this.blockFilterCount = selfie.blockFilterCount; + this.duplicateCount = selfie.duplicateCount; + this.blockedAnyPartyHostnames.fromSelfie(selfie.blockedAnyPartyHostnames); + this.blocked3rdPartyHostnames.fromSelfie(selfie.blocked3rdPartyHostnames); + + var factories = { + '[]': FilterBucket, + 'a': FilterPlain, + 'ah': FilterPlainHostname, + '0a': FilterPlainPrefix0, + '0ah': FilterPlainPrefix0Hostname, + '1a': FilterPlainPrefix1, + '1ah': FilterPlainPrefix1Hostname, + '|a': FilterPlainLeftAnchored, + '|ah': FilterPlainLeftAnchoredHostname, + 'a|': FilterPlainRightAnchored, + 'a|h': FilterPlainRightAnchoredHostname, + '*': FilterSingleWildcard, + '*h': FilterSingleWildcardHostname, + '0*': FilterSingleWildcardPrefix0, + '0*h': FilterSingleWildcardPrefix0Hostname, + '|*': FilterSingleWildcardLeftAnchored, + '|*h': FilterSingleWildcardLeftAnchoredHostname, + '*|': FilterSingleWildcardRightAnchored, + '*|h': FilterSingleWildcardRightAnchoredHostname, + '*+': FilterManyWildcards, + '*+h': FilterManyWildcardsHostname + }; + + var catKey, tokenKey; + var dict = this.categories, subdict; + var bucket = null; + var rawText = selfie.categories; + var rawEnd = rawText.length; + var lineBeg = 0, lineEnd; + var line, pos, what, factory; + while ( lineBeg < rawEnd ) { + lineEnd = rawText.indexOf('\n', lineBeg); + if ( lineEnd < 0 ) { + lineEnd = rawEnd; + } + line = rawText.slice(lineBeg, lineEnd); + lineBeg = lineEnd + 1; + pos = line.indexOf('\t'); + what = line.slice(0, pos); + if ( what === 'k1' ) { + catKey = decode(line.slice(pos + 1)); + subdict = dict[catKey] = {}; + bucket = null; + continue; + } + if ( what === 'k2' ) { + tokenKey = decode(line.slice(pos + 1)); + bucket = null; + continue; + } + factory = factories[what]; + if ( bucket === null ) { + bucket = subdict[tokenKey] = factory.fromSelfie(line.slice(pos + 1)); + continue; + } + // When token key is reused, it can't be anything + // else than FilterBucket + bucket.add(factory.fromSelfie(line.slice(pos + 1))); + } +}; + +/******************************************************************************/ + FilterContainer.prototype.toDomainBits = function(domain) { if ( domain === undefined ) { return 0; @@ -989,6 +1382,13 @@ FilterContainer.prototype.add = function(s) { var parsed = this.filterParser.parse(s); + // Ignore rules with other conditions for now + if ( parsed.unsupported ) { + this.rejectedCount += 1; + // console.log('µBlock> abp-filter.js/FilterContainer.add(): unsupported filter "%s"', s); + return false; + } + // Ignore element-hiding filters if ( parsed.elemHiding ) { return false; @@ -1002,12 +1402,6 @@ FilterContainer.prototype.add = function(s) { this.processedFilterCount += 1; - // Ignore rules with other conditions for now - if ( parsed.unsupported ) { - // console.log('µBlock> abp-filter.js/FilterContainer.add(): unsupported filter "%s"', s); - return false; - } - // Ignore optionless hostname rules, these will be taken care of by µBlock. if ( parsed.hostname && parsed.fopts === '' && parsed.action === BlockAction && reHostnameRule.test(parsed.f) ) { return false; @@ -1145,7 +1539,7 @@ FilterContainer.prototype.addToCategory = function(category, tokenKey, filter) { categoryBucket[tokenKey] = filter; return; } - if ( filterEntry instanceof FilterBucket ) { + if ( filterEntry.fid === '[]' ) { filterEntry.add(filter); return; } diff --git a/js/abp-hide-filters.js b/js/abp-hide-filters.js index d0dd1c1b5..4b7d2e9e1 100644 --- a/js/abp-hide-filters.js +++ b/js/abp-hide-filters.js @@ -24,13 +24,20 @@ /******************************************************************************/ -µBlock.abpHideFilters = (function(){ +µBlock.cosmeticFilteringEngine = (function(){ /******************************************************************************/ var µb = µBlock; +/******************************************************************************/ + +// Could be replaced with encodeURIComponent/decodeURIComponent, +// which seems faster on Firefox. +var encode = JSON.stringify; +var decode = JSON.parse; + /******************************************************************************/ /* var histogram = function(label, buckets) { @@ -86,6 +93,16 @@ FilterPlain.prototype.retrieve = function(s, out) { } }; +FilterPlain.prototype.fid = '#'; + +FilterPlain.prototype.toSelfie = function() { + return this.s; +}; + +FilterPlain.fromSelfie = function(s) { + return new FilterPlain(s); +}; + /******************************************************************************/ // Id- and class-based filters with extra selector stuff following. @@ -104,6 +121,50 @@ FilterPlainMore.prototype.retrieve = function(s, out) { } }; +FilterPlainMore.prototype.fid = '#+'; + +FilterPlainMore.prototype.toSelfie = function() { + return this.s; +}; + +FilterPlainMore.fromSelfie = function(s) { + return new FilterPlainMore(s); +}; + +/******************************************************************************/ + +var FilterBucket = function(a, b) { + this.f = null; + this.filters = []; + if ( a !== undefined ) { + this.filters[0] = a; + if ( b !== undefined ) { + this.filters[1] = b; + } + } +}; + +FilterBucket.prototype.add = function(a) { + this.filters.push(a); +}; + +FilterBucket.prototype.retrieve = function(s, out) { + var i = this.filters.length; + while ( i-- ) { + this.filters[i].retrieve(s, out); + } +}; + +FilterBucket.prototype.fid = '[]'; + +FilterBucket.prototype.toSelfie = function() { + return this.filters.length.toString(); +}; + +FilterBucket.fromSelfie = function() { + return new FilterBucket(); +}; + /******************************************************************************/ // Any selector specific to a hostname @@ -127,6 +188,17 @@ FilterHostname.prototype.retrieve = function(hostname, out) { } }; +FilterHostname.prototype.fid = 'h'; + +FilterHostname.prototype.toSelfie = function() { + return encode(this.s) + '\t' + this.hostname; +}; + +FilterHostname.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterHostname(decode(s.slice(0, pos)), s.slice(pos + 1)); +}; + /******************************************************************************/ // Any selector specific to an entity @@ -144,28 +216,15 @@ FilterEntity.prototype.retrieve = function(entity, out) { } }; -/******************************************************************************/ -/******************************************************************************/ +FilterEntity.prototype.fid = 'e'; -// TODO: evaluate the gain (if any) from avoiding the use of an array for when -// there are only two filters (or three, etc.). I suppose there is a specific -// number of filters below which using an array is more of an overhead than -// using a couple of property members. -// i.e. FilterBucket2, FilterBucket3, FilterBucketN. - -var FilterBucket = function(a, b) { - this.filters = [a, b]; +FilterEntity.prototype.toSelfie = function() { + return encode(this.s) + '\t' + this.entity; }; -FilterBucket.prototype.add = function(a) { - this.filters.push(a); -}; - -FilterBucket.prototype.retrieve = function(s, out) { - var i = this.filters.length; - while ( i-- ) { - this.filters[i].retrieve(s, out); - } +FilterEntity.fromSelfie = function(s) { + var pos = s.indexOf('\t'); + return new FilterEntity(decode(s.slice(0, pos)), s.slice(pos + 1)); }; /******************************************************************************/ @@ -419,6 +478,7 @@ FilterContainer.prototype.reset = function() { this.hostnameDonthide = {}; this.entityHide = {}; this.entityDonthide = {}; + // permanent // [class], [id] this.lowGenericFilters = {}; @@ -436,11 +496,12 @@ FilterContainer.prototype.reset = function() { this.highMediumGenericDonthideCount = 0; // everything else - this.highHighGenericHide = []; - this.highHighGenericDonthide = []; + this.highHighGenericHide = ''; + this.highHighGenericDonthide = ''; this.highHighGenericHideCount = 0; this.highHighGenericDonthideCount = 0; + // hostname, entity-based filters this.hostnameFilters = {}; this.entityFilters = {}; }; @@ -705,6 +766,120 @@ FilterContainer.prototype.freeze = function() { /******************************************************************************/ +FilterContainer.prototype.toSelfie = function() { + var selfieFromDict = function(dict) { + var selfie = []; + var bucket, ff, n, i, f; + for ( var k in dict ) { + if ( dict.hasOwnProperty(k) === false ) { + continue; + } + // We need to encode the key because there could be a `\n` + // character in it, which would trip the code at parse time. + selfie.push('k\t' + encode(k)); + bucket = dict[k]; + selfie.push(bucket.fid + '\t' + bucket.toSelfie()); + if ( bucket.fid !== '[]' ) { + continue; + } + ff = bucket.filters; + n = ff.length; + for ( i = 0; i < n; i++ ) { + f = ff[i]; + selfie.push(f.fid + '\t' + f.toSelfie()); + } + } + return selfie.join('\n'); + }; + + return { + acceptedCount: this.acceptedCount, + duplicateCount: this.duplicateCount, + hostnameSpecificFilters: selfieFromDict(this.hostnameFilters), + entitySpecificFilters: selfieFromDict(this.entityFilters), + lowGenericFilters: selfieFromDict(this.lowGenericFilters), + highLowGenericHide: this.highLowGenericHide, + highLowGenericDonthide: this.highLowGenericDonthide, + highLowGenericHideCount: this.highLowGenericHideCount, + highLowGenericDonthideCount: this.highLowGenericDonthideCount, + highMediumGenericHide: this.highMediumGenericHide, + highMediumGenericDonthide: this.highMediumGenericDonthide, + highMediumGenericHideCount: this.highMediumGenericHideCount, + highMediumGenericDonthideCount: this.highMediumGenericDonthideCount, + highHighGenericHide: this.highHighGenericHide, + highHighGenericDonthide: this.highHighGenericDonthide, + highHighGenericHideCount: this.highHighGenericHideCount, + highHighGenericDonthideCount: this.highHighGenericDonthideCount + }; +}; + +/******************************************************************************/ + +FilterContainer.prototype.fromSelfie = function(selfie) { + var factories = { + '[]': FilterBucket, + '#': FilterPlain, + '#+': FilterPlainMore, + 'h': FilterHostname, + 'e': FilterEntity + }; + + var dictFromSelfie = function(selfie) { + var dict = {}; + var dictKey; + var bucket = null; + var rawText = selfie; + var rawEnd = rawText.length; + var lineBeg = 0, lineEnd; + var line, pos, what, factory; + while ( lineBeg < rawEnd ) { + lineEnd = rawText.indexOf('\n', lineBeg); + if ( lineEnd < 0 ) { + lineEnd = rawEnd; + } + line = rawText.slice(lineBeg, lineEnd); + lineBeg = lineEnd + 1; + pos = line.indexOf('\t'); + what = line.slice(0, pos); + if ( what === 'k' ) { + dictKey = decode(line.slice(pos + 1)); + bucket = null; + continue; + } + factory = factories[what]; + if ( bucket === null ) { + bucket = dict[dictKey] = factory.fromSelfie(line.slice(pos + 1)); + continue; + } + // When token key is reused, it can't be anything + // else than FilterBucket + bucket.add(factory.fromSelfie(line.slice(pos + 1))); + } + return dict; + }; + + this.frozen = true; + this.acceptedCount = selfie.acceptedCount; + this.duplicateCount = selfie.duplicateCount; + this.hostnameFilters = dictFromSelfie(selfie.hostnameSpecificFilters); + this.entityFilters = dictFromSelfie(selfie.entitySpecificFilters); + this.lowGenericFilters = dictFromSelfie(selfie.lowGenericFilters); + this.highLowGenericHide = selfie.highLowGenericHide; + this.highLowGenericDonthide = selfie.highLowGenericDonthide; + this.highLowGenericHideCount = selfie.highLowGenericHideCount; + this.highLowGenericDonthideCount = selfie.highLowGenericDonthideCount; + this.highMediumGenericHide = selfie.highMediumGenericHide; + this.highMediumGenericDonthide = selfie.highMediumGenericDonthide; + this.highMediumGenericHideCount = selfie.highMediumGenericHideCount; + this.highMediumGenericDonthideCount = selfie.highMediumGenericDonthideCount; + this.highHighGenericHide = selfie.highHighGenericHide; + this.highHighGenericDonthide = selfie.highHighGenericDonthide; + this.highHighGenericHideCount = selfie.highHighGenericHideCount; + this.highHighGenericDonthideCount = selfie.highHighGenericDonthideCount; +}; + +/******************************************************************************/ + FilterContainer.prototype.addToSelectorCache = function(details) { var hostname = details.hostname; if ( typeof hostname !== 'string' || hostname === '' ) { diff --git a/js/assets.js b/js/assets.js index 68e0bf631..6baf90bdb 100644 --- a/js/assets.js +++ b/js/assets.js @@ -43,16 +43,23 @@ File system structure: /******************************************************************************/ +var oneSecond = 1000; +var oneMinute = 60 * oneSecond; +var oneHour = 60 * oneMinute; +var oneDay = 24 * oneHour; + +/******************************************************************************/ + var repositoryRoot = µBlock.projectServerRoot; var nullFunc = function() {}; var reIsExternalPath = /^https?:\/\/[a-z0-9]/; var reIsUserPath = /^assets\/user\//; var lastRepoMetaTimestamp = 0; -var refreshRepoMetaPeriod = 6 * 60 * 60 * 1000; +var refreshRepoMetaPeriod = 5 * oneHour; var exports = { autoUpdate: true, - autoUpdateDelay: 2 * 24 * 60 * 60 * 1000 + autoUpdateDelay: 4 * oneDay }; /******************************************************************************/ @@ -349,7 +356,7 @@ exports.setHomeURL = function(path, homeURL) { entry = metadata.entries[path] = new AssetEntry(); } entry.homeURL = homeURL; - } + }; getRepoMetadata(onRepoMetadataReady); }; @@ -548,9 +555,9 @@ var readRepoCopyAsset = function(path, callback) { var onCacheMetaReady = function(entries) { // Fetch from remote if: // - Auto-update enabled AND (not in cache OR in cache but obsolete) + var timestamp = entries[path]; var homeURL = assetEntry.homeURL; if ( exports.autoUpdate && typeof homeURL === 'string' && homeURL !== '' ) { - var timestamp = entries[path]; var obsolete = Date.now() - exports.autoUpdateDelay; if ( typeof timestamp !== 'number' || timestamp <= obsolete ) { //console.log('µBlock> readRepoCopyAsset("%s") / onCacheMetaReady(): not cached or obsolete', path); diff --git a/js/background.js b/js/background.js index ef52d70d9..2c2ef1a7b 100644 --- a/js/background.js +++ b/js/background.js @@ -63,6 +63,7 @@ return { updateAssetsEvery: 75 * oneHour + 23 * oneMinute + 53 * oneSecond + 605, projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/', userFiltersPath: 'assets/user/filters.txt', + pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', // permanent lists permanentLists: { @@ -86,11 +87,21 @@ return { remoteBlacklists: { }, + firstUpdateAfter: 5 * oneMinute, + nextUpdateAfter: 7 * oneHour, + + selfieMagic: 'ccolmudazpvm', + selfieAfter: 7 * oneMinute, + pageStores: {}, storageQuota: chrome.storage.local.QUOTA_BYTES, storageUsed: 0, + noopFunc: function(){}, + + apiErrorCount: 0, + // so that I don't have to care for last comma dummy: 0 }; diff --git a/js/liquid-dict.js b/js/liquid-dict.js index 1ce408884..3c81408a2 100644 --- a/js/liquid-dict.js +++ b/js/liquid-dict.js @@ -19,6 +19,9 @@ Home: https://github.com/gorhill/uBlock */ +/* jshint bitwise: false */ +/* global µBlock */ + /******************************************************************************/ µBlock.LiquidDict = (function() { @@ -94,12 +97,20 @@ LiquidDict.prototype.makeKey = function(word) { if ( len > 255 ) { len = 255; } - var i = len >> 2; + var i8 = len >>> 3; + var i4 = len >>> 2; + var i2 = len >>> 1; + + // Be sure the msb is not set, this will guarantee a valid unicode + // character (because 0xD800-0xDFFF). return String.fromCharCode( - (word.charCodeAt( 0) & 0x03) << 14 | - (word.charCodeAt( i) & 0x03) << 12 | - (word.charCodeAt( i+i) & 0x03) << 10 | - (word.charCodeAt(i+i+i) & 0x03) << 8 | + (word.charCodeAt( i8) & 0x01) << 14 | + (word.charCodeAt( i4 ) & 0x01) << 13 | + (word.charCodeAt( i4+i8) & 0x01) << 12 | + (word.charCodeAt(i2 ) & 0x01) << 11 | + (word.charCodeAt(i2 +i8) & 0x01) << 10 | + (word.charCodeAt(i2+i4 ) & 0x01) << 9 | + (word.charCodeAt(i2+i4+i8) & 0x01) << 8 , len ); }; @@ -187,6 +198,26 @@ LiquidDict.prototype.reset = function() { /******************************************************************************/ +LiquidDict.prototype.toSelfie = function() { + return { + count: this.count, + bucketCount: this.bucketCount, + frozenBucketCount: this.frozenBucketCount, + dict: this.dict + }; +}; + +/******************************************************************************/ + +LiquidDict.prototype.fromSelfie = function(selfie) { + this.count = selfie.count; + this.bucketCount = selfie.bucketCount; + this.frozenBucketCount = selfie.frozenBucketCount; + this.dict = selfie.dict; +}; + +/******************************************************************************/ + return LiquidDict; /******************************************************************************/ diff --git a/js/messaging-handlers.js b/js/messaging-handlers.js index 4a23c891b..7c61d8ff8 100644 --- a/js/messaging-handlers.js +++ b/js/messaging-handlers.js @@ -121,7 +121,7 @@ var onMessage = function(request, sender, callback) { switch ( request.what ) { case 'retrieveDomainCosmeticSelectors': if ( pageStore && pageStore.getNetFilteringSwitch() ) { - response = µb.abpHideFilters.retrieveDomainSelectors(request); + response = µb.cosmeticFilteringEngine.retrieveDomainSelectors(request); } break; @@ -162,12 +162,12 @@ var onMessage = function(request, sender, callback) { switch ( request.what ) { case 'retrieveGenericCosmeticSelectors': if ( pageStore && pageStore.getNetFilteringSwitch() ) { - response = µb.abpHideFilters.retrieveGenericSelectors(request); + response = µb.cosmeticFilteringEngine.retrieveGenericSelectors(request); } break; case 'injectedSelectors': - µb.abpHideFilters.addToSelectorCache(request); + µb.cosmeticFilteringEngine.addToSelectorCache(request); break; case 'blockedRequests': @@ -250,8 +250,8 @@ var getLists = function(callback) { available: null, current: µb.remoteBlacklists, cosmetic: µb.userSettings.parseAllABPHideFilters, - netFilterCount: µb.abpFilters.getFilterCount(), - cosmeticFilterCount: µb.abpHideFilters.getFilterCount(), + netFilterCount: µb.netFilteringEngine.getFilterCount(), + cosmeticFilterCount: µb.cosmeticFilteringEngine.getFilterCount(), autoUpdate: µb.userSettings.autoUpdate, userFiltersPath: µb.userFiltersPath, cache: null diff --git a/js/profiler.js b/js/profiler.js index 6f03ecb2e..05745d134 100644 --- a/js/profiler.js +++ b/js/profiler.js @@ -40,12 +40,15 @@ var quickProfiler = (function() { prompt = s || ''; tstart = timer.now(); }; - var stop = function() { + var stop = function(period) { + if ( period === undefined ) { + period = 10000; + } var now = timer.now(); count += 1; time += (now - tstart); - if ( (now - lastlog) > 10000 ) { - console.log('µBlock > %s: %s ms (%d samples)', prompt, avg().toFixed(3), count); + if ( (now - lastlog) >= period ) { + console.log('µBlock> %s: %s ms (%d samples)', prompt, avg().toFixed(3), count); lastlog = now; } }; diff --git a/js/start.js b/js/start.js index 8acbb0790..826303b75 100644 --- a/js/start.js +++ b/js/start.js @@ -31,36 +31,76 @@ /******************************************************************************/ var µb = µBlock; -var bufferTime = 0 * 60 * 1000; -var exports = {}; var jobCallback = function() { + // Simpler to fire restart here, and safe given how far this will happen + // in the future. + restart(); + + // If auto-update is disabled, check again in a while. if ( µb.userSettings.autoUpdate !== true ) { return; } - // TODO: need smarter update, currently blindly reloading all. - µb.loadUpdatableAssets(true); + + var onMetadataReady = function(metadata) { + // Check PSL + var mdEntry = metadata[µb.pslPath]; + if ( mdEntry.repoObsolete ) { + // console.log('µBlock.updater> updating all updatable assets'); + µb.loadUpdatableAssets({ update: true }); + return; + } + // Check used filter lists + var lists = µb.remoteBlacklists; + for ( var path in lists ) { + if ( lists.hasOwnProperty(path) === false ) { + continue; + } + if ( lists[path].off ) { + continue; + } + if ( metadata.hasOwnProperty(path) === false ) { + continue; + } + mdEntry = metadata[path]; + if ( mdEntry.cacheObsolete || mdEntry.repoObsolete ) { + // console.log('µBlock.updater> updating only filter lists'); + µb.loadUpdatableAssets({ update: true, psl: false }); + return; + } + } + + // console.log('µBlock.updater> all is up to date'); + }; + + µb.assets.metadata(onMetadataReady); }; // https://www.youtube.com/watch?v=cIrGQD84F1g /******************************************************************************/ -exports.restart = function() { +var restart = function(after) { + if ( after === undefined ) { + after = µb.nextUpdateAfter; + } + µb.asyncJobs.add( 'autoUpdateAssets', null, jobCallback, - µb.updateAssetsEvery - bufferTime, - true + after, + false ); }; -exports.restart(); - /******************************************************************************/ -return exports; +return { + restart: restart +}; + +/******************************************************************************/ })(); diff --git a/js/storage.js b/js/storage.js index be6bf56c6..559361a8c 100644 --- a/js/storage.js +++ b/js/storage.js @@ -139,12 +139,18 @@ /******************************************************************************/ µBlock.appendUserFilters = function(content) { + var µb = this; + + var onFiltersReady = function() { + }; + var onSaved = function(details) { if ( details.error ) { return; } - µBlock.loadUbiquitousBlacklists(); + µb.loadFilterLists(onFiltersReady); }; + var onLoaded = function(details) { if ( details.error ) { return; @@ -152,8 +158,9 @@ if ( details.content.indexOf(content.trim()) !== -1 ) { return; } - µBlock.saveUserFilters(details.content + '\n' + content, onSaved); + µb.saveUserFilters(details.content + '\n' + content, onSaved); }; + if ( content.length > 0 ) { this.loadUserFilters(onLoaded); } @@ -256,15 +263,21 @@ /******************************************************************************/ -µBlock.loadUbiquitousBlacklists = function() { +µBlock.loadFilterLists = function(callback) { var µb = this; var blacklistLoadCount; + if ( typeof callback !== 'function' ) { + callback = this.noopFunc; + } + var loadBlacklistsEnd = function() { - µb.abpFilters.freeze(); - µb.abpHideFilters.freeze(); - µb.messaging.announce({ what: 'loadUbiquitousBlacklistCompleted' }); + µb.netFilteringEngine.freeze(); + µb.cosmeticFilteringEngine.freeze(); chrome.storage.local.set({ 'remoteBlacklists': µb.remoteBlacklists }); + µb.messaging.announce({ what: 'loadUbiquitousBlacklistCompleted' }); + µb.toSelfieAsync(); + callback(); }; var mergeBlacklist = function(details) { @@ -277,10 +290,9 @@ var loadBlacklistsStart = function(lists) { µb.remoteBlacklists = lists; - - // rhill 2013-12-10: set all existing entries to `false`. - µb.abpFilters.reset(); - µb.abpHideFilters.reset(); + µb.netFilteringEngine.reset(); + µb.cosmeticFilteringEngine.reset(); + µb.destroySelfie(); var locations = Object.keys(lists); blacklistLoadCount = locations.length; if ( blacklistLoadCount === 0 ) { @@ -322,11 +334,11 @@ // Useful references: // https://adblockplus.org/en/filter-cheatsheet // https://adblockplus.org/en/filters - var abpFilters = this.abpFilters; - var abpHideFilters = this.abpHideFilters; + var netFilteringEngine = this.netFilteringEngine; + var cosmeticFilteringEngine = this.cosmeticFilteringEngine; var parseCosmeticFilters = this.userSettings.parseAllABPHideFilters; - var duplicateCount = abpFilters.duplicateCount + abpHideFilters.duplicateCount; - var acceptedCount = abpFilters.acceptedCount + abpHideFilters.acceptedCount; + var duplicateCount = netFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount; + var acceptedCount = netFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount; var reLocalhost = /(^|\s)(localhost\.localdomain|localhost|local|broadcasthost|0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)(?=\s|$)/g; var reAdblockFilter = /^[^a-z0-9:]|[^a-z0-9]$|[^a-z0-9_:.-]/; var reAdblockHostFilter = /^\|\|([a-z0-9.-]+[a-z0-9])\^?$/; @@ -360,7 +372,7 @@ // 2014-05-18: ABP element hide filters are allowed to contain space // characters if ( parseCosmeticFilters ) { - if ( abpHideFilters.add(line) ) { + if ( cosmeticFilteringEngine.add(line) ) { continue; } } @@ -396,7 +408,7 @@ // Likely an ABP net filter? if ( reAdblockFilter.test(line) ) { - if ( abpFilters.add(line) ) { + if ( netFilteringEngine.add(line) ) { continue; } // rhill 2014-01-22: Transpose possible Adblock Plus-filter syntax @@ -412,13 +424,13 @@ continue; } - abpFilters.addAnyPartyHostname(line); + netFilteringEngine.addAnyPartyHostname(line); } // For convenience, store the number of entries for this // blacklist, user might be happy to know this information. - duplicateCount = abpFilters.duplicateCount + abpHideFilters.duplicateCount - duplicateCount; - acceptedCount = abpFilters.acceptedCount + abpHideFilters.acceptedCount - acceptedCount; + duplicateCount = netFilteringEngine.duplicateCount + cosmeticFilteringEngine.duplicateCount - duplicateCount; + acceptedCount = netFilteringEngine.acceptedCount + cosmeticFilteringEngine.acceptedCount - acceptedCount; this.remoteBlacklists[details.path].entryCount = acceptedCount + duplicateCount; this.remoteBlacklists[details.path].entryUsedCount = acceptedCount; @@ -449,66 +461,192 @@ } // Now force reload - this.loadUpdatableAssets(update); + this.loadUpdatableAssets({ update: update, psl: update }); }; /******************************************************************************/ µBlock.loadPublicSuffixList = function(callback) { + if ( typeof callback !== 'function' ) { + callback = this.noopFunc; + } var applyPublicSuffixList = function(details) { // TODO: Not getting proper suffix list is a bit serious, I think // the extension should be force-restarted if it occurs.. if ( !details.error ) { publicSuffixList.parse(details.content, punycode.toASCII); } - if ( typeof callback === 'function' ) { - callback(); - } + callback(); }; - this.assets.get( - 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', - applyPublicSuffixList - ); + this.assets.get(this.pslPath, applyPublicSuffixList); }; /******************************************************************************/ // Load updatable assets -µBlock.loadUpdatableAssets = function(update) { +µBlock.loadUpdatableAssets = function(details) { + var µb = this; + + details = details || {}; + var update = details.update !== false; + this.assets.autoUpdate = update || this.userSettings.autoUpdate; this.assets.autoUpdateDelay = this.updateAssetsEvery; - this.loadPublicSuffixList(); - this.loadUbiquitousBlacklists(); - // It could be a manual update, so we reset the auto-updater - if ( update ) { - this.updater.restart(); + var onFiltersReady = function() { + if ( update ) { + µb.updater.restart(); + } + }; + + var onPSLReady = function() { + µb.loadFilterLists(onFiltersReady); + }; + + if ( details.psl !== false ) { + this.loadPublicSuffixList(onPSLReady); + } else { + this.loadFilterLists(onFiltersReady); } }; /******************************************************************************/ +µBlock.toSelfie = function() { + var selfie = { + magic: this.selfieMagic, + publicSuffixList: publicSuffixList.toSelfie(), + filterLists: this.remoteBlacklists, + netFilteringEngine: this.netFilteringEngine.toSelfie(), + cosmeticFilteringEngine: this.cosmeticFilteringEngine.toSelfie(), + }; + chrome.storage.local.set({ selfie: selfie }); + // console.log('µBlock.toSelfie> made a selfie!'); +}; + +// This is to be sure the selfie is generated in a sane manner: the selfie will +// be generated if the user doesn't change his filter lists selection for +// some set time. +µBlock.toSelfieAsync = function(after) { + if ( typeof after !== 'number' ) { + after = this.selfieAfter; + } + this.asyncJobs.add( + 'toSelfie', + null, + this.toSelfie.bind(this), + after, + false + ); +}; + +/******************************************************************************/ + +µBlock.fromSelfie = function(callback) { + var µb = this; + + if ( typeof callback !== 'function' ) { + callback = this.noopFunc; + } + + var onSelfieReady = function(store) { + var selfie = store.selfie; + if ( typeof selfie !== 'object' || selfie.magic !== µb.selfieMagic ) { + callback(false); + return; + } + if ( publicSuffixList.fromSelfie(selfie.publicSuffixList) !== true ) { + callback(false); + return; + } + // console.log('µBlock.fromSelfie> selfie looks good'); + µb.remoteBlacklists = selfie.filterLists; + µb.netFilteringEngine.fromSelfie(selfie.netFilteringEngine); + µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine); + callback(true); + }; + + chrome.storage.local.get('selfie', onSelfieReady); +}; + +/******************************************************************************/ + +µBlock.destroySelfie = function() { + chrome.storage.local.remove('selfie'); +}; + +/******************************************************************************/ + // Load all µBlock.load = function() { var µb = this; - // User whitelist directives and filters need the Public Suffix List to be - // available -- because the way they are stored internally. + // Final initialization steps after all needed assets are in memory + var onAllDone = function(wasAutoUpdated) { + // Initialize internal state with maybe already existing tabs + var bindToTabs = function(tabs) { + var scriptStart = function(tabId) { + var scriptEnd = function() { + chrome.tabs.executeScript(tabId, { + file: 'js/contentscript-end.js', + allFrames: true, + runAt: 'document_idle' + }); + }; + chrome.tabs.executeScript(tabId, { + file: 'js/contentscript-start.js', + allFrames: true, + runAt: 'document_idle' + }, scriptEnd); + }; + var i = tabs.length, tab; + while ( i-- ) { + tab = tabs[i]; + µb.bindTabToPageStats(tab.id, tab.url); + // https://github.com/gorhill/uBlock/issues/129 + scriptStart(tab.id); + } + }; + chrome.tabs.query({ url: 'http://*/*' }, bindToTabs); + chrome.tabs.query({ url: 'https://*/*' }, bindToTabs); + + // https://github.com/gorhill/uBlock/issues/184 + // If we restored a selfie, check for updates not too far + // in the future. + µb.updater.restart(wasAutoUpdated ? µb.nextUpdateAfter : µb.firstUpdateAfter); + }; + + // Filters are in memory + var onFiltersReady = function() { + onAllDone(µb.userSettings.autoUpdate); + }; + + // Load order because dependencies: + // User settings -> PSL -> [filter lists, user whitelist] var onPSLReady = function() { µb.loadWhitelist(); - µb.loadUbiquitousBlacklists(); + µb.loadFilterLists(onFiltersReady); }; - // Public Suffix List loader needs the user settings need to be available - // because we need to know whether to auto-update the list or not. - var onUserSettingsReady = function() { - µb.assets.autoUpdate = µb.userSettings.autoUpdate || true; + // If no selfie available, take the long way, i.e. load and parse + // raw data. + var onSelfieReady = function(success) { + if ( success === true ) { + onAllDone(false); + return; + } + µb.assets.autoUpdate = µb.userSettings.autoUpdate; µb.loadPublicSuffixList(onPSLReady); }; - this.loadUserSettings(onUserSettingsReady); + // User settings are in memory + var onUserSettingsReady = function() { + µb.fromSelfie(onSelfieReady); + }; + + this.loadUserSettings(onUserSettingsReady); this.loadLocalSettings(); this.getBytesInUse(); }; diff --git a/js/tab.js b/js/tab.js index efc635374..8369f4410 100644 --- a/js/tab.js +++ b/js/tab.js @@ -59,41 +59,6 @@ chrome.tabs.onRemoved.addListener(onTabRemoved); })(); -/******************************************************************************/ - -// Initialize internal state with maybe already existing tabs -// This needs to be executed once, hence it has its own scope, which will -// allow the code to be flushed once completed. - -(function(){ - var µb = µBlock; - var bindToTabs = function(tabs) { - var scriptStart = function(tabId) { - var scriptEnd = function() { - chrome.tabs.executeScript(tabId, { - file: 'js/contentscript-end.js', - allFrames: true, - runAt: 'document_idle' - }); - }; - chrome.tabs.executeScript(tabId, { - file: 'js/contentscript-start.js', - allFrames: true, - runAt: 'document_idle' - }, scriptEnd); - }; - var i = tabs.length, tab; - while ( i-- ) { - tab = tabs[i]; - µb.bindTabToPageStats(tab.id, tab.url); - // https://github.com/gorhill/uBlock/issues/129 - scriptStart(tab.id); - } - }; - chrome.tabs.query({ url: 'http://*/*' }, bindToTabs); - chrome.tabs.query({ url: 'https://*/*' }, bindToTabs); -})(); - /******************************************************************************/ /******************************************************************************/ diff --git a/js/traffic.js b/js/traffic.js index 65aeb8f5d..36abaf65c 100644 --- a/js/traffic.js +++ b/js/traffic.js @@ -80,7 +80,7 @@ var onBeforeRequest = function(details) { var reason = false; if ( pageStore.getNetFilteringSwitch() ) { - reason = µb.abpFilters.matchString(requestContext, requestURL, requestType, requestHostname); + reason = µb.netFilteringEngine.matchString(requestContext, requestURL, requestType, requestHostname); } // Record what happened. pageStore.recordRequest(requestType, requestURL, reason); @@ -168,7 +168,7 @@ var onBeforeSendHeaders = function(details) { // in multiple tabs. var reason = false; if ( pageStore.getNetFilteringSwitch() ) { - reason = µb.abpFilters.matchStringExactType( + reason = µb.netFilteringEngine.matchStringExactType( pageDetails, requestURL, 'popup', diff --git a/lib/publicsuffixlist.min.js b/lib/publicsuffixlist.min.js index a2607d428..e114a8ebf 100644 --- a/lib/publicsuffixlist.min.js +++ b/lib/publicsuffixlist.min.js @@ -1,17 +1,17 @@ /*! Home: https://github.com/gorhill/publicsuffixlist.js */ -;(function(f){var b={};var h={};var a=480; -var g=/[^a-z0-9.-]/;function i(k){if(!k||k.charAt(0)==="."){return""}k=k.toLowerCase();var l=c(k);if(l===k){return"" -}var m=k.lastIndexOf(".",k.lastIndexOf(".",k.length-l.length)-1);if(m<=0){return k}return k.slice(m+1) -}function c(k){if(!k){return""}var l;while(true){l=k.indexOf(".");if(l<0){return k}if(j(b,k)){return k.slice(l+1) -}if(j(h,k)){return k}if(j(h,"*"+k.slice(l))){return k}k=k.slice(l+1)}}function j(t,r){var q=r.lastIndexOf("."); -var m,v;if(q<0){m=r;v=r}else{m=r.slice(q+1);v=r.slice(0,q)}var s=t[m];if(!s){return false}if(typeof s==="string"){return s.indexOf(" "+v+" ")>=0 -}var n=v.length;var w=s[n];if(!w){return false}var k=0;var u=Math.floor(w.length/n+0.5);var p,o;while(k>1; -o=w.substr(n*p,n);if(vo){k=p+1}else{return true}}}return false}function d(r,p){b={}; -h={};r=r.toLowerCase();var o=0,l;var m=r.length;var s,q,n,k;while(o=0){s=s.slice(0,n)}s=s.trim(); -if(!s){continue}if(g.test(s)){s=p(s)}if(s.charAt(0)==="!"){q=b;s=s.slice(1)}else{q=h}n=s.lastIndexOf("."); -if(n<0){k=s}else{k=s.slice(n+1);s=s.slice(0,n)}if(!q[k]){q[k]=[]}if(s){q[k].push(s)}}e(b);e(h)}function e(m){var o,q,p,k; -for(var n in m){if(!m.hasOwnProperty(n)){continue}o=m[n].join(" ");if(!o){m[n]="";continue}if(o.length=0 +}var p=x.length;var y=u[p];if(!y){return false}var n=0;var w=Math.floor(y.length/p+0.5);var r,q;while(n>1;q=y.substr(p*r,p); +if(xq){n=r+1}else{return true}}}return false}function f(u,s){d={};k={};u=u.toLowerCase();var r=0,o;var p=u.length; +var v,t,q,n;while(r=0){v=v.slice(0,q)}v=v.trim();if(!v){continue}if(j.test(v)){v=s(v)}if(v.charAt(0)==="!"){t=d;v=v.slice(1) +}else{t=k}q=v.lastIndexOf(".");if(q<0){n=v}else{n=v.slice(q+1);v=v.slice(0,q)}if(!t.hasOwnProperty(n)){t[n]=[]}if(v){t[n].push(v) +}}h(d);h(k)}function h(o){var q,s,r,n;for(var p in o){if(!o.hasOwnProperty(p)){continue}q=o[p].join(" ");if(!q){o[p]="";continue +}if(q.length