implement 'image-unique' and 'chapter-unique' options (#303)

The default value for both is 'false', i.e. duplicate URLs are NOT
ignored.

The previous behavior was to always ignore duplicate URLs to make
'--abort-on-skip' work properly when new images where added to the
beginning of a collection while gallery-dl is running.

docs/configuration.rst

@@ -381,6 +381,28 @@ Description Like `image-filter`__, but applies to delegated URLs
 __ `extractor.*.image-filter`_
 
 
+extractor.*.image-unique
+------------------------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Ignore image URLs that have been encountered before during the
+            current extractor run.
+=========== =====
+
+
+extractor.*.chapter-unique
+--------------------------
+=========== =====
+Type        ``bool``
+Default     ``false``
+Description Like `image-unique`__, but applies to delegated URLs
+            like manga-chapters, etc.
+=========== =====
+
+__ `extractor.*.image-unique`_
+
+
 
 Extractor-specific Options
 ==========================

gallery_dl/job.py

@@ -29,15 +29,9 @@ class Job():
         extr.log.job = self
         extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url)
 
-        # url predicates
-        self.pred_url = self._prepare_predicates(
-            "image", [util.UniquePredicate()], True)
+        self.pred_url = self._prepare_predicates("image", True)
+        self.pred_queue = self._prepare_predicates("chapter", False)
 
-        # queue predicates
-        self.pred_queue = self._prepare_predicates(
-            "chapter", [], False)
-
-        # category transfer
         if parent and parent.extractor.config(
                 "category-transfer", parent.extractor.categorytransfer):
             self.extractor.category = parent.extractor.category
@@ -142,7 +136,12 @@ class Job():
         if self.userkwds:
             kwdict.update(self.userkwds)
 
-    def _prepare_predicates(self, target, predicates, skip=True):
+    def _prepare_predicates(self, target, skip=True):
+        predicates = []
+
+        if self.extractor.config(target + "-unique"):
+            predicates.append(util.UniquePredicate())
+
         pfilter = self.extractor.config(target + "-filter")
         if pfilter:
             try: