update build_testresult_db.py script

gallery_dl/job.py

@@ -521,18 +521,23 @@ class TestJob(DownloadJob):
 class DataJob(Job):
     """Collect extractor results and dump them"""
 
-    def __init__(self, url, parent=None, file=sys.stdout):
+    def __init__(self, url, parent=None, file=sys.stdout, ensure_ascii=True):
         Job.__init__(self, url, parent)
         self.file = file
         self.data = []
+        self.ascii = config.get(("output", "ascii"), ensure_ascii)
 
     def run(self):
         # collect data
         try:
             for msg in self.extractor:
                 self.dispatch(msg)
+        except exception.StopExtraction:
+            pass
         except Exception as exc:
             self.data.append((exc.__class__.__name__, str(exc)))
+        except KeyboardInterrupt:
+            raise
         except BaseException:
             pass
 
@@ -543,8 +548,7 @@ class DataJob(Job):
         # dump to 'file'
         json.dump(
             self.data, self.file,
-            sort_keys=True, indent=2,
-            ensure_ascii=config.get(("output", "ascii"), True),
+            sort_keys=True, indent=2, ensure_ascii=self.ascii,
         )
         self.file.write("\n")
 

scripts/build_testresult_db.py

@@ -7,27 +7,47 @@ import datetime
 ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.realpath(ROOTDIR))
 from gallery_dl import extractor, job, config
+from test.test_results import setup_test_config
+
+
+# filter test cases
 
 tests = [
-   ([url[0] for url in extr.test if url[1]], extr)
-   for extr in extractor.extractors()
-   if hasattr(extr, "test")
+    (idx, extr, url, result)
+
+    for extr in extractor.extractors()
+    if hasattr(extr, "test") and extr.test
+    if len(sys.argv) <= 1 or extr.category in sys.argv
+
+    for idx, (url, result) in enumerate(extr.test)
+    if result
 ]
 
-if len(sys.argv) > 1:
-    tests = [
-        (urls, extr)
-        for urls, extr in tests
-        if extr.category in sys.argv
-    ]
+
+# setup target directory
 
 path = os.path.join(ROOTDIR, "archive/testdb", str(datetime.date.today()))
 os.makedirs(path, exist_ok=True)
-config.load()
 
-for urls, extr in tests:
-    for i, url in enumerate(urls):
-        name = "%s-%s-%d.json" % (extr.category, extr.subcategory, i)
-        print(name)
+
+for idx, extr, url, result in tests:
+
+    # filename
+    name = "{}-{}-{}.json".format(extr.category, extr.subcategory, idx)
+    print(name)
+
+    # config values
+    setup_test_config()
+
+    if "options" in result:
+        for key, value in result["options"]:
+            config.set(key.split("."), value)
+    if "range" in result:
+        config.set(("image-range",), result["range"])
+
+    # write test data
+    try:
         with open(os.path.join(path, name), "w") as outfile:
-            job.DataJob(url, file=outfile).run()
+            job.DataJob(url, file=outfile, ensure_ascii=False).run()
+    except KeyboardInterrupt:
+        sys.exit()

test/test_results.py

@@ -28,23 +28,29 @@ BROKEN = {
 }
 
 
+def setup_test_config():
+    name = "gallerydl"
+    email = "gallerydl@openaliasbox.org"
+
+    config.clear()
+    config.set(("cache", "file"), ":memory:")
+    config.set(("downloader", "part"), False)
+    config.set(("extractor", "timeout"), 60)
+    config.set(("extractor", "username"), name)
+    config.set(("extractor", "password"), name)
+    config.set(("extractor", "nijie", "username"), email)
+    config.set(("extractor", "seiga", "username"), email)
+    config.set(("extractor", "deviantart", "client-id"), "7777")
+    config.set(("extractor", "deviantart", "client-secret"),
+               "ff14994c744d9208e5caeec7aab4a026")
+    config.set(("extractor", "tumblr", "api-key"),
+               "0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6")
+
+
 class TestExtractorResults(unittest.TestCase):
 
     def setUp(self):
-        name = "gallerydl"
-        email = "gallerydl@openaliasbox.org"
-        config.set(("cache", "file"), ":memory:")
-        config.set(("downloader", "part"), False)
-        config.set(("downloader", "timeout"), 60)
-        config.set(("extractor", "username"), name)
-        config.set(("extractor", "password"), name)
-        config.set(("extractor", "nijie", "username"), email)
-        config.set(("extractor", "seiga", "username"), email)
-        config.set(("extractor", "deviantart", "client-id"), "7777")
-        config.set(("extractor", "deviantart", "client-secret"),
-                   "ff14994c744d9208e5caeec7aab4a026")
-        config.set(("extractor", "tumblr", "api-key"),
-                   "0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6")
+        setup_test_config()
 
     def tearDown(self):
         config.clear()