[postprocessor:metadata] filter private entries

i.e. keys starting with an underscore

gallery_dl/job.py

@@ -161,11 +161,6 @@ class Job():
         if self.ulog:
             self.ulog.info(url)
 
-    @staticmethod
-    def _filter(kwdict):
-        """Return a copy of 'kwdict' without "private" entries"""
-        return {k: v for k, v in kwdict.items() if k[0] != "_"}
-
 
 class DownloadJob(Job):
     """Download images into appropriate directory/filename locations"""
@@ -479,6 +474,9 @@ class DataJob(Job):
         self.data = []
         self.ascii = config.get(("output", "ascii"), ensure_ascii)
 
+        private = config.get(("output", "private"))
+        self.filter = (lambda x: x) if private else util.filter_dict
+
     def run(self):
         # collect data
         try:
@@ -501,16 +499,16 @@ class DataJob(Job):
         return 0
 
     def handle_url(self, url, kwdict):
-        self.data.append((Message.Url, url, self._filter(kwdict)))
+        self.data.append((Message.Url, url, self.filter(kwdict)))
 
     def handle_urllist(self, urls, kwdict):
-        self.data.append((Message.Urllist, list(urls), self._filter(kwdict)))
+        self.data.append((Message.Urllist, list(urls), self.filter(kwdict)))
 
     def handle_directory(self, kwdict):
-        self.data.append((Message.Directory, self._filter(kwdict)))
+        self.data.append((Message.Directory, self.filter(kwdict)))
 
     def handle_queue(self, url, kwdict):
-        self.data.append((Message.Queue, url, self._filter(kwdict)))
+        self.data.append((Message.Queue, url, self.filter(kwdict)))
 
     def handle_finalize(self):
         self.file.close()

gallery_dl/postprocessor/metadata.py

@@ -58,7 +58,7 @@ class MetadataPP(PostProcessor):
         file.write("\n")
 
     def _write_json(self, file, kwdict):
-        util.dump_json(kwdict, file, self.ascii, self.indent)
+        util.dump_json(util.filter_dict(kwdict), file, self.ascii, self.indent)
 
 
 __postprocessor__ = MetadataPP

gallery_dl/util.py

@@ -78,6 +78,11 @@ def transform_dict(a, func):
             a[key] = func(value)
 
 
+def filter_dict(a):
+    """Return a copy of 'a' without "private" entries"""
+    return {k: v for k, v in a.items() if k[0] != "_"}
+
+
 def number_to_string(value, numbers=(int, float)):
     """Convert numbers (int, float) to string; Return everything else as is."""
     return str(value) if value.__class__ in numbers else value

test/test_postprocessor.py

@@ -151,6 +151,9 @@ class MetadataTest(BasePostprocessorTest):
             "ascii"    : True,
             "indent"   : 2,
             "extension": "JSON",
+        }, {
+            "public"   : "hello",
+            "_private" : "world",
         })
 
         self.assertEqual(pp.write    , pp._write_json)
@@ -167,7 +170,8 @@ class MetadataTest(BasePostprocessorTest):
         self.assertEqual(self._output(m), """{
   "category": "test",
   "extension": "ext",
-  "filename": "file"
+  "filename": "file",
+  "public": "hello"
 }
 """)
 

test/test_results.py

@@ -207,7 +207,7 @@ class ResultJob(job.DownloadJob):
     def _update_kwdict(self, kwdict, to_list=True):
         if to_list:
             self.kwdict_list.append(kwdict.copy())
-        kwdict = self._filter(kwdict)
+        kwdict = util.filter_dict(kwdict)
         self.kwdict_hash.update(
             json.dumps(kwdict, sort_keys=True, default=str).encode())
 

test/test_util.py

@@ -358,6 +358,21 @@ class TestOther(unittest.TestCase):
         self.assertEqual(
             d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}})
 
+    def test_filter_dict(self):
+        d = {}
+        r = util.filter_dict(d)
+        self.assertEqual(r, d)
+        self.assertIsNot(r, d)
+
+        d = {"foo": 123, "bar": [], "baz": None}
+        r = util.filter_dict(d)
+        self.assertEqual(r, d)
+        self.assertIsNot(r, d)
+
+        d = {"foo": 123, "_bar": [], "__baz__": None}
+        r = util.filter_dict(d)
+        self.assertEqual(r, {"foo": 123})
+
     def test_number_to_string(self, f=util.number_to_string):
         self.assertEqual(f(1)     , "1")
         self.assertEqual(f(1.0)   , "1.0")