From 3fc1e1294991f8bd45df93f7c1592c4791826570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 21 Nov 2019 16:57:39 +0100
Subject: [PATCH] [postprocessor:metadata] filter private entries

i.e. keys starting with an underscore
---
 gallery_dl/job.py                    | 16 +++++++---------
 gallery_dl/postprocessor/metadata.py |  2 +-
 gallery_dl/util.py                   |  5 +++++
 test/test_postprocessor.py           |  6 +++++-
 test/test_results.py                 |  2 +-
 test/test_util.py                    | 15 +++++++++++++++
 6 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/gallery_dl/job.py b/gallery_dl/job.py
index 9c763368..a928a8e0 100644
--- a/gallery_dl/job.py
+++ b/gallery_dl/job.py
@@ -161,11 +161,6 @@ class Job():
         if self.ulog:
             self.ulog.info(url)
 
-    @staticmethod
-    def _filter(kwdict):
-        """Return a copy of 'kwdict' without "private" entries"""
-        return {k: v for k, v in kwdict.items() if k[0] != "_"}
-
 
 class DownloadJob(Job):
     """Download images into appropriate directory/filename locations"""
@@ -479,6 +474,9 @@ class DataJob(Job):
         self.data = []
         self.ascii = config.get(("output", "ascii"), ensure_ascii)
 
+        private = config.get(("output", "private"))
+        self.filter = (lambda x: x) if private else util.filter_dict
+
     def run(self):
         # collect data
         try:
@@ -501,16 +499,16 @@ class DataJob(Job):
         return 0
 
     def handle_url(self, url, kwdict):
-        self.data.append((Message.Url, url, self._filter(kwdict)))
+        self.data.append((Message.Url, url, self.filter(kwdict)))
 
     def handle_urllist(self, urls, kwdict):
-        self.data.append((Message.Urllist, list(urls), self._filter(kwdict)))
+        self.data.append((Message.Urllist, list(urls), self.filter(kwdict)))
 
     def handle_directory(self, kwdict):
-        self.data.append((Message.Directory, self._filter(kwdict)))
+        self.data.append((Message.Directory, self.filter(kwdict)))
 
     def handle_queue(self, url, kwdict):
-        self.data.append((Message.Queue, url, self._filter(kwdict)))
+        self.data.append((Message.Queue, url, self.filter(kwdict)))
 
     def handle_finalize(self):
         self.file.close()
diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py
index 467ef112..0e559559 100644
--- a/gallery_dl/postprocessor/metadata.py
+++ b/gallery_dl/postprocessor/metadata.py
@@ -58,7 +58,7 @@ class MetadataPP(PostProcessor):
         file.write("\n")
 
     def _write_json(self, file, kwdict):
-        util.dump_json(kwdict, file, self.ascii, self.indent)
+        util.dump_json(util.filter_dict(kwdict), file, self.ascii, self.indent)
 
 
 __postprocessor__ = MetadataPP
diff --git a/gallery_dl/util.py b/gallery_dl/util.py
index fb51edff..854ad5c8 100644
--- a/gallery_dl/util.py
+++ b/gallery_dl/util.py
@@ -78,6 +78,11 @@ def transform_dict(a, func):
             a[key] = func(value)
 
 
+def filter_dict(a):
+    """Return a copy of 'a' without "private" entries"""
+    return {k: v for k, v in a.items() if k[0] != "_"}
+
+
 def number_to_string(value, numbers=(int, float)):
     """Convert numbers (int, float) to string; Return everything else as is."""
     return str(value) if value.__class__ in numbers else value
diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py
index 0ab89dbf..b55f0648 100644
--- a/test/test_postprocessor.py
+++ b/test/test_postprocessor.py
@@ -151,6 +151,9 @@ class MetadataTest(BasePostprocessorTest):
             "ascii"    : True,
             "indent"   : 2,
             "extension": "JSON",
+        }, {
+            "public"   : "hello",
+            "_private" : "world",
         })
 
         self.assertEqual(pp.write    , pp._write_json)
@@ -167,7 +170,8 @@ class MetadataTest(BasePostprocessorTest):
         self.assertEqual(self._output(m), """{
   "category": "test",
   "extension": "ext",
-  "filename": "file"
+  "filename": "file",
+  "public": "hello"
 }
 """)
 
diff --git a/test/test_results.py b/test/test_results.py
index 796b80bd..f253aa30 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -207,7 +207,7 @@ class ResultJob(job.DownloadJob):
     def _update_kwdict(self, kwdict, to_list=True):
         if to_list:
             self.kwdict_list.append(kwdict.copy())
-        kwdict = self._filter(kwdict)
+        kwdict = util.filter_dict(kwdict)
         self.kwdict_hash.update(
             json.dumps(kwdict, sort_keys=True, default=str).encode())
 
diff --git a/test/test_util.py b/test/test_util.py
index 9b252a37..5a103cf3 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -358,6 +358,21 @@ class TestOther(unittest.TestCase):
         self.assertEqual(
             d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}})
 
+    def test_filter_dict(self):
+        d = {}
+        r = util.filter_dict(d)
+        self.assertEqual(r, d)
+        self.assertIsNot(r, d)
+
+        d = {"foo": 123, "bar": [], "baz": None}
+        r = util.filter_dict(d)
+        self.assertEqual(r, d)
+        self.assertIsNot(r, d)
+
+        d = {"foo": 123, "_bar": [], "__baz__": None}
+        r = util.filter_dict(d)
+        self.assertEqual(r, {"foo": 123})
+
     def test_number_to_string(self, f=util.number_to_string):
         self.assertEqual(f(1)     , "1")
         self.assertEqual(f(1.0)   , "1.0")