From 76b01b64cfe6843998e81dc73f9d6fac6d6c77b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Wed, 25 Jan 2023 11:10:09 +0100
Subject: [PATCH] [kemonoparty] remove MD5 hash extraction (#3531)

This partially reverts commit 20d6194ffa529b750d4bd5e1540c5a49ef11de71.
---
 gallery_dl/extractor/kemonoparty.py | 53 ++++++++++++-----------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/gallery_dl/extractor/kemonoparty.py b/gallery_dl/extractor/kemonoparty.py
index 189e2baa..33e8370a 100644
--- a/gallery_dl/extractor/kemonoparty.py
+++ b/gallery_dl/extractor/kemonoparty.py
@@ -16,6 +16,7 @@ import re
 
 BASE_PATTERN = r"(?:https?://)?(?:www\.|beta\.)?(kemono|coomer)\.party"
 USER_PATTERN = BASE_PATTERN + r"/([^/?#]+)/user/([^/?#]+)"
+HASH_PATTERN = r"/[0-9a-f]{2}/[0-9a-f]{2}/([0-9a-f]{64})"
 
 
 class KemonopartyExtractor(Extractor):
@@ -35,17 +36,13 @@ class KemonopartyExtractor(Extractor):
         Extractor.__init__(self, match)
         self.session.headers["Referer"] = self.root + "/"
 
-        self._find_hash = re.compile(
-            r"/(?:[0-9a-f]{2}/[0-9a-f]{2}/([0-9a-f]{64})|"  # sha256
-            r"attachments/\w+/\w+/\w+/([0-9a-f]{32}))"      # md5
-        ).match
-
     def items(self):
         self._prepare_ddosguard_cookies()
 
         self._find_inline = re.compile(
             r'src="(?:https?://(?:kemono|coomer)\.party)?(/inline/[^"]+'
             r'|/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{64}\.[^"]+)').findall
+        find_hash = re.compile(HASH_PATTERN).match
         generators = self._build_file_generators(self.config("files"))
         duplicates = self.config("duplicates")
         comments = self.config("comments")
@@ -90,12 +87,16 @@ class KemonopartyExtractor(Extractor):
                     g(post) for g in generators):
                 url = file["path"]
 
-                file["hash"] = hash = self._hash(url)
-                if hash and not duplicates:
-                    if hash in hashes:
-                        self.log.debug("Skipping %s (duplicate)", url)
-                        continue
-                    hashes.add(hash)
+                match = find_hash(url)
+                if match:
+                    file["hash"] = hash = match.group(1)
+                    if not duplicates:
+                        if hash in hashes:
+                            self.log.debug("Skipping %s (duplicate)", url)
+                            continue
+                        hashes.add(hash)
+                else:
+                    file["hash"] = ""
 
                 files.append(file)
 
@@ -156,12 +157,6 @@ class KemonopartyExtractor(Extractor):
         for path in self._find_inline(post["content"] or ""):
             yield {"path": path, "name": path, "type": "inline"}
 
-    def _hash(self, url, default=""):
-        match = self._find_hash(url)
-        if match:
-            return match.group(1) or match.group(2)
-        return default
-
     def _build_file_generators(self, filetypes):
         if filetypes is None:
             return (self._attachments, self._file, self._inline)
@@ -332,12 +327,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
                        r"f51c10adc9dabd86e92bd52339f298b9\.txt",
             "content": "da39a3ee5e6b4b0d3255bfef95601890afd80709",  # empty
         }),
-        # extract md5
-        ("https://kemono.party/subscribestar/user/alcorart/post/184330", {
-            "pattern": r"https://kemono.party/data/attachments/"
-                       r"subscribestar/alcorart/184330/07923b489299b79.+\.png",
-            "keyword": {"hash": "07923b489299b7990b4e374ce643b11d"},
-        }),
+        ("https://kemono.party/subscribestar/user/alcorart/post/184330"),
         ("https://www.kemono.party/subscribestar/user/alcorart/post/184330"),
         ("https://beta.kemono.party/subscribestar/user/alcorart/post/184330"),
     )
@@ -374,16 +364,17 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
             "pattern": r"https://kemono\.party/data/("
                        r"e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|"
                        r"51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)",
+            "keyword": {"hash": "re:e377e3525164559484ace2e64425b0cec1db08"
+                                "|51453640a5e0a4d23fbf57fb85390f9c5ec154"},
             "count": ">= 2",
-            "keyword": {"hash": "re:^(e377e35251645594|51453640a5e0a4d2).+$"},
         }),
         # 'inline' files
         (("https://kemono.party/discord"
           "/server/315262215055736843/channel/315262215055736843#general"), {
             "pattern": r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$",
-            "range": "1-5",
             "options": (("image-filter", "type == 'inline'"),),
             "keyword": {"hash": ""},
+            "range": "1-5",
         }),
     )
 
@@ -397,6 +388,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
         find_inline = re.compile(
             r"https?://(?:cdn\.discordapp.com|media\.discordapp\.net)"
             r"(/[A-Za-z0-9-._~:/?#\[\]@!$&'()*+,;%=]+)").findall
+        find_hash = re.compile(HASH_PATTERN).match
 
         posts = self.posts()
         max_posts = self.config("max-posts")
@@ -407,16 +399,13 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
             files = []
             append = files.append
             for attachment in post["attachments"]:
+                match = find_hash(attachment["path"])
+                attachment["hash"] = match.group(1) if match else ""
                 attachment["type"] = "attachment"
-                attachment["hash"] = self._hash(attachment["path"])
                 append(attachment)
             for path in find_inline(post["content"] or ""):
-                append({
-                    "hash": "",
-                    "path": "https://cdn.discordapp.com" + path,
-                    "name": path,
-                    "type": "inline",
-                })
+                append({"path": "https://cdn.discordapp.com" + path,
+                        "name": path, "type": "inline", "hash": ""})
 
             post["channel_name"] = self.channel_name
             post["date"] = text.parse_datetime(