[imgur] rewrite

gallery_dl/extractor/imgur.py

@@ -29,44 +29,41 @@ class ImgurExtractor(Extractor):
         self.album = match.group(1)
 
     def items(self):
-        page = self.request("https://imgur.com/a/" + self.album).text
+        data = self.get_job_metadata()
-        data = self.get_job_metadata(page)
-        images = self.get_images(page)
-        data["count"] = len(images)
         yield Message.Version, 1
         yield Message.Directory, data
-        for image in images:
+        for num, url in enumerate(self.get_image_urls(), 1):
-            data.update(image)
+            name, ext = os.path.splitext(url[20:])
-            yield Message.Url, image["url"], data
+            data["num"] = num
+            data["name"] = name
+            data["extension"] = ext[1:]
+            yield Message.Url, url, data
 
-    def get_job_metadata(self, page):
+    def get_job_metadata(self):
         """Collect metadata for extractor-job"""
-        title, _ = text.extract(page, '<meta property="og:title" content="', '"')
+        page = self.request("https://imgur.com/a/" + self.album).text
-        return {
+        data = {
             "category": info["category"],
             "album-key": self.album,
-            "title": title,
-            # "date": ...,
         }
+        return text.extract_all(page, (
+            ('title', '<meta property="og:title" content="', '"'),
+            ('date' , '"create_datetime":"', '"'),
+            ('count', '"num_images":', ','),
+        ), values=data)[0]
 
-    def get_images(self, page):
+    def get_image_urls(self):
-        """Build a list of all images in this album"""
+        """Yield urls of all images in this album"""
-        images = []
-        pos = 0
         num = 0
         while True:
-            url   , pos = text.extract(page, 'property="og:image" content="', '"', pos)
+            url = "https://imgur.com/a/{}/all/page/{}?scrolled".format(self.album, num)
-            if not url:
+            page = self.request(url).text
-                return images
+            pos = begin = text.extract(page, '<div class="posts">', '')[1]
-            width , pos = text.extract(page, 'property="og:image:width" content="', '"', pos)
+            while True:
-            height, pos = text.extract(page, 'property="og:image:height" content="', '"', pos)
+                url, pos = text.extract(page, '<a href="', '"', pos)
-            name = os.path.splitext(text.filename_from_url(url))
+                if not url:
+                    break
+                yield "https:" + url
+            if pos == begin:
+                return
             num += 1
-            images.append({
-                "url": "https" + url[4:],
-                "width": width,
-                "height": height,
-                "name": name[0],
-                "extension": name[1][1:],
-                "num": num,
-            })