Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-22 10:42:34 +01:00
Code Issues Releases Wiki Activity

update some extractors to use https

Browse Source
This commit is contained in:
Mike Fährmann 2017-04-20 13:20:41 +02:00
parent 342371086b
commit 13dc5d72bc
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
12 changed files with 43 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
gallery_dl/extractor/deviantart.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from http://www.deviantart.com/"""
"""Extract images from https://www.deviantart.com/"""
from .common import Extractor, Message
from .. import text, exception
@ -107,9 +107,9 @@ class DeviantartImageExtractor(DeviantartExtractor):
return (self.api.deviation(deviation_id),)
class DeviantartFavouritesExtractor(DeviantartExtractor):
class DeviantartFavoriteExtractor(DeviantartExtractor):
"""Extractor for an artist's favourites from deviantart.com"""
subcategory = "favourites"
subcategory = "favorite"
directory_fmt = ["{category}", "{subcategory}",
"{collection[owner]} - {collection[title]}"]
pattern = [r"(?:https?://)?([^\.]+)\.deviantart\.com/favourites"
@ -117,12 +117,12 @@ class DeviantartFavouritesExtractor(DeviantartExtractor):
test = [
("http://h3813067.deviantart.com/favourites/", {
"url": "71345ce3bef5b19bd2a56d7b96e6b5ddba747c2e",
"keyword": "35a275b0f737aa9bd1f32ba13604d6e9a7054a14",
"keyword": "51e88d400c3fb69ae0b5a618ef21a282697185fe",
"content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e",
}),
("http://rosuuri.deviantart.com/favourites/58951174/Useful", {
"url": "9e8d971c80db099b95d1c785399e2bc6eb96cd07",
"keyword": "cf65309a880799a4a82a7b2f0389e5bc88f5730f",
"keyword": "ea42a4e238f7e54caa250fa2f5abef103cbf30ab",
}),
]

10
gallery_dl/extractor/dynastyscans.py
View File

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mike Fährmann
# Copyright 2015-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract manga-chapters from http://dynasty-scans.com/"""
"""Extract manga-chapters from https://dynasty-scans.com/"""
from .common import Extractor, Message
from .. import text
@ -24,16 +24,16 @@ class DynastyscansChapterExtractor(Extractor):
test = [
(("http://dynasty-scans.com/chapters/"
"hitoribocchi_no_oo_seikatsu_ch33"), {
"url": "63950fa1dfdef58ab842c1b9b854c5c1d650cfa0",
"url": "ba945116382eff26b0e52fe1804e6d58fee7734d",
"keyword": "81bfda5b98b34ac2a7324bd9e2abad3df9cc7673",
}),
(("http://dynasty-scans.com/chapters/"
"new_game_the_spinoff_special_13"), {
"url": "6b28c733481ac498da341e85a9eb155864491731",
"url": "2cd5e04bd16f842dc884c145a44cf0c64ec27a21",
"keyword": "93b75d0c0aaeb849c99f2225a4b97f466bc3ace9",
}),
]
url_base = "http://dynasty-scans.com/"
url_base = "https://dynasty-scans.com/"
def __init__(self, match):
Extractor.__init__(self)

4
gallery_dl/extractor/gelbooru.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from http://gelbooru.com/"""
"""Extract images from https://gelbooru.com/"""
from . import booru
from .. import config
@ -15,7 +15,7 @@ from .. import config
class GelbooruExtractor(booru.XMLBooruExtractor):
"""Base class for gelbooru extractors"""
category = "gelbooru"
api_url = "http://gelbooru.com/"
api_url = "https://gelbooru.com/"
pagestart = 0
pagekey = "pid"

21
gallery_dl/extractor/hentaifoundry.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from http://www.hentai-foundry.com/"""
"""Extract images from https://www.hentai-foundry.com/"""
from .common import Extractor, Message
from .. import text, exception
@ -25,15 +25,15 @@ class HentaifoundryUserExtractor(Extractor):
r"user/([^/]+)/profile"),
]
test = [
("http://www.hentai-foundry.com/pictures/user/Tenpura", {
"url": "35124cf236ffec596092446322b8f0ad603571c5",
("https://www.hentai-foundry.com/pictures/user/Tenpura", {
"url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28",
"keyword": "6e9a549feb9bafebd9d9342ef3c8ccad33a7031c",
}),
("http://www.hentai-foundry.com/user/asdq/profile", {
"exception": exception.NotFoundError,
}),
]
url_base = "http://www.hentai-foundry.com/pictures/user/"
url_base = "https://www.hentai-foundry.com/pictures/user/"
def __init__(self, match):
Extractor.__init__(self)
@ -85,7 +85,7 @@ class HentaifoundryUserExtractor(Extractor):
page, '//pictures.hentai-foundry.com', '"', pos)
data = {"index": index, "title": text.unescape(title)}
text.nameext_from_url(url, data)
return "http://pictures.hentai-foundry.com" + url, data
return "https://pictures.hentai-foundry.com" + url, data
def set_filters(self, token):
"""Set site-internal filters to show all images"""
@ -107,11 +107,14 @@ class HentaifoundryUserExtractor(Extractor):
"rating_female": 1,
"rating_futa": 1,
"rating_other": 1,
"rating_scat": 1,
"rating_incest": 1,
"rating_rape": 1,
"filter_media": "A",
"filter_order": "date_new",
"filter_type": 0,
}
self.request("http://www.hentai-foundry.com/site/filters",
self.request("https://www.hentai-foundry.com/site/filters",
method="post", data=formdata)
@ -127,7 +130,7 @@ class HentaifoundryImageExtractor(Extractor):
test = [
(("http://www.hentai-foundry.com/"
"pictures/user/Tenpura/407501/shimakaze"), {
"url": "b68d1b0121b97e01a878beeb2e43b07cb881b5a9",
"url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3",
"keyword": "304479cfe00fbb723886be78b2bd6b9306a31d8a",
"content": "91bf01497c39254b6dfb234a18e8f01629c77fd1",
}),
@ -149,7 +152,7 @@ class HentaifoundryImageExtractor(Extractor):
def get_image_metadata(self):
"""Collect metadata for an image"""
url = "http://www.hentai-foundry.com/pictures/user/{}/{}".format(
url = "https://www.hentai-foundry.com/pictures/user/{}/{}".format(
self.artist, self.index)
response = self.session.get(url + "?enterAgree=1")
if response.status_code == 404:
@ -165,4 +168,4 @@ class HentaifoundryImageExtractor(Extractor):
"title": text.unescape(title),
}
text.nameext_from_url(url, data)
return "http://pictures.hentai-foundry.com" + url, data
return "https://pictures.hentai-foundry.com" + url, data

6
gallery_dl/extractor/imgchili.py
View File

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 Mike Fährmann
# Copyright 2014-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from http://imgchili.net/"""
"""Extract images from https://imgchili.net/"""
from .common import Extractor, Message
from .. import text
@ -16,7 +16,7 @@ class ImgchiliExtractor(Extractor):
"""Base class for imgchili extractors"""
category = "imgchili"
directory_fmt = ["{category}"]
url_base = "http://imgchili.net/"
url_base = "https://imgchili.net/"
def __init__(self, match):
Extractor.__init__(self)

6
gallery_dl/extractor/khinsider.py
View File

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann
# Copyright 2016-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract soundtracks from http://khinsider.com/"""
"""Extract soundtracks from https://downloads.khinsider.com/"""
from .common import AsynchronousExtractor, Message
from .. import text, exception
@ -31,7 +31,7 @@ class KhinsiderSoundtrackExtractor(AsynchronousExtractor):
self.album = match.group(1)
def items(self):
url = ("http://downloads.khinsider.com/game-soundtracks/album/" +
url = ("https://downloads.khinsider.com/game-soundtracks/album/" +
self.album)
page = self.request(url, encoding="utf-8").text
data = self.get_job_metadata(page)

6
gallery_dl/extractor/nhentai.py
View File

@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mike Fährmann
# Copyright 2015-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from http://nhentai.net/"""
"""Extract images from https://nhentai.net/"""
from .common import Extractor, Message
from .. import text
@ -46,7 +46,7 @@ class NhentaiGalleryExtractor(Extractor):
def get_gallery_info(self):
"""Extract and return gallery-info"""
page = self.request("http://nhentai.net/g/" + self.gid + "/1/").text
page = self.request("https://nhentai.net/g/" + self.gid + "/1/").text
media_url, pos = text.extract(
page, ".reader({\n\t\t\tmedia_url: '", "'")
json_data, pos = text.extract(

6
gallery_dl/extractor/pixiv.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images and ugoira from http://www.pixiv.net/"""
"""Extract images and ugoira from https://www.pixiv.net/"""
from .common import Extractor, Message
from .. import config, text, exception
@ -31,8 +31,8 @@ class PixivUserExtractor(Extractor):
"exception": exception.NotFoundError,
}),
]
member_url = "http://www.pixiv.net/member_illust.php"
illust_url = "http://www.pixiv.net/member_illust.php?mode=medium"
member_url = "https://www.pixiv.net/member_illust.php"
illust_url = "https://www.pixiv.net/member_illust.php?mode=medium"
def __init__(self, match):
Extractor.__init__(self)

2
gallery_dl/extractor/powermanga.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for http://powermanga.org/"""
"""Extractors for https://powermanga.org/"""
from . import foolslide

8
gallery_dl/extractor/safebooru.py
View File

@ -6,7 +6,7 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract images from http://safebooru.org/"""
"""Extract images from https://safebooru.org/"""
from . import booru
@ -14,7 +14,7 @@ from . import booru
class SafebooruExtractor(booru.XMLBooruExtractor):
"""Base class for safebooru extractors"""
category = "safebooru"
api_url = "http://safebooru.org/index.php"
api_url = "https://safebooru.org/index.php"
pagestart = 0
pagekey = "pid"
@ -28,7 +28,7 @@ class SafebooruTagExtractor(SafebooruExtractor, booru.BooruTagExtractor):
pattern = [(r"(?:https?://)?(?:www\.)?safebooru\.org/(?:index\.php)?"
r"\?page=post&s=list&tags=([^&]+)")]
test = [("http://safebooru.org/index.php?page=post&s=list&tags=bonocho", {
"url": "c91e04ffbdf317fae95b2e160c8345503d9fb730",
"url": "17c61b386530cf4c30842c9f580d15ef1cd09586",
"content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb",
})]
@ -39,6 +39,6 @@ class SafebooruPostExtractor(SafebooruExtractor, booru.BooruPostExtractor):
pattern = [(r"(?:https?://)?(?:www\.)?safebooru\.org/(?:index\.php)?"
r"\?page=post&s=view&id=(\d+)")]
test = [("http://safebooru.org/index.php?page=post&s=view&id=1169132", {
"url": "bcb6047665729c7c9db243a27f41cbef9af1ecef",
"url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b",
"content": "93b293b27dabd198afafabbaf87c49863ac82f27",
})]

3
gallery_dl/job.py
View File

@ -61,7 +61,6 @@ class Job():
exc.__class__.__name__, err)
log.debug("Traceback", exc_info=True)
def dispatch(self, msg):
"""Call the appropriate message handler"""
if msg[0] == Message.Url:
@ -297,7 +296,7 @@ class DataJob(Job):
copy = [
part.copy() if hasattr(part, "copy") else part
for part in msg
]
]
self.data.append(copy)
except Exception as exc:
self.data.append((exc.__class__.__name__, str(exc)))

2
test/test_extractors.py
View File

@ -51,7 +51,7 @@ skip = [
# dont work on travis-ci
"exhentai", "kissmanga", "mangafox", "dynastyscans",
# temporary issues
"mangashare", "readcomics",
"mangashare", "readcomics", "pawoo",
]
# enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1: