2016-09-02 19:11:16 +02:00
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
2018-01-30 22:49:16 +01:00
|
|
|
# Copyright 2016-2018 Mike Fährmann
|
2016-09-02 19:11:16 +02:00
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
# published by the Free Software Foundation.
|
|
|
|
|
|
|
|
"""Extract images from https://www.pinterest.com"""
|
|
|
|
|
|
|
|
from .common import Extractor, Message
|
2018-03-06 14:28:21 +01:00
|
|
|
from .. import text, util, exception
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2017-02-01 00:53:19 +01:00
|
|
|
|
2016-09-03 10:00:32 +02:00
|
|
|
class PinterestExtractor(Extractor):
|
|
|
|
"""Base class for pinterest extractors"""
|
2016-09-02 19:11:16 +02:00
|
|
|
category = "pinterest"
|
2017-09-10 22:20:47 +02:00
|
|
|
filename_fmt = "{category}_{pin_id}.{extension}"
|
2018-01-30 22:49:16 +01:00
|
|
|
archive_fmt = "{pin_id}"
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2016-09-03 10:00:32 +02:00
|
|
|
def __init__(self):
|
2016-09-02 19:11:16 +02:00
|
|
|
Extractor.__init__(self)
|
2017-09-09 17:31:42 +02:00
|
|
|
self.api = PinterestAPI(self)
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2016-09-03 10:00:32 +02:00
|
|
|
def data_from_pin(self, pin):
|
|
|
|
"""Get image url and metadata from a pin-object"""
|
2016-09-02 19:11:16 +02:00
|
|
|
img = pin["image"]["original"]
|
2016-09-03 10:00:32 +02:00
|
|
|
url = img["url"]
|
2016-09-02 19:11:16 +02:00
|
|
|
data = {
|
2018-03-06 14:28:21 +01:00
|
|
|
"pin_id": util.safe_int(pin["id"]),
|
2016-09-02 19:11:16 +02:00
|
|
|
"note": pin["note"],
|
2018-03-06 14:28:21 +01:00
|
|
|
"width": util.safe_int(img["width"]),
|
|
|
|
"height": util.safe_int(img["height"]),
|
2016-09-02 19:11:16 +02:00
|
|
|
}
|
2016-09-03 10:00:32 +02:00
|
|
|
return url, text.nameext_from_url(url, data)
|
|
|
|
|
|
|
|
|
|
|
|
class PinterestPinExtractor(PinterestExtractor):
|
2016-09-12 10:20:57 +02:00
|
|
|
"""Extractor for images from a single pin from pinterest.com"""
|
2016-09-03 10:00:32 +02:00
|
|
|
subcategory = "pin"
|
2017-05-08 15:08:39 +02:00
|
|
|
pattern = [r"(?:https?://)?(?:[^./]+\.)?pinterest\.[^/]+/pin/([^/?#&]+)"]
|
2017-01-03 12:12:36 +01:00
|
|
|
test = [
|
|
|
|
("https://www.pinterest.com/pin/858146903966145189/", {
|
2018-03-06 14:28:21 +01:00
|
|
|
"url": "afb3c26719e3a530bb0e871c480882a801a4e8a5",
|
|
|
|
"keyword": "f651cb271247f306d1d30385d49c7b82da44c2b1",
|
2017-01-03 12:12:36 +01:00
|
|
|
"content": "d3e24bc9f7af585e8c23b9136956bd45a4d9b947",
|
|
|
|
}),
|
|
|
|
("https://www.pinterest.com/pin/858146903966145188/", {
|
2018-04-16 15:43:31 +02:00
|
|
|
"exception": exception.StopExtraction,
|
2017-01-03 12:12:36 +01:00
|
|
|
}),
|
2018-04-17 17:12:42 +02:00
|
|
|
("https://www.pinterest.com/pin/85814690396614518/", {
|
|
|
|
"exception": exception.NotFoundError,
|
|
|
|
}),
|
2017-01-03 12:12:36 +01:00
|
|
|
]
|
2016-09-03 10:00:32 +02:00
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
PinterestExtractor.__init__(self)
|
|
|
|
self.pin_id = match.group(1)
|
|
|
|
|
|
|
|
def items(self):
|
|
|
|
pin = self.api.pin(self.pin_id)
|
|
|
|
url, data = self.data_from_pin(pin)
|
|
|
|
yield Message.Version, 1
|
|
|
|
yield Message.Directory, data
|
|
|
|
yield Message.Url, url, data
|
|
|
|
|
|
|
|
|
|
|
|
class PinterestBoardExtractor(PinterestExtractor):
|
2016-09-12 10:20:57 +02:00
|
|
|
"""Extractor for images from a board from pinterest.com"""
|
2016-09-03 10:00:32 +02:00
|
|
|
subcategory = "board"
|
|
|
|
directory_fmt = ["{category}", "{user}", "{board}"]
|
2017-05-08 15:08:39 +02:00
|
|
|
pattern = [r"(?:https?://)?(?:[^./]+\.)?pinterest\.[^/]+/"
|
|
|
|
r"(?!pin/)([^/?#&]+)/([^/?#&]+)"]
|
2017-01-03 12:12:36 +01:00
|
|
|
test = [
|
|
|
|
("https://www.pinterest.com/g1952849/test-/", {
|
2018-03-06 14:28:21 +01:00
|
|
|
"url": "85911dfca313f3f7f48c2aa0bc684f539d1d80a6",
|
|
|
|
"keyword": "c54cf5aa830994f2ed4871efa7154a5fdaa1c2ce",
|
2017-01-03 12:12:36 +01:00
|
|
|
}),
|
|
|
|
("https://www.pinterest.com/g1952848/test/", {
|
|
|
|
"exception": exception.NotFoundError,
|
|
|
|
}),
|
|
|
|
]
|
2016-09-03 10:00:32 +02:00
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
PinterestExtractor.__init__(self)
|
|
|
|
self.user, self.board = match.groups()
|
|
|
|
|
|
|
|
def items(self):
|
|
|
|
board = self.api.board(self.user, self.board)
|
|
|
|
data = self.data_from_board(board)
|
|
|
|
num = data["count"]
|
2016-09-02 19:11:16 +02:00
|
|
|
yield Message.Version, 1
|
|
|
|
yield Message.Directory, data
|
2016-09-03 10:00:32 +02:00
|
|
|
for pin in self.api.board_pins(self.user, self.board):
|
|
|
|
url, pdata = self.data_from_pin(pin)
|
|
|
|
data.update(pdata)
|
|
|
|
data["num"] = num
|
|
|
|
num -= 1
|
|
|
|
yield Message.Url, url, data
|
|
|
|
|
|
|
|
def data_from_board(self, board):
|
|
|
|
"""Get metadata from a board-object"""
|
|
|
|
data = {
|
|
|
|
"user": self.user,
|
2018-03-06 14:28:21 +01:00
|
|
|
"board_id": util.safe_int(board["id"]),
|
2016-09-03 10:00:32 +02:00
|
|
|
"board": board["name"],
|
|
|
|
"count": board["counts"]["pins"],
|
|
|
|
}
|
|
|
|
return data
|
2016-09-02 19:11:16 +02:00
|
|
|
|
|
|
|
|
2017-01-27 21:51:28 +01:00
|
|
|
class PinterestPinitExtractor(PinterestExtractor):
|
2017-05-08 15:08:39 +02:00
|
|
|
"""Extractor for images from a pin.it URL"""
|
2017-01-27 21:51:28 +01:00
|
|
|
subcategory = "pinit"
|
2017-05-08 15:08:39 +02:00
|
|
|
pattern = [r"(?:https?://)?(pin\.it/[^/?#&]+)"]
|
2017-01-27 21:51:28 +01:00
|
|
|
test = [
|
|
|
|
("https://pin.it/Hvt8hgT", {
|
|
|
|
"url": "8daad8558382c68f0868bdbd17d05205184632fa",
|
|
|
|
}),
|
|
|
|
("https://pin.it/Hvt8hgS", {
|
|
|
|
"exception": exception.NotFoundError,
|
2017-05-08 15:08:39 +02:00
|
|
|
}),
|
2017-01-27 21:51:28 +01:00
|
|
|
]
|
|
|
|
|
|
|
|
def __init__(self, match):
|
|
|
|
PinterestExtractor.__init__(self)
|
|
|
|
self.url = "https://" + match.group(1)
|
|
|
|
|
|
|
|
def items(self):
|
|
|
|
response = self.session.head(self.url)
|
|
|
|
location = response.headers.get("Location")
|
2017-01-27 22:39:44 +01:00
|
|
|
if not location or location in ("https://api.pinterest.com/None",
|
|
|
|
"https://www.pinterest.com"):
|
2017-01-27 21:51:28 +01:00
|
|
|
raise exception.NotFoundError("pin")
|
2017-09-12 16:19:00 +02:00
|
|
|
yield Message.Queue, location, {}
|
2017-01-27 21:51:28 +01:00
|
|
|
|
|
|
|
|
2016-09-02 19:11:16 +02:00
|
|
|
class PinterestAPI():
|
|
|
|
"""Minimal interface for the pinterest API"""
|
|
|
|
|
2018-04-17 17:12:42 +02:00
|
|
|
def __init__(self, extractor, access_token=None):
|
2018-04-16 15:43:31 +02:00
|
|
|
self.log = extractor.log
|
2018-04-17 17:12:42 +02:00
|
|
|
self.session = extractor.session
|
|
|
|
self.access_token = (
|
|
|
|
access_token or
|
|
|
|
extractor.config("access-token") or
|
|
|
|
"AfyIXxi1MJ6et0NlIl_vBchHbex-FSWylPyr2GJE2uu3W8A97QAAAAA"
|
|
|
|
)
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2016-09-03 10:00:32 +02:00
|
|
|
def pin(self, pin_id, fields="id,image,note"):
|
2016-09-02 19:11:16 +02:00
|
|
|
"""Query information about a pin"""
|
2018-04-17 17:12:42 +02:00
|
|
|
endpoint = "pins/{}/".format(pin_id)
|
2016-09-02 19:11:16 +02:00
|
|
|
params = {"fields": fields}
|
2018-04-17 17:12:42 +02:00
|
|
|
return self._call(endpoint, params)["data"]
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2016-09-03 10:00:32 +02:00
|
|
|
def board(self, user, board, fields="id,name,counts"):
|
|
|
|
"""Query information about a board"""
|
2018-04-17 17:12:42 +02:00
|
|
|
endpoint = "boards/{}/{}/".format(user, board)
|
2016-09-03 10:00:32 +02:00
|
|
|
params = {"fields": fields}
|
2018-04-17 17:12:42 +02:00
|
|
|
return self._call(endpoint, params)["data"]
|
2016-09-03 10:00:32 +02:00
|
|
|
|
2018-04-17 17:12:42 +02:00
|
|
|
def board_pins(self, user, board, fields="id,image,note", limit=100):
|
2016-09-02 19:11:16 +02:00
|
|
|
"""Yield all pins of a specific board"""
|
2018-04-17 17:12:42 +02:00
|
|
|
endpoint = "boards/{}/{}/pins/".format(user, board)
|
|
|
|
params = {"fields": fields, "limit": limit}
|
|
|
|
return self._pagination(endpoint, params)
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2018-04-17 17:12:42 +02:00
|
|
|
def _call(self, endpoint, params):
|
|
|
|
params["access_token"] = self.access_token
|
|
|
|
url = "https://api.pinterest.com/v1/" + endpoint
|
2016-09-02 19:11:16 +02:00
|
|
|
|
2018-04-17 17:12:42 +02:00
|
|
|
response = self.session.get(url, params=params)
|
|
|
|
status = response.status_code
|
2016-09-04 11:00:38 +02:00
|
|
|
data = response.json()
|
2018-04-17 17:12:42 +02:00
|
|
|
|
|
|
|
if 200 <= status < 400 and data.get("data"):
|
2018-04-16 15:43:31 +02:00
|
|
|
return data
|
|
|
|
|
2018-04-17 17:12:42 +02:00
|
|
|
msg = data.get("message", "")
|
|
|
|
if status == 404:
|
2018-04-16 15:43:31 +02:00
|
|
|
msg = msg.partition(" ")[0].lower()
|
2016-09-04 11:00:38 +02:00
|
|
|
raise exception.NotFoundError(msg)
|
2018-04-17 17:12:42 +02:00
|
|
|
self.log.error("API request failed: %s", msg or "")
|
|
|
|
raise exception.StopExtraction()
|
|
|
|
|
|
|
|
def _pagination(self, endpoint, params):
|
|
|
|
while True:
|
|
|
|
response = self._call(endpoint, params)
|
|
|
|
yield from response["data"]
|
|
|
|
|
|
|
|
cursor = response["page"]["cursor"]
|
|
|
|
if not cursor:
|
|
|
|
return
|
|
|
|
params["cursor"] = cursor
|