Mirrors/gallery-dl
1
0
Fork 0
mirror of https://github.com/mikf/gallery-dl.git synced 2024-11-25 20:22:36 +01:00
Code Issues Releases Wiki Activity

update '--write-pages' (#737)

Browse Source 
- fix infinite recursion for responses with multiple entries in
  'history'
- hide values of Set-Cookie headers
- only write the response content by default
  (use '-o write-pages=all' to also include HTTP headers)
This commit is contained in:
Mike Fährmann 2020-06-18 15:07:30 +02:00
parent 7e8a747c56
commit 1ae1df0d27
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
gallery_dl/extractor/common.py
View File

@ -328,14 +328,15 @@ class Extractor():
test = (test, None)
yield test
def _dump_response(self, response):
def _dump_response(self, response, history=True):
"""Write the response content to a .dump file in the current directory.
The file name is derived from the response url,
replacing special characters with "_"
"""
for resp in response.history:
self._dump_response(resp)
if history:
for resp in response.history:
self._dump_response(resp, False)
if hasattr(Extractor, "_dump_index"):
Extractor._dump_index += 1
@ -350,7 +351,8 @@ class Extractor():
try:
with open(fname + ".dump", 'wb') as fp:
util.dump_response(response, fp)
util.dump_response(
response, fp, headers=(self._write_pages == "all"))
except Exception as e:
self.log.warning("Failed to dump HTTP request (%s: %s)",
e.__class__.__name__, e)

21
gallery_dl/util.py
View File

@ -120,13 +120,14 @@ def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4):
fp.write("\n")
def dump_response(response, fp=sys.stdout,
headers=True, content=True, hide_auth=True):
def dump_response(response, fp, *,
headers=False, content=True, hide_auth=True):
"""Write the contents of 'response' into a file-like object"""
if headers:
request = response.request
req_headers = request.headers.copy()
res_headers = response.headers.copy()
outfmt = """\
{request.method} {request.url}
Status: {response.status_code} {response.reason}
@ -145,11 +146,17 @@ Response Headers
atype, sep, _ = authorization.partition(" ")
req_headers["Authorization"] = atype + " ***" if sep else "***"
cookies = req_headers.get("Cookie")
if cookies:
cookie = req_headers.get("Cookie")
if cookie:
req_headers["Cookie"] = ";".join(
cookie.partition("=")[0] + "=***"
for cookie in cookies.split(";")
c.partition("=")[0] + "=***"
for c in cookie.split(";")
)
set_cookie = res_headers.get("Set-Cookie")
if set_cookie:
res_headers["Set-Cookie"] = re.sub(
r"(^|, )([^ =]+)=[^,;]*", r"\1\2=***", set_cookie,
)
fp.write(outfmt.format(
@ -161,7 +168,7 @@ Response Headers
),
response_headers="\n".join(
name + ": " + value
for name, value in response.headers.items()
for name, value in res_headers.items()
),
).encode())