Mercurial > codedump

annotate kemonopartydownloader.py @ 97:f1f4f6da04bd

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
kemonopartydownloader.py: convert to percent formatting, add typing to functions and "fix" the drive downloading committer: GitHub <noreply@github.com>
author Paper <37962225+mrpapersonic@users.noreply.github.com>
date Sat, 13 Aug 2022 20:27:58 -0400
parents d2e0edd4a070
children e4bf37150a3f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
1 import argparse
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
2 import http.cookiejar
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
3 import os
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
4 import re
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
5 import requests # pip install requests
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
6 import time
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
7 import math
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
8 import zipfile
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
9 import urllib.parse
96
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
10 import sys
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
11 from urllib.error import HTTPError
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
12 from http.client import BadStatusLine
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
13
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
14
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
15 def under_num(maximum: int, num: int) -> int:
96
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
16 return num if num <= maximum else maximum
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
17
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
18 def download_folder_from_google_drive(link: str) -> int:
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
19 session = requests.Session()
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
20 session.headers = {
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
21 'origin': 'https://drive.google.com',
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
22 'content-type': 'application/json',
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
23 }
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
24 key = "AIzaSyC1qbk75NzWBvSaDh6KnsjjA9pIrP4lYIE" # google anonymous key
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
25 takeoutjs = session.post("https://takeout-pa.clients6.google.com/v1/exports?key=%s" % (key), data='{"items":[{"id":"%s"}]}' % (link.split("?")[0].split("/")[-1])).json()
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
26 takeoutid = str(takeoutjs["exportJob"]["id"])
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
27 storagePath = None
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
28 while storagePath is None:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
29 succeededjson = session.get("https://takeout-pa.clients6.google.com/v1/exports/%s?key=%s" % (takeoutid, key)).json()
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
30 if succeededjson["exportJob"]["status"] == "SUCCEEDED":
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
31 storagePath = str(succeededjson["exportJob"]["archives"][0]["storagePath"])
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
32 time.sleep(1)
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
33 size = 0
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
34 for path, dirs, files in os.walk("./%s/Drive - %s" % (output, sanitize(i["title"]))):
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
35 for f in files:
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
36 fp = os.path.join(path, f)
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
37 size += os.path.getsize(fp)
96
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
38 try:
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
39 if size >= int(succeededjson["exportJob"]["archives"][0]["sizeOfContents"]):
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
40 print(" %s already downloaded!" % (succeededjson["exportJob"]["archives"][0]["fileName"]))
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
41 return 1
96
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
42 except Exception as e:
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
43 print(" %s download failed! %s" % (succeededjson["exportJob"]["archives"][0]["fileName"], str(e)))
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
44 print(e)
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
45 response = session.get(storagePath, stream=True)
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
46 amountdone = 0
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
47 with open(succeededjson["exportJob"]["archives"][0]["fileName"], "wb") as f:
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
48 for chunk in response.iter_content(1024):
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
49 if chunk: # filter out keep-alive new chunks
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
50 f.write(chunk)
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
51 amountdone += 1024
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
52 print(" downloading %s: %.2f%%" % (succeededjson["exportJob"]["archives"][0]["fileName"], (amountdone / int(succeededjson['exportJob']['archives'][0]['compressedSize'])) * 100), end="\r")
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
53 print(" downloaded %s: 100.00%% " % (succeededjson["exportJob"]["archives"][0]["fileName"]))
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
54 unzip(succeededjson["exportJob"]["archives"][0]["fileName"], "./%s/Drive - %s" % (output, sanitize(i["title"])))
96
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
55 os.remove(succeededjson["exportJob"]["archives"][0]["fileName"])
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
56 return 0
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
57
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
58
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
59 def unzip(src_path: str, dst_dir: str, pwd=None) -> None:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
60 with zipfile.ZipFile(src_path) as zf:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
61 members = zf.namelist()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
62 for member in members:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
63 arch_info = zf.getinfo(member)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
64 arch_name = arch_info.filename.replace('/', os.path.sep)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
65 dst_path = os.path.join(dst_dir, arch_name)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
66 dst_path = os.path.normpath(dst_path)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
67 if not os.path.exists(dst_path):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
68 zf.extract(arch_info, dst_dir, pwd)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
69
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
70
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
71 def download_from_dropbox(link: str) -> None:
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
72 responsehead = req.head(link.split("?")[0] + "?dl=1", allow_redirects=True)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
73 if responsehead.status_code == 404:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
74 print(" dropbox link not available!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
75 return
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
76 if not os.path.exists(output + "/Dropbox - " + sanitize(i["title"])):
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
77 os.makedirs(output + "/Dropbox - " + sanitize(i["title"]))
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
78 filename = "%s/Dropbox - %s/%s" % (output, sanitize(i["title"]), sanitize(responsehead.headers["Content-Disposition"].split("'")[-1]))
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
79 if os.path.exists(urllib.parse.unquote(os.path.splitext(filename)[0])) and os.path.isdir(urllib.parse.unquote(os.path.splitext(filename)[0])):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
80 print(" file(s) already downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
81 return
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
82 filesize = os.stat(filename).st_size if os.path.exists(filename) else 0
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
83 # will always be 0 if it's a folder...
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
84 if filesize == 0:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
85 with req.get(link.split("?")[0] + "?dl=1", stream=True, headers={"Range": "bytes=%d-" % (filesize)}) as r:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
86 r.raise_for_status()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
87 with open(filename, "ab") as f:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
88 for chunk in r.iter_content(chunk_size=4096):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
89 f.write(chunk)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
90 filesize += 4096
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
91 print(" file %s downloading..." % (urllib.parse.unquote(responsehead.headers["Content-Disposition"].split("'")[-1])), end="\r")
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
92 print(" file %s successfully downloaded!" % (urllib.parse.unquote(responsehead.headers["Content-Disposition"].split("'")[-1])))
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
93 if responsehead.headers["Content-Disposition"].split("'")[-1].endswith(".zip"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
94 unzip(filename, urllib.parse.unquote(os.path.splitext(filename)[0]))
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
95 os.remove(filename)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
96
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
97
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
98 def download_file_from_google_drive(drive_id: str, out: str = "") -> None: # https://stackoverflow.com/questions/25010369/wget-curl-large-file-from-google-drive/39225039
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
99 def get_confirm_token(response: requests.Response):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
100 for key, value in response.cookies.items():
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
101 if key.startswith('download_warning'):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
102 return value
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
103
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
104 return None
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
105
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
106 def save_response_content(response: requests.Response):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
107 amountdone = 0
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
108 CHUNK_SIZE = 4096
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
109 filename = "%s/Drive - %s/%s" % (output, sanitize(i["title"]), out)
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
110 if not os.path.exists():
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
111 os.makedirs(filename)
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
112 destination = filename + "/" + sanitize(response.headers["Content-Disposition"].split("'")[-1])
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
113 filesize = os.stat(destination).st_size if os.path.exists(destination) else 0
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
114
55
4e5000c9b48f Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 54
diff changeset
115 if os.path.exists(destination) and filesize == int(response.headers["Content-Range"].partition('/')[-1]):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
116 print(" " + os.path.basename(destination) + " already downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
117 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
118
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
119 with open(destination, "wb") as f:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
120 for chunk in response.iter_content(CHUNK_SIZE):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
121 if chunk: # filter out keep-alive new chunks
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
122 f.write(chunk)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
123 amountdone += CHUNK_SIZE
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
124 print(" downloading %s: %.2f%%" % (os.path.basename(destination), (amountdone / int(response.headers["Content-Range"].partition('/')[-1]))), end="\r")
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
125 print(" downloaded %s: %.2f%% " % (os.path.basename(destination), 100.0))
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
126
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
127 URL = "https://docs.google.com/uc?export=download"
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
128
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
129 session = requests.Session()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
130
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
131 headers = {
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
132 "Range": "bytes=0-",
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
133 }
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
134
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
135 session.proxies = req.proxies
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
136
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
137 response = session.get(URL, headers=headers, params={'id': drive_id}, stream=True)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
138
55
4e5000c9b48f Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 54
diff changeset
139 while response.status_code == 403:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
140 time.sleep(30)
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
141 response = session.get(URL, headers=headers, params={'id': drive_id}, stream=True)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
142
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
143 if response.status_code == 404:
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
144 return # bypass when root folder has no files
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
145
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
146 token = get_confirm_token(response)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
147
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
148 if token:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
149 params = {'id': drive_id, 'confirm': token}
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
150 response = session.get(URL, headers=headers, params=params, stream=True)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
151
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
152 save_response_content(response)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
153
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
154
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
155 def sanitize(filename: str) -> str:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
156 return re.sub(r"[\/:*?\"<>|]", "_", filename).strip()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
157
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
158
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
159 def find_urls(s: str) -> list:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
160 urllist = []
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
161 for findall in re.findall(r"""http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+""", s):
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
162 urllist.append(findall.split("<")[0].split(">")[-1])
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
163 return urllist
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
164
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
165
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
166 def download_file(i: dict, x: dict, count: int) -> None:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
167 filename = "%s/%s_%dp_%s_%s" % (output, i["id"], count, sanitize(i["title"]), x["name"])
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
168 amountdone = 0
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
169 filesize = os.stat(filename).st_size if os.path.exists(filename) else 0
96
d2e0edd4a070 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 56
diff changeset
170 serverhead = req.head("https://kemono.party/data" + x['path'], allow_redirects=True)
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
171 for i in range(500):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
172 serverfilesize = int(serverhead.headers["Content-Length"])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
173 if filesize < serverfilesize:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
174 with req.get("https://kemono.party/data" + x['path'], stream=True, headers={"Range": "bytes=%d-" % (filesize)}) as r:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
175 r.raise_for_status()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
176 with open(filename, "ab") as f:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
177 for chunk in r.iter_content(chunk_size=4096):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
178 f.write(chunk)
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
179 amountdone += len(chunk)
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
180 print(" downloading image %d: %.2f%%" % (count, under_num(100, round(((filesize + amountdone) / serverfilesize) * 100, 2))), end="\r")
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
181 print(" downloaded image " + str(count) + ": 100.00% ")
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
182 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
183 else:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
184 print(" image " + str(count) + " already downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
185 return
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
186 time.sleep(10)
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
187 print(" download timed out!")
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
188 return
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
189
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
190
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
191 def parse_json(i: dict, count: int) -> None:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
192 seen_gdrive_ids = set()
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
193 unique_urls = []
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
194 for url in find_urls(i["content"]):
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
195 parsed_url = urllib.parse.urlparse(url)
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
196 if parsed_url.netloc == "drive.google.com":
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
197 if parsed_url.path.startswith("drive/folders"):
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
198 if parsed_url.path.split("/")[-1] not in seen_gdrive_ids:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
199 unique_urls.append(url)
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
200 seen_gdrive_ids.add(parsed_url.path.split("/")[-1])
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
201 elif parsed_url.path == "open" and parsed_url.query.startswith == "id":
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
202 if parsed_url.query.split("=")[-1] not in seen_gdrive_ids:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
203 unique_urls.append(req.head(url).headers["Location"], allow_redirects=True)
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
204 seen_gdrive_ids.add(parsed_url.query.split("=")[-1])
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
205 elif parsed_url.path.startswith("file/"):
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
206 if parsed_url.path.split("/")[-2] not in seen_gdrive_ids:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
207 unique_urls.append(url)
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
208 seen_gdrive_ids.add(url.split("?")[0].split("/")[-2])
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
209 elif parsed_url.netloc in ["dropbox.com", "www.dropbox.com"]:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
210 if url not in unique_urls:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
211 unique_urls.append(url)
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
212 else:
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
213 pass
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
214 for url in unique_urls:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
215 if url.startswith("https://drive.google.com/drive/folders/"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
216 # Google Drive folder downloading
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
217 print(" Google Drive link found! attempting to download its files...")
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
218 download_folder_from_google_drive(url)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
219 elif url.startswith("https://drive.google.com/file/"):
55
4e5000c9b48f Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 54
diff changeset
220 print(" Google Drive link found! attempting to download its files...")
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
221 download_file_from_google_drive(url.split("?")[0].split("/")[-2])
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
222 elif url.startswith("https://www.dropbox.com/"):
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
223 print(" Dropbox link found! attempting to download its files...")
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
224 download_from_dropbox(url)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
225 for x in i["attachments"]:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
226 count += 1
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
227 while not os.path.exists("%s/%d_%dp_%s_%s" % (output, int(i["id"]) - 1, count, sanitize(i["title"]), x["name"])):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
228 try:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
229 download_file(i, x, count)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
230 break
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
231 except (HTTPError, BadStatusLine):
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
232 while 1:
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
233 time.sleep(10)
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
234 download_file(i, x, count)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
235 except Exception as e:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
236 print(e)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
237 time.sleep(10)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
238
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
239
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
240 def get_amount_of_posts(s: str, u: str):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
241 amount = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
242 while 1:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
243 data = req.get("https://kemono.party/api/%s/user/%s?o=%d" % (s, u, amount)).json()
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
244 if len(data) < 25:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
245 return math.ceil((amount + 1) / 25)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
246 amount += 25
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
247
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
248
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
249 parser = argparse.ArgumentParser(description="Downloads files from kemono.party")
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
250 parser.add_argument("-u", "--url", help="user URL", metavar='<url>', required=True)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
251 parser.add_argument("-c", "--cookies", help="", metavar='<cookies>', required=True) # required because of DDoS-GUARD
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
252 parser.add_argument("-p", "--proxy", help="proxy\n supported types: http, https, socks5 (requires pysocks)", metavar='<proxy>') # SOCKS proxy support is through PySocks - pip install pysocks
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
253 parser.add_argument("-o", "--output", help="output folder, defaults to user ID", metavar='<output>')
56
bde647ac9554 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 55
diff changeset
254 parser.add_argument("--test-download-services", dest="testdownloadservices", action="store_true", help="test download services")
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
255 args = parser.parse_args()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
256
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
257 req = requests.Session()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
258
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
259 if args.testdownloadservices:
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
260 output = "download_services_test"
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
261 i = {"title": "DEEZ NUTS"}
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
262 download_folder_from_google_drive("https://drive.google.com/drive/folders/1rZN2ejZnGdF0EpaZuknlDp26a0qSjsEI")
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
263 download_from_dropbox("https://www.dropbox.com/s/yg405bpznyobo3u/test.txt?dl=0") # File
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
264 download_from_dropbox("https://www.dropbox.com/sh/ne3c7bxtkt5tg4s/AABYPNGfHoil4HO_btudw0wPa?dl=0") # Folder
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
265 exit()
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
266
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
267 if args.proxy:
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
268 req.proxies = {}
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
269 if args.proxy[:6] == "socks5":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
270 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
271 httpsproxy = args.proxy
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
272 elif args.proxy[:5] == "https":
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
273 httpsproxy = args.proxy
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
274 elif args.proxy[:4] == "http":
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
275 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
276 else:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
277 print("unknown proxy format! defaulting to HTTP...")
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
278 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
279 if httpproxy:
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
280 req.proxies["http"] = httpproxy
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
281 if httpsproxy:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
282 req.proxies["https"] = httpsproxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
283
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
284 cj = http.cookiejar.MozillaCookieJar(args.cookies)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
285 cj.load(ignore_expires=True)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
286 req.cookies = cj
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
287
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
288 try:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
289 int(args.url)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
290 print("do not input user IDs in --url! use a link instead")
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
291 exit()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
292 except Exception:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
293 pass
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
294
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
295 # TODO: use urlparse here...
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
296 if args.url.split("/")[-2] == "post":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
297 service = args.url.split("/")[-5]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
298 user = args.url.split("/")[-3]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
299 post = args.url.split("/")[-1]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
300 elif args.url.split("/")[-2] == "user":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
301 service = args.url.split("/")[-3]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
302 user = args.url.split("/")[-1]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
303
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
304 if not args.output:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
305 output = "%s-%s" % (service, user)
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
306 else:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
307 output = args.output
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
308
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
309 if not os.path.isdir(output):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
310 if os.path.exists(output):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
311 os.remove(output)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
312 os.makedirs(output)
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
313
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
314 try:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
315 post
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
316 pages = 1
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
317 except Exception:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
318 pages = get_amount_of_posts(service, user)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
319 for page in range(pages):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
320 try:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
321 post
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
322 userdata = req.get("https://kemono.party/api/%s/user/%s/post/%s" % (service, user, post)).json()
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
323 except Exception:
97
f1f4f6da04bd kemonopartydownloader.py: convert to percent formatting,
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 96
diff changeset
324 userdata = req.get("https://kemono.party/api/%s/user/%s?o=%s" % (service, user, (page * 25))).json()
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
325 for i in userdata:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
326 print(i["id"])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
327 count = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
328 parse_json(i, count)