Mercurial > codedump

annotate kemonopartydownloader.py @ 54:5a5d47a795c6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Update kemonopartydownloader.py how to easily make your code thrice as long committer: GitHub <noreply@github.com>
author Paper <37962225+mrpapersonic@users.noreply.github.com>
date Fri, 06 Aug 2021 02:50:53 -0400
parents ae64a0c8831b
children 4e5000c9b48f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
1 # example args.url: https://kemono.party/fanbox/user/5375435/post/2511461
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
2 # created by Paper in 2021
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
3 # please do not share without crediting me!
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
4 import argparse
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
5 import http.cookiejar
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
6 import os
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
7 import re
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
8 import requests # pip install requests
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
9 import time
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
10 import math
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
11 import zipfile
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
12 import urllib.parse
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
13 from urllib.error import HTTPError
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
14
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
15
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
16 def get_google_drive_subfolder_ids(link):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
17 gdrive = requests.get(link).text
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
18 drivefiles = re.findall(r"\[\"(.{33}?)\",\[\"(.{33}?)\"\],\"(.+?)\",\"(.+?)\"", gdrive) # format: ["id","name","mimetype"
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
19 seen = set()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
20 unique_ids = []
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
21 for files in drivefiles:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
22 if files[3] != "application/vnd.google-apps.folder":
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
23 continue
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
24 if files[0] not in seen:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
25 unique_ids.append(files[0])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
26 seen.add(files[0])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
27 return unique_ids
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
28
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
29
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
30 def unzip(src_path, dst_dir, pwd=None):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
31 with zipfile.ZipFile(src_path) as zf:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
32 members = zf.namelist()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
33 for member in members:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
34 arch_info = zf.getinfo(member)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
35 arch_name = arch_info.filename.replace('/', os.path.sep)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
36 dst_path = os.path.join(dst_dir, arch_name)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
37 dst_path = os.path.normpath(dst_path)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
38 if not os.path.exists(dst_path):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
39 zf.extract(arch_info, dst_dir, pwd)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
40
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
41
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
42 def download_from_dropbox(link):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
43 responsehead = requests.head(link.split("?")[0] + "?dl=1", allow_redirects=True)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
44 if responsehead.status_code == 404:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
45 print(" dropbox link not available!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
46 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
47 if not os.path.exists(output + "\\Dropbox - " + sanitize(i["title"])):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
48 os.makedirs(output + "\\Dropbox - " + sanitize(i["title"]))
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
49 filename = output + "\\Dropbox - " + sanitize(i["title"]) + "\\" + sanitize(responsehead.headers["Content-Disposition"].split("'")[-1])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
50 if os.path.exists(urllib.parse.unquote(os.path.splitext(filename)[0])) and os.path.isdir(urllib.parse.unquote(os.path.splitext(filename)[0])):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
51 print(" file(s) already downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
52 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
53 if os.path.exists(filename):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
54 filesize = os.stat(filename).st_size
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
55 else:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
56 filesize = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
57 serverfilesize = int(responsehead.headers["Content-Length"])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
58 if filesize < serverfilesize:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
59 with req.get(link.split("?")[0] + "?dl=1", stream=True, headers={"Range": f"bytes={filesize}-"}) as r:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
60 r.raise_for_status()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
61 with open(filename, "ab") as f:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
62 for chunk in r.iter_content(chunk_size=4096):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
63 f.write(chunk)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
64 filesize += 4096
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
65 print(" file {0} downloading: ".format(urllib.parse.unquote(responsehead.headers["Content-Disposition"].split("'")[-1])) + str(round((filesize / serverfilesize) * 100)) + "%\r", end="")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
66 print(" {0} successfully downloaded!".format(urllib.parse.unquote(responsehead.headers["Content-Disposition"].split("'")[-1])))
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
67 if responsehead.headers["Content-Disposition"].split("'")[-1].endswith(".zip"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
68 unzip(filename, urllib.parse.unquote(os.path.splitext(filename)[0]))
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
69 os.remove(filename)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
70
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
71
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
72 def download_file_from_google_drive(id): # https://stackoverflow.com/questions/25010369/wget-curl-large-file-from-google-drive/39225039 ;)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
73 def get_confirm_token(response):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
74 for key, value in response.cookies.items():
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
75 if key.startswith('download_warning'):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
76 return value
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
77
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
78 return None
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
79
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
80 def save_response_content(response):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
81 amountdone = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
82 CHUNK_SIZE = 32768
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
83 if not os.path.exists(output + "\\Drive - " + sanitize(i["title"])):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
84 os.makedirs(output + "\\Drive - " + sanitize(i["title"]))
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
85 destination = output + "\\Drive - " + sanitize(i["title"]) + "\\" + sanitize(response.headers["Content-Disposition"].split("'")[-1])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
86 if os.path.exists(destination):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
87 filesize = os.stat(destination).st_size
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
88 else:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
89 filesize = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
90
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
91 if os.path.exists(destination):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
92 print(" " + os.path.basename(destination) + " already downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
93 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
94
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
95 with open(destination, "wb") as f:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
96 for chunk in response.iter_content(CHUNK_SIZE):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
97 if chunk: # filter out keep-alive new chunks
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
98 f.write(chunk)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
99 amountdone += CHUNK_SIZE
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
100 print(" downloading {0}: ".format(os.path.basename(destination)) + " " + str(round(filesize + amountdone / int(response.headers["Content-Range"].partition('/')[-1])) * 100) + "%\r", end="")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
101 print(" downloaded {0}".format(os.path.basename(destination)) + ": 100% ")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
102
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
103 URL = "https://docs.google.com/uc?export=download"
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
104
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
105 session = requests.Session()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
106
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
107 headers = {
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
108 "Range": "bytes=0-",
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
109 }
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
110
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
111 response = session.get(URL, headers=headers, params={'id': id}, stream=True)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
112
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
113 while response.status_code == "403":
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
114 time.sleep(30)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
115 response = session.get(URL, headers=headers, params={'id': id}, stream=True)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
116
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
117 token = get_confirm_token(response)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
118
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
119 if token:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
120 params = {'id': id, 'confirm': token}
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
121 response = session.get(URL, headers=headers, params=params, stream=True)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
122
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
123 save_response_content(response)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
124
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
125
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
126 def sanitize(filename):
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
127 return re.sub(r"[\/:*?\"<>|]", "_", filename).strip()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
128
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
129
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
130 def find_urls(s):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
131 urllist = []
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
132 for findall in re.findall("href=\\\"(https://.+?)\\\"", s):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
133 urllist.append(re.sub(r"<[^<]+?>", "", re.sub(r"[^a-zA-Z0-9<>]+$", "", findall)))
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
134 return urllist
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
135
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
136
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
137 def downloadfile(i, x, count):
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
138 filename = "{4}\\{0}_{1}p_{2}_{3}".format(i["id"], count, sanitize(i["title"]), os.path.basename(x["path"]), output)
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
139 if os.path.exists(filename):
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
140 filesize = os.stat(filename).st_size
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
141 else:
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
142 filesize = 0
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
143 serverhead = req.head("https://data.kemono.party" + x['path'])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
144 try:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
145 serverfilesize = int(serverhead.headers["Content-Length"])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
146 if filesize < serverfilesize:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
147 with req.get(f"https://data.kemono.party{x['path']}", stream=True, headers={"Range": f"bytes={filesize}-"}) as r:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
148 r.raise_for_status()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
149 with open(filename, "ab") as f:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
150 for chunk in r.iter_content(chunk_size=4096):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
151 f.write(chunk)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
152 print(" image " + str(count) + " successfully downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
153 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
154 else:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
155 print(" image " + str(count) + " already downloaded!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
156 return
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
157 except Exception as e:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
158 print(" error downloading file!")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
159 print(e)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
160
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
161
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
162 def parse_json(i, count):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
163 seen = set()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
164 unique_urls = []
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
165 for url in find_urls(i["content"]):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
166 if url.startswith("https://drive.google.com/drive/folders"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
167 if url.split("/")[-1].split("?")[0] not in seen:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
168 unique_urls.append(url)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
169 seen.add(url.split("/")[-1].split("?")[0])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
170 elif url.startswith("https://drive.google.com/open?id="):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
171 if url.split("?id=")[-1] not in seen:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
172 unique_urls.append(requests.head(url).headers["Location"])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
173 seen.add(url.split("/")[-1].split("?")[0])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
174 elif url.startswith("https://drive.google.com/file/"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
175 if url.split("?")[0].split("/")[-2] not in seen:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
176 unique_urls.append(url)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
177 seen.add(url.split("?")[0].split("/")[-2])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
178 elif url.startswith("https://www.dropbox.com"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
179 download_from_dropbox(url)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
180 else: # TODO: add MEGA, or some sort of other file hosting website(s). gdrive and dropbox seem like the most popular ones atm
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
181 pass
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
182 for url in unique_urls:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
183 if url.startswith("https://drive.google.com/drive/folders/"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
184 # Google Drive folder downloading
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
185 # NOTE: this doesn't currently support subfolders! they seem like a pain in the ass to implement without the api...
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
186 print(" Google Drive link found! attempting to download its files...")
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
187 unique_ids = [url.split("?")[0].split("/")[-1]]
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
188 drive_ids_to_download = []
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
189 while len(unique_ids) > 0:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
190 for myid in unique_ids:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
191 unique_ids = get_google_drive_subfolder_ids("https://drive.google.com/drive/folders/" + myid)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
192 for ids in unique_ids:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
193 drive_ids_to_download.append(ids)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
194 for ids in drive_ids_to_download:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
195 gdrive = requests.get("https://drive.google.com/drive/folders/" + ids).text
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
196 driveids = re.findall(r'jsdata=" M2rjcd;_;\d (?:.+?);(.+?);', gdrive)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
197 for driveid in driveids:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
198 if not driveid.startswith("driveweb|"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
199 download_file_from_google_drive(driveid)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
200 elif url.startswith("https://drive.google.com/file/"):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
201 download_file_from_google_drive(url.split("?")[0].split("/")[-2])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
202 for x in i["attachments"]:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
203 count += 1
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
204 while not os.path.exists("{4}\\{0}_{1}p_{2}_{3}".format(int(i["id"]) - 1, count, sanitize(i["title"]), os.path.basename(x["path"]), output)):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
205 try:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
206 downloadfile(i, x, count)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
207 break
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
208 except HTTPError:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
209 time.sleep(10)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
210 downloadfile(i, x, count)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
211 except Exception as e:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
212 print(e)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
213 time.sleep(10)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
214
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
215
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
216 def get_amount_of_posts(s, u):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
217 amount = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
218 while 1:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
219 data = req.get("https://kemono.party/api/{0}/user/{1}?o={2}".format(s, u, amount)).json()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
220 if len(data) < 25:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
221 return math.ceil((amount + 1) / 25)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
222 amount += 25
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
223
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
224
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
225 parser = argparse.ArgumentParser(description="Downloads (deleted) videos from YTPMV creators")
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
226 parser.add_argument("-u", "--url", help="user URL", metavar='<url>', required=True)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
227 parser.add_argument("-c", "--cookies", help="", metavar='<cookies>', required=True) # required because of DDoS-GUARD
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
228 parser.add_argument("-p", "--proxy", help="proxy\n supported types: http, https, socks5 (requires pysocks)", metavar='<proxy>') # SOCKS proxy support is through PySocks - pip install pysocks
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
229 parser.add_argument("-o", "--output", help="output folder, defaults to user ID", metavar='<output>')
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
230 parser.add_argument("--test-download-services", dest="testdownloadservices", nargs="+", help="test download services\nsupported: gdrive, dropbox", metavar="<service>")
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
231 args = parser.parse_args()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
232
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
233 req = requests.Session()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
234
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
235 if args.proxy:
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
236 req.proxies = {}
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
237 if args.proxy[:6] == "socks5":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
238 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
239 httpsproxy = args.proxy
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
240 elif args.proxy[:5] == "https":
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
241 httpsproxy = args.proxy
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
242 elif args.proxy[:4] == "http":
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
243 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
244 else:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
245 print("unknown proxy format! defaulting to HTTP...")
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
246 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
247 if httpproxy:
53
ae64a0c8831b Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 51
diff changeset
248 req.proxies["http"] = httpproxy
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
249 if httpsproxy:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
250 req.proxies["https"] = httpsproxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
251
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
252 cj = http.cookiejar.MozillaCookieJar(args.cookies)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
253 cj.load(ignore_expires=True)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
254 req.cookies = cj
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
255
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
256 try:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
257 int(args.url)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
258 print("do not input user IDs in --url! use a link instead")
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
259 exit()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
260 except Exception:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
261 pass
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
262
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
263 if args.url.split("/")[-2] == "post":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
264 service = args.url.split("/")[-5]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
265 user = args.url.split("/")[-3]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
266 post = args.url.split("/")[-1]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
267 elif args.url.split("/")[-2] == "user":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
268 service = args.url.split("/")[-3]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
269 user = args.url.split("/")[-1]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
270
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
271 if not args.output:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
272 output = user
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
273 else:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
274 output = args.output
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
275
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
276 if not os.path.isdir(output):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
277 if os.path.exists(output):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
278 os.remove(output)
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
279 os.makedirs(output)
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
280
54
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
281 if args.testdownloadservices:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
282 i = {
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
283 "title": "Test"
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
284 }
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
285 if "gdrive" in args.testdownloadservices:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
286 unique_ids = ["1sMVOcUesv4Ua_KJ-eQ_CMS_5KkrZGFdF"]
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
287 drive_ids_to_download = [unique_ids[0].split("?")[0].split("/")[-1]]
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
288 while len(unique_ids) > 0:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
289 for i in unique_ids:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
290 unique_ids = get_google_drive_subfolder_ids("https://drive.google.com/drive/folders/" + i)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
291 for ids in unique_ids:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
292 drive_ids_to_download.append(ids)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
293 print(drive_ids_to_download)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
294 if "dropbox" in args.testdownloadservices:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
295 download_from_dropbox("https://www.dropbox.com/s/yg405bpznyobo3u/test.txt?dl=0") # File
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
296 download_from_dropbox("https://www.dropbox.com/sh/ne3c7bxtkt5tg4s/AABYPNGfHoil4HO_btudw0wPa?dl=0") # Folder
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
297 exit()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
298
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
299 try:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
300 post
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
301 pages = 1
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
302 except Exception:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
303 pages = get_amount_of_posts(service, user)
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
304 for page in range(pages):
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
305 try:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
306 post
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
307 userdata = req.get("https://kemono.party/api/{0}/user/{1}/post/{2}".format(service, user, post)).json()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
308 except Exception:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
309 userdata = req.get("https://kemono.party/api/{0}/user/{1}?o={2}".format(service, user, (page * 25))).json()
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
310 for i in userdata:
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
311 print(i["id"])
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
312 post = i["id"]
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
313 count = 0
5a5d47a795c6 Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 53
diff changeset
314 parse_json(i, count)