Mercurial > codedump

annotate kemonopartydownloader.py @ 52:3857b2919db0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Update README.md committer: GitHub <noreply@github.com>
author Paper <37962225+mrpapersonic@users.noreply.github.com>
date Sat, 31 Jul 2021 03:02:43 -0400
parents 6eb6c657c7eb
children ae64a0c8831b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
1 # example args.url: https://kemono.party/fanbox/user/5375435/post/2511461
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
2 import argparse
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
3 import http.cookiejar
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
4 import os
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
5 import re
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
6 import requests # pip install requests
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
7 import time
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
8 from urllib.error import HTTPError
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
9
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
10
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
11 def sanitize(filename):
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
12 return re.sub(r"[\/:*?\"<>|]", "_", filename)
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
13
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
14
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
15 def downloadfile(i, x, count):
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
16 filename = "{4}\\{0}_{1}p_{2}_{3}".format(i["id"], count, sanitize(i["title"]), os.path.basename(x["path"]), output)
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
17 if os.path.exists(filename):
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
18 filesize = os.stat(filename).st_size
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
19 else:
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
20 filesize = 0
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
21 if str(filesize) != req.head(f"https://data.kemono.party{x['path']}").headers["Content-Length"]:
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
22 print("unfinished download")
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
23 with req.get(f"https://data.kemono.party{x['path']}", stream=True, headers={"Range": f"bytes={filesize}-"}) as r:
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
24 r.raise_for_status()
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
25 with open(filename, "ab") as f:
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
26 for chunk in r.iter_content(chunk_size=4096):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
27 f.write(chunk)
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
28 print("image " + str(count) + " successfully downloaded!")
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
29 return
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
30 else:
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
31 print("image " + str(count) + " already downloaded!")
51
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
32 return
6eb6c657c7eb Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 49
diff changeset
33
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
34
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
35 parser = argparse.ArgumentParser(description="Downloads (deleted) videos from YTPMV creators")
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
36 parser.add_argument("-u", "--url", help="user URL", metavar='<url>', required=True)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
37 parser.add_argument("-c", "--cookies", help="", metavar='<url>', required=True) # required because of DDoS-GUARD
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
38 parser.add_argument("-p", "--proxy", help="proxy\n supported types: http, https, socks5 (requires pysocks)", metavar='<url>') # SOCKS proxy support is through PySocks - pip install pysocks
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
39 parser.add_argument("-o", "--output", help="output folder, defaults to user ID", metavar='<url>')
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
40 args = parser.parse_args()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
41
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
42 req = requests.Session()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
43
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
44 if args.proxy:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
45 if args.proxy[:6] == "socks5":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
46 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
47 httpsproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
48 elif args.proxy[0:5] == "https":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
49 httpsproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
50 elif args.proxy[0:4] == "http":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
51 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
52 else:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
53 print("unknown proxy format! defaulting to HTTP...")
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
54 httpproxy = args.proxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
55 if httpproxy:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
56 req.proxies = {
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
57 "http": httpproxy,
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
58 }
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
59 if httpsproxy:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
60 req.proxies["https"] = httpsproxy
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
61
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
62 cj = http.cookiejar.MozillaCookieJar(args.cookies)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
63 cj.load(ignore_expires=True)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
64 req.cookies = cj
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
65
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
66 try:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
67 int(args.url)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
68 print("do not input user IDs here! use a link instead")
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
69 exit()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
70 except Exception:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
71 pass
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
72
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
73 if args.url.split("/")[-2] == "post":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
74 service = args.url.split("/")[-5]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
75 user = args.url.split("/")[-3]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
76 post = args.url.split("/")[-1]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
77 userdata = req.get("https://kemono.party/api/{0}/user/{1}/post/{2}".format(service, user, post)).json()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
78 elif args.url.split("/")[-2] == "user":
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
79 service = args.url.split("/")[-3]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
80 user = args.url.split("/")[-1]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
81 userdata = req.get("https://kemono.party/api/{0}/user/{1}".format(service, user)).json()
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
82
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
83 if not args.output:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
84 output = user
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
85 else:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
86 output = args.output
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
87
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
88 if not os.path.isdir(output):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
89 if os.path.exists(output):
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
90 os.remove(output)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
91 os.mkdir(output)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
92
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
93 for i in userdata:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
94 print(i["id"])
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
95 post = i["id"]
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
96 count = 0
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
97 for x in i["attachments"]:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
98 count += 1
49
717feded576e Update kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents: 46
diff changeset
99 while not os.path.exists("{4}\\{0}_{1}p_{2}_{3}".format(int(i["id"]) - 1, count, sanitize(i["title"]), os.path.basename(x["path"]), output)):
46
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
100 try:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
101 downloadfile(i, x, count)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
102 break
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
103 except HTTPError:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
104 time.sleep(10)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
105 downloadfile(i, x, count)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
106 except Exception as e:
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
107 print(e)
522ad91a230e Add kemonopartydownloader.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
108 time.sleep(10)