Mercurial > codedump

annotate from.py @ 101:a7d2fb3751a0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Create from.py committer: GitHub <noreply@github.com>
author Paper <37962225+mrpapersonic@users.noreply.github.com>
date Sun, 28 Aug 2022 19:48:44 -0400
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
101
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
1 import math
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
2 import requests
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
3 import shutil
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
4 import os
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
5 import time
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
6 import urllib.parse
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
7 import re
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
8 ID = "ID"
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
9 CHN_ID = "CHN_ID"
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
10
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
11 def find_urls(s):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
12 urllist = []
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
13 for findall in re.findall(r"""http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+""", s):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
14 urllist.append(findall.split("<")[0].split(">")[-1])
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
15 return urllist
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
16
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
17 def download_file(url, local_filename):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
18 with requests.get(url, stream=True) as r:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
19 with open(local_filename, 'wb') as f:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
20 shutil.copyfileobj(r.raw, f)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
21 return
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
22
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
23
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
24 session = requests.Session()
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
25
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
26 session.headers = {
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
27 'authority': 'discord.com',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
28 'x-super-properties': 'eyJvcyI6IldpbmRvd3MiLCJicm93c2VyIjoiRGlzY29yZCBDbGllbnQiLCJyZWxlYXNlX2NoYW5uZWwiOiJzdGFibGUiLCJjbGllbnRfdmVyc2lvbiI6IjEuMC45MDAyIiwib3NfdmVyc2lvbiI6IjEwLjAuMTkwNDMiLCJvc19hcmNoIjoieDY0Iiwic3lzdGVtX2xvY2FsZSI6ImVuLVVTIiwiY2xpZW50X2J1aWxkX251bWJlciI6OTQyOTQsImNsaWVudF9ldmVudF9zb3VyY2UiOm51bGx9',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
29 'authorization': 'TOKEN',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
30 'x-debug-options': 'bugReporterEnabled',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
31 'accept-language': 'en-US',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
32 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) discord/1.0.9002 Chrome/83.0.4103.122 Electron/9.3.5 Safari/537.36',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
33 'accept': '*/*',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
34 'sec-fetch-site': 'same-origin',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
35 'sec-fetch-mode': 'cors',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
36 'sec-fetch-dest': 'empty',
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
37 }
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
38
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
39 response = session.get("https://discord.com/api/v9/guilds/%s/messages/search?has=link&channel_id=%s&include_nsfw=true" % (ID, CHN_ID)).json()
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
40
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
41 for done in range(math.ceil(int(response["total_results"])/25)):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
42 currentresponse = session.get("https://discord.com/api/v9/guilds/%s/messages/search?has=link&channel_id=%s&include_nsfw=true&offset=%d" % (ID, CHN_ID, done*25)).json()
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
43 for i in currentresponse["messages"]:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
44 for x in find_urls(i[0]["content"]):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
45 if urllib.parse.urlparse(x).netloc.find("tenor.com") != -1:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
46 continue
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
47 try:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
48 headresponse = session.head(x)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
49 except Exception as e:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
50 print("failed to download " + x.split("/")[-1] + " " + type(e).__name__)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
51 time.sleep(1)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
52 continue
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
53 if headresponse.headers["Content-Type"] == "video/mp4":
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
54 if os.path.exists(x.split("/")[-1]):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
55 if not os.path.getsize(x.split("/")[-1]) < int(headresponse.headers["Content-Length"]):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
56 print(x.split("/")[-1] + " already downloaded!")
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
57 continue
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
58 try:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
59 download_file(x, x.split("/")[-1])
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
60 print(x.split("/")[-1] + " downloaded!")
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
61 except Exception as e:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
62 print("failed to download " + x.split("/")[-1] + " " + type(e).__name__)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
63 continue
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
64 time.sleep(1)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
65 for x in i[0]["attachments"]:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
66 if os.path.exists(x["filename"]):
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
67 if not os.path.getsize(x["filename"]) < x["size"]:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
68 print(x["filename"] + " already downloaded!")
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
69 continue
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
70 try:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
71 download_file(x["url"], x["filename"])
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
72 print(x["filename"] + " downloaded!")
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
73 except Exception as e:
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
74 print(e)
a7d2fb3751a0 Create from.py
Paper <37962225+mrpapersonic@users.noreply.github.com>
parents:
diff changeset
75 time.sleep(1)