Mercurial > codedump

comparison from.py @ 101:a7d2fb3751a0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Create from.py committer: GitHub <noreply@github.com>
author Paper <37962225+mrpapersonic@users.noreply.github.com>
date Sun, 28 Aug 2022 19:48:44 -0400
parents
children
comparison
equal deleted inserted replaced
100:b14e2a096ebf 101:a7d2fb3751a0
1 import math
2 import requests
3 import shutil
4 import os
5 import time
6 import urllib.parse
7 import re
8 ID = "ID"
9 CHN_ID = "CHN_ID"
10
11 def find_urls(s):
12 urllist = []
13 for findall in re.findall(r"""http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+""", s):
14 urllist.append(findall.split("<")[0].split(">")[-1])
15 return urllist
16
17 def download_file(url, local_filename):
18 with requests.get(url, stream=True) as r:
19 with open(local_filename, 'wb') as f:
20 shutil.copyfileobj(r.raw, f)
21 return
22
23
24 session = requests.Session()
25
26 session.headers = {
27 'authority': 'discord.com',
28 'x-super-properties': 'eyJvcyI6IldpbmRvd3MiLCJicm93c2VyIjoiRGlzY29yZCBDbGllbnQiLCJyZWxlYXNlX2NoYW5uZWwiOiJzdGFibGUiLCJjbGllbnRfdmVyc2lvbiI6IjEuMC45MDAyIiwib3NfdmVyc2lvbiI6IjEwLjAuMTkwNDMiLCJvc19hcmNoIjoieDY0Iiwic3lzdGVtX2xvY2FsZSI6ImVuLVVTIiwiY2xpZW50X2J1aWxkX251bWJlciI6OTQyOTQsImNsaWVudF9ldmVudF9zb3VyY2UiOm51bGx9',
29 'authorization': 'TOKEN',
30 'x-debug-options': 'bugReporterEnabled',
31 'accept-language': 'en-US',
32 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) discord/1.0.9002 Chrome/83.0.4103.122 Electron/9.3.5 Safari/537.36',
33 'accept': '*/*',
34 'sec-fetch-site': 'same-origin',
35 'sec-fetch-mode': 'cors',
36 'sec-fetch-dest': 'empty',
37 }
38
39 response = session.get("https://discord.com/api/v9/guilds/%s/messages/search?has=link&channel_id=%s&include_nsfw=true" % (ID, CHN_ID)).json()
40
41 for done in range(math.ceil(int(response["total_results"])/25)):
42 currentresponse = session.get("https://discord.com/api/v9/guilds/%s/messages/search?has=link&channel_id=%s&include_nsfw=true&offset=%d" % (ID, CHN_ID, done*25)).json()
43 for i in currentresponse["messages"]:
44 for x in find_urls(i[0]["content"]):
45 if urllib.parse.urlparse(x).netloc.find("tenor.com") != -1:
46 continue
47 try:
48 headresponse = session.head(x)
49 except Exception as e:
50 print("failed to download " + x.split("/")[-1] + " " + type(e).__name__)
51 time.sleep(1)
52 continue
53 if headresponse.headers["Content-Type"] == "video/mp4":
54 if os.path.exists(x.split("/")[-1]):
55 if not os.path.getsize(x.split("/")[-1]) < int(headresponse.headers["Content-Length"]):
56 print(x.split("/")[-1] + " already downloaded!")
57 continue
58 try:
59 download_file(x, x.split("/")[-1])
60 print(x.split("/")[-1] + " downloaded!")
61 except Exception as e:
62 print("failed to download " + x.split("/")[-1] + " " + type(e).__name__)
63 continue
64 time.sleep(1)
65 for x in i[0]["attachments"]:
66 if os.path.exists(x["filename"]):
67 if not os.path.getsize(x["filename"]) < x["size"]:
68 print(x["filename"] + " already downloaded!")
69 continue
70 try:
71 download_file(x["url"], x["filename"])
72 print(x["filename"] + " downloaded!")
73 except Exception as e:
74 print(e)
75 time.sleep(1)