diff sanae.site/guess.py @ 135:0c3cd90e91f7 default tip

*: add sanae.site scraper scripts
author Paper <paper@tflc.us>
date Sat, 24 Jan 2026 15:10:05 -0500
parents
children
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sanae.site/guess.py	Sat Jan 24 15:10:05 2026 -0500
@@ -0,0 +1,116 @@
+import requests
+import re
+import os
+import time
+import datetime
+import json
+import lxml
+
+from lxml.html.clean import Cleaner
+
+ROOT = "https://donsebagay.mom/"
+
+# Global requests session
+SESSION = requests.Session()
+
+# Raw string for regex
+FILEPATTERN = re.compile(r"^[A-Z0-9\-_]{3}$")
+USERPATTERN = re.compile(r"<small class=\"text-white fw-semibold\">(.+?)</small>")
+# NOTE: This keeps the quotes in tact; you must remove them yourself with strip()
+DISPOSITIONPATTERN = re.compile(r"^attachment;?(?:\s+)filename=(.+?)$")
+DATEPATTERN = re.compile(r"<i class=\"bi bi-calendar\"></i> Uploaded: <strong>(.+?)</strong>")
+UNLISTEDPATTERN = re.compile(r"<i class=\"bi bi-eye-slash me-1\"></i>Unlisted")
+YOUTUBEPATTERN = re.compile(r"<a href=\"https://www\.youtube\.com/watch\?v=([A-Za-z0-9_\-]{11})\"")
+
+CLEANER = Cleaner()
+# Remove JS and CSS; we don't want them
+CLEANER.javascript = True
+CLEANER.style = True
+
+# Filter all of the files in the current directory
+for i in filter(lambda x: bool(re.match(FILEPATTERN, x)), os.listdir()):
+    # I should have done this before but I was lazy and I really
+    # just wanted to get everything downloaded in time.
+    #
+    # What is important now is saving all of the metadata, such as
+    # the original filename, upload date, and uploader.
+    # Sometimes the uploader is simply "Anonymous" and in this case
+    # we should probably just throw it out.
+    # The original filename and date are likely always available.
+    # The date is stored in an asctime()-like format in the "date"
+    # header. The original filename is in the content-disposition
+    # header. In case either of these two are available, we should
+    # guess the extension by the contents.
+
+    print(i)
+
+    # JSON to write; the below lines will add metadata to this
+    info = dict()
+
+    # assume public visibility
+    info["visibility"] = "public"
+
+    try:
+        page = SESSION.get(ROOT + i)
+        text = lxml.html.tostring(CLEANER.clean_html(lxml.html.document_fromstring(page.text))).decode('utf-8')
+
+        if True:
+            # basic ass regex
+            usermatch = re.search(USERPATTERN, text)
+
+            if usermatch:
+                info["username"] = usermatch.group(1)
+
+        if True:
+            # basic regex part 2
+            datematch = re.search(DATEPATTERN, text)
+
+            if datematch:
+                d = datetime.datetime.strptime(datematch.group(1), "%d/%m/%Y %H:%M")
+                info["date"] = d.isoformat()
+
+        if True:
+            unlistmatch = re.search(UNLISTEDPATTERN, text)
+            if unlistmatch:
+                info["visibility"] = "unlisted"
+
+        if True:
+            yturlmatch = re.search(YOUTUBEPATTERN, text)
+            if yturlmatch:
+                info["yturl"] = "https://www.youtube.com/watch?v=" + yturlmatch.group(1)
+
+    except Exception as e:
+        print(e)
+        pass
+
+    # Now request headers for this; it contains everything we need
+    try:
+        head = SESSION.head(ROOT + i + "?raw=1&download=1")
+
+        hdrs = head.headers
+
+        if 'content-disposition' in hdrs:
+            # Filenames are in UTF-8, but HTTP headers are supposed to be latin-1.
+            # So encode it and re-decode it as proper UTF-8
+            m = re.match(DISPOSITIONPATTERN, hdrs['content-disposition'].encode('iso-8859-1').decode('utf-8'))
+            if m:
+                info["filename"] = m.group(1).strip("\"")
+    except Exception as e:
+        print(e)
+        pass
+
+    if True:
+        if "filename" in info:
+            n = i + " - " + info["filename"]
+            try:
+                os.rename(i, n)
+                i = n
+            except:
+                pass
+
+        with open(i + ".json", "w", encoding="utf-8") as f:
+            json.dump(info, f)
+
+    # Sleep for 1 second to not overload the servers
+    # and possibly get bot-killed
+    time.sleep(2)