# HG changeset patch
# User Paper <paper@tflc.us>
# Date 1769285405 18000
# Node ID 0c3cd90e91f7a1956580171bad494c1c66d86a40
# Parent  c27afe8ead5fd760b38ec58666bc5e7710d9dec2
*: add sanae.site scraper scripts

diff -r c27afe8ead5f -r 0c3cd90e91f7 sanae.site/README
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sanae.site/README	Sat Jan 24 15:10:05 2026 -0500
@@ -0,0 +1,31 @@
+These two scripts were used to scrape files and metadata off sanae.site, a
+short-lived file-upload service ran by vhoda. He completely killed off any
+access from the sanae.site domain, but it was still accessible via
+donsebagay.mom, and that's what the ROOT variables contain.
+
+The first script "scrape.py" was used just to download all of the files.
+These were the most important to save after all. After this, I wrote
+"guess.py" which scraped all of the metadata (as the filename implies,
+this was originally going to "guess" the file extension for each file,
+but the servers were still up so I just scraped the metadata)
+
+The "guess.py" script requires the lxml package, which you will probably
+already have installed. This is only used to strip off <script> and <style>
+tags from the file.
+
+The resulting files from these scripts should be of the format:
+	"[id] - [filename w/ extension]"
+	"[id] - [filename w/ extension].json"
+
+Of which the latter is a JSON object that may or may not contain any of the
+following fields:
+	"filename"   -- original filename, HTTP 'Content-Disposition' header
+	"date"       -- the date and time of upload, ISO format
+	"visibility" -- "public" if accessible from a user's page, "unlisted"
+	                if not. private videos cannot be accessed as this
+	                script has no login details nor cookies.
+	"yturl"      -- the original YouTube URL, if this is a YouTube download
+	"username"   -- the username of the uploader; this includes the "!"
+	                prefix. this will be "Anonymous" if the website
+	                provided it as that.
+	                for some files, e.g. FLAC, this is not available :(
diff -r c27afe8ead5f -r 0c3cd90e91f7 sanae.site/guess.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sanae.site/guess.py	Sat Jan 24 15:10:05 2026 -0500
@@ -0,0 +1,116 @@
+import requests
+import re
+import os
+import time
+import datetime
+import json
+import lxml
+
+from lxml.html.clean import Cleaner
+
+ROOT = "https://donsebagay.mom/"
+
+# Global requests session
+SESSION = requests.Session()
+
+# Raw string for regex
+FILEPATTERN = re.compile(r"^[A-Z0-9\-_]{3}$")
+USERPATTERN = re.compile(r"<small class=\"text-white fw-semibold\">(.+?)</small>")
+# NOTE: This keeps the quotes in tact; you must remove them yourself with strip()
+DISPOSITIONPATTERN = re.compile(r"^attachment;?(?:\s+)filename=(.+?)$")
+DATEPATTERN = re.compile(r"<i class=\"bi bi-calendar\"></i> Uploaded: <strong>(.+?)</strong>")
+UNLISTEDPATTERN = re.compile(r"<i class=\"bi bi-eye-slash me-1\"></i>Unlisted")
+YOUTUBEPATTERN = re.compile(r"<a href=\"https://www\.youtube\.com/watch\?v=([A-Za-z0-9_\-]{11})\"")
+
+CLEANER = Cleaner()
+# Remove JS and CSS; we don't want them
+CLEANER.javascript = True
+CLEANER.style = True
+
+# Filter all of the files in the current directory
+for i in filter(lambda x: bool(re.match(FILEPATTERN, x)), os.listdir()):
+    # I should have done this before but I was lazy and I really
+    # just wanted to get everything downloaded in time.
+    #
+    # What is important now is saving all of the metadata, such as
+    # the original filename, upload date, and uploader.
+    # Sometimes the uploader is simply "Anonymous" and in this case
+    # we should probably just throw it out.
+    # The original filename and date are likely always available.
+    # The date is stored in an asctime()-like format in the "date"
+    # header. The original filename is in the content-disposition
+    # header. In case either of these two are available, we should
+    # guess the extension by the contents.
+
+    print(i)
+
+    # JSON to write; the below lines will add metadata to this
+    info = dict()
+
+    # assume public visibility
+    info["visibility"] = "public"
+
+    try:
+        page = SESSION.get(ROOT + i)
+        text = lxml.html.tostring(CLEANER.clean_html(lxml.html.document_fromstring(page.text))).decode('utf-8')
+
+        if True:
+            # basic ass regex
+            usermatch = re.search(USERPATTERN, text)
+
+            if usermatch:
+                info["username"] = usermatch.group(1)
+
+        if True:
+            # basic regex part 2
+            datematch = re.search(DATEPATTERN, text)
+
+            if datematch:
+                d = datetime.datetime.strptime(datematch.group(1), "%d/%m/%Y %H:%M")
+                info["date"] = d.isoformat()
+
+        if True:
+            unlistmatch = re.search(UNLISTEDPATTERN, text)
+            if unlistmatch:
+                info["visibility"] = "unlisted"
+
+        if True:
+            yturlmatch = re.search(YOUTUBEPATTERN, text)
+            if yturlmatch:
+                info["yturl"] = "https://www.youtube.com/watch?v=" + yturlmatch.group(1)
+
+    except Exception as e:
+        print(e)
+        pass
+
+    # Now request headers for this; it contains everything we need
+    try:
+        head = SESSION.head(ROOT + i + "?raw=1&download=1")
+
+        hdrs = head.headers
+
+        if 'content-disposition' in hdrs:
+            # Filenames are in UTF-8, but HTTP headers are supposed to be latin-1.
+            # So encode it and re-decode it as proper UTF-8
+            m = re.match(DISPOSITIONPATTERN, hdrs['content-disposition'].encode('iso-8859-1').decode('utf-8'))
+            if m:
+                info["filename"] = m.group(1).strip("\"")
+    except Exception as e:
+        print(e)
+        pass
+
+    if True:
+        if "filename" in info:
+            n = i + " - " + info["filename"]
+            try:
+                os.rename(i, n)
+                i = n
+            except:
+                pass
+
+        with open(i + ".json", "w", encoding="utf-8") as f:
+            json.dump(info, f)
+
+    # Sleep for 1 second to not overload the servers
+    # and possibly get bot-killed
+    time.sleep(2)
diff -r c27afe8ead5f -r 0c3cd90e91f7 sanae.site/scrape.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sanae.site/scrape.py	Sat Jan 24 15:10:05 2026 -0500
@@ -0,0 +1,45 @@
+import requests
+import itertools
+import time
+import random
+
+random.seed()
+
+# Root URL
+ROOT = "https://donsebagay.mom/"
+# This file allows you to start/stop the script at any time, and it
+# will pick up right where it left off.
+FILE = "donseba-status.txt"
+# String containing all possible characters in a file ID
+CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"
+
+def all_possibilities(l):
+	# maybe we need products of 2 as well...
+	yield from itertools.product(*([l] * 3))
+
+s = requests.Session()
+
+try:
+	with open(FILE, "r", encoding="utf-8") as f:
+		skip = f.read()
+except:
+	skip = None
+	pass
+
+for x in all_possibilities(CHARS):
+	x = ''.join(x)
+	if skip is not None:
+		if x == skip:
+			skip = None
+		continue
+	r = s.get(ROOT + x + "?raw=1&download=1")
+	print(x, r)
+	# For some reason, vhoda has a "Page Not Found" page
+	# that doesn't return a 404 status code, so we have to
+	# detect it through 'Content-Type' instead
+	if r.headers["Content-Type"] != "text/html; charset=UTF-8":
+		with open(x, "wb") as f:
+			f.write(r.content)
+	with open(FILE, "w") as f:
+		f.write(x)
+	time.sleep(random.random() * 5)