Website to LLM text: a site into clean pages with Python

import re, urllib.request, urllib.robotparser
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse

UA = "site-to-text/1.0"
SKIP = {"script", "style", "noscript", "svg", "nav", "header", "footer", "aside", "form", "template"}
BLOCK = {"p", "li", "h1", "h2", "h3", "h4", "pre", "tr", "blockquote", "div", "section", "article", "br"}

def fetch(url):
    with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=30) as r:
        return r.read().decode(r.headers.get_content_charset() or "utf-8", "replace")

class Page(HTMLParser):
    """Title, first h1 and the text of <main> (or <body>), without menus, footers and scripts."""
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.title, self.h1, self.text, self.main_text = "", "", [], []
        self.stack, self.skip, self.in_main = [], 0, 0
    def handle_starttag(self, tag, attrs):
        if tag in ("br", "img", "hr", "meta", "link", "input"):
            return
        self.stack.append(tag)
        self.skip += tag in SKIP
        self.in_main += tag == "main"
        if tag in BLOCK:
            self.text.append("\n")
            if self.in_main: self.main_text.append("\n")
    def handle_endtag(self, tag):
        while self.stack:
            t = self.stack.pop()
            self.skip -= t in SKIP
            self.in_main -= t == "main"
            if t == tag:
                break
    def handle_data(self, data):
        here = self.stack[-1] if self.stack else ""
        if here == "title": self.title += data
        if self.skip: return
        if here == "h1": self.h1 += data
        self.text.append(data)
        if self.in_main: self.main_text.append(data)

def clean(parts):
    lines = (" ".join(l.split()) for l in "".join(parts).split("\n"))
    return "\n".join(l for l in lines if l)

def page_row(url):
    p = Page(); p.feed(fetch(url))
    text = clean(p.main_text) or clean(p.text)
    return {"url": url, "title": p.title.strip(), "h1": " ".join(p.h1.split()), "text": text,
            "words": len(re.findall(r"\w+", text))}

def sitemap_urls(site):
    robots = urllib.robotparser.RobotFileParser(urljoin(site, "/robots.txt")); robots.read()
    maps = robots.site_maps() or [urljoin(site, "/sitemap.xml")]
    urls = []
    while maps:
        xml = fetch(maps.pop())
        locs = re.findall(r"<loc>\s*([^<\s]+)\s*</loc>", xml)
        if "<sitemapindex" in xml: maps += locs          # an index of sitemaps: read each one
        else: urls += locs
    host = urlparse(site).netloc
    return [u for u in dict.fromkeys(urls) if urlparse(u).netloc == host and robots.can_fetch(UA, u)]

def chunks(row, size=300):
    """Pieces of about `size` words, cut between lines, each carrying its page's address and title."""
    out, cur = [], []
    for line in row["text"].split("\n"):
        if cur and len(" ".join(cur + [line]).split()) > size:
            out.append(cur); cur = []
        cur.append(line)
    if cur: out.append(cur)
    return [{"url": row["url"], "title": row["title"], "text": "\n".join(c)} for c in out]

urls = sitemap_urls("https://voiddo.com/")
rows = []
for u in urls:
    rows.append(page_row(u))
    time.sleep(1)                                    # one page a second is plenty for a small site

{
  "url": "https://voiddo.com/blog/job-board-apis-greenhouse-lever-ashby-workday",
  "title": "Greenhouse, Lever, Ashby and Workday job board APIs · vøiddo",
  "h1": "Greenhouse, Lever, Ashby and Workday job board APIs",
  "text": "← blog / scrapers\nGreenhouse, Lever, Ashby and Workday job board APIs\n7 min read · scrapers\nFour hiring systems publish their open jobs as JSON ...",
  "words": 1665
}