#!/usr/bin/env python3
# MIT licence. Copyright (c) 2026 Tuper. Written by an AI agent (Claude); use at your own risk.
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software, to deal in it without restriction, subject to the usual MIT conditions: https://opensource.org/license/mit
"""Site audit for the operation's public sites (stdlib only).

Usage:
  python3 tools/site-audit.py                 # all 7 sites, external links checked
  python3 tools/site-audit.py --no-external   # skip external links
  python3 tools/site-audit.py --site dutyroll.tuper.app [--json out.json]

Per site it reads /sitemap.xml, fetches every sitemap URL with ?nopv=1 (rule 65: no
beacon rows), parses the HTML, follows every internal link and asset once, and tries
each external link once (10 s timeout). It reports:
  - non-200 pages / links / assets
  - HTML pages reachable by link but missing from the sitemap
  - missing or duplicate <title>; missing meta description / canonical / viewport
  - h1 count != 1
  - missing pv beacon (pv.js) where the site uses it on other pages
  - http:// links/assets to our own hosts (mixed content)
  - ?src= install/invite links whose target does not answer 200/30x
  - sitemap <url> entries without <lastmod>, with a malformed one, or one in the future
  - og-image (#261, notes/seo-meta.md s5): every page has og:image, and each image answers
    200 image/png at 1200x630
  - ld-json: every page has a <script type="application/ld+json"> that parses, whose @type(s) are in
    LD_TYPES, and whose main objects (anything but BreadcrumbList/FAQPage) have url (or
    mainEntityOfPage) == the canonical; one @graph block or several blocks are both accepted
  - llms-txt: /llms.txt answers 200 text/plain and starts with '# '
  - funnel (#656): every page of a product site links to tuper.app/open with ?src=<tag>; every
    tuper.app/open* page links to a product host only with ?src=
401/403/429 from external hosts are listed as info-external-blocked (bot walls), not counted.
Exit code 1 if any finding.
"""
import argparse, html.parser, json, re, ssl, sys, urllib.error, urllib.parse, urllib.request, xml.etree.ElementTree as ET
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor

SITES = ["tuper.app", "formwell.tuper.app", "indexkeeper.tuper.app", "keepsorted.tuper.app",
         "replacekit.tuper.app", "dutyroll.tuper.app", "stockcount.tuper.app"]
PRODUCT_HOSTS = {"dutyroll.tuper.app", "formwell.tuper.app", "keepsorted.tuper.app", "replacekit.tuper.app",
                 "indexkeeper.tuper.app", "stockcount.tuper.app"}
OWN = re.compile(r"(^|\.)tuper\.(app|gg)$")
UA = "tuper-site-audit-bot/1.0 (+https://tuper.app; our own link checker)"  # "bot" so our apps count it as a bot, not a human view
CTX = ssl.create_default_context()
LD_TYPES = {"Product", "SoftwareApplication", "WebApplication", "Article", "FAQPage", "BreadcrumbList", "Organization",
            "CollectionPage", "WebPage", "Dataset"}


class NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, *a, **k):
        return None


def fetch(url, follow=True, timeout=15, method="GET"):
    """Return (status, final_url, content_type, body_text_or_None)."""
    handlers = [urllib.request.HTTPSHandler(context=CTX)]
    if not follow:
        handlers.append(NoRedirect)
    opener = urllib.request.build_opener(*handlers)
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"}, method=method)
    try:
        with opener.open(req, timeout=timeout) as r:
            ct = r.headers.get("Content-Type", "")
            body = r.read(3_000_000) if method == "GET" else b""
            text = body.decode("utf-8", "replace") if "html" in ct or "xml" in ct or ct.startswith("text/plain") else None
            return r.status, r.geturl(), ct, text
    except urllib.error.HTTPError as e:
        return e.code, url, e.headers.get("Content-Type", "") if e.headers else "", None
    except Exception as e:  # timeouts, DNS, TLS
        return 0, url, "", f"ERR {type(e).__name__}: {e}"


def nopv(url):
    p = urllib.parse.urlsplit(url)
    q = urllib.parse.parse_qsl(p.query, keep_blank_values=True)
    if not any(k == "nopv" for k, _ in q):
        q.append(("nopv", "1"))
    return urllib.parse.urlunsplit((p.scheme, p.netloc, p.path, urllib.parse.urlencode(q), ""))


def norm(url):
    """Page identity: scheme+host+path, no query/fragment, no trailing index.html."""
    p = urllib.parse.urlsplit(url)
    path = p.path or "/"
    if path.endswith("/index.html"):
        path = path[: -len("index.html")]
    return f"https://{p.netloc.lower()}{path}"


class Page(html.parser.HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.titles, self.h1, self.links, self.assets = [], 0, [], []
        self.meta, self.canonical, self.scripts = {}, None, []
        self._in_title, self._t = False, ""
        self.props, self.ld, self._in_ld = {}, [], False
        self.alts, self.lang = [], None

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "html":
            self.lang = a.get("lang")
        if tag == "title":
            self._in_title, self._t = True, ""
        elif tag == "h1":
            self.h1 += 1
        elif tag == "a" and a.get("href"):
            self.links.append(a["href"])
        elif tag == "link":
            rel = (a.get("rel") or "").lower()
            if rel == "canonical":
                self.canonical = a.get("href")
            elif rel == "alternate" and a.get("hreflang") and a.get("href"):
                self.alts.append((a["hreflang"].strip(), a["href"].strip()))
            elif a.get("href") and rel in ("stylesheet", "icon", "apple-touch-icon", "manifest", "preload", "shortcut icon"):
                self.assets.append(a["href"])
        elif tag == "meta" and a.get("name"):
            self.meta[a["name"].lower()] = a.get("content", "")
        elif tag == "meta" and a.get("property"):
            self.props[a["property"].lower()] = a.get("content", "")
        if tag == "script" and (a.get("type") or "").lower() == "application/ld+json":
            self._in_ld = True
            self.ld.append("")
        elif tag in ("script", "img", "source", "iframe", "video", "audio") and a.get("src"):
            self.assets.append(a["src"])
            if tag == "script":
                self.scripts.append(a["src"])

    def handle_endtag(self, tag):
        if tag == "script":
            self._in_ld = False
        if tag == "title" and self._in_title:
            self._in_title = False
            self.titles.append(self._t.strip())

    def handle_data(self, d):
        if self._in_title:
            self._t += d
        if self._in_ld:
            self.ld[-1] += d


def png_size(url):
    """(status, content_type, (w, h) or None) of an image URL."""
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "image/*"})
    try:
        with urllib.request.urlopen(req, timeout=15, context=CTX) as r:
            head = r.read(32)
            ct = r.headers.get("Content-Type", "")
            wh = (int.from_bytes(head[16:20], "big"), int.from_bytes(head[20:24], "big")) if head[:8] == b"\x89PNG\r\n\x1a\n" else None
            return r.status, ct, wh
    except urllib.error.HTTPError as e:
        return e.code, "", None
    except Exception:
        return 0, "", None


def check_ld(url, canonical, blocks):
    """Findings (strings) for a page's JSON-LD blocks (one @graph block or several blocks both fine)."""
    if not blocks:
        return ["no application/ld+json block"]
    out, mains = [], []
    for raw in blocks:
        try:
            data = json.loads(raw)
        except ValueError as e:
            out.append(f"does not parse: {e}")
            continue
        objs = data.get("@graph", [data]) if isinstance(data, dict) else data
        for o in objs:
            t = o.get("@type") if isinstance(o, dict) else None
            for tt in (t if isinstance(t, list) else [t]):
                if tt not in LD_TYPES:
                    out.append(f"@type {tt!r} not in the allowed set")
            if t not in ("BreadcrumbList", "FAQPage"):
                mains.append(o)
    # a page with only a BreadcrumbList (privacy, terms) is fine: notes/seo-meta.md s3 names no main type for it
    want = norm(urllib.parse.urljoin(url, canonical or url))
    for o in mains:
        me = o.get("mainEntityOfPage")
        u = o.get("url") or (me.get("@id") if isinstance(me, dict) else me) or ""
        if norm(u) != want:
            out.append(f"{o.get('@type')} url {u!r} != canonical {canonical!r}")
    return out


def audit_site(host, check_external=True):
    F = defaultdict(list)  # kind -> list of strings
    base = f"https://{host}/"
    st, _, _, sm = fetch(base + "sitemap.xml")
    sitemap = []
    if st != 200 or not sm:
        F["sitemap"].append(f"sitemap.xml -> {st}")
    else:
        try:
            root = ET.fromstring(sm.encode())
            sitemap = [e.text.strip() for e in root.iter() if e.tag.endswith("loc") and e.text]
            import datetime
            today = datetime.datetime.now(datetime.timezone.utc).date().isoformat()
            for u in root.iter():
                if not u.tag.endswith("}url") and u.tag != "url":
                    continue
                loc = next((c.text or "").strip() for c in u if c.tag.endswith("loc"))
                lm = next(((c.text or "").strip() for c in u if c.tag.endswith("lastmod")), None)
                if lm is None:
                    F["sitemap-lastmod"].append(f"{loc}: no <lastmod>")
                elif not re.fullmatch(r"\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2}))?", lm):
                    F["sitemap-lastmod"].append(f"{loc}: malformed lastmod {lm!r}")
                elif lm[:10] > today:
                    F["sitemap-lastmod"].append(f"{loc}: lastmod {lm} is in the future")
        except ET.ParseError as e:
            F["sitemap"].append(f"sitemap.xml parse error {e}")
    sm_norm = {norm(u) for u in sitemap}
    pages = {}  # norm -> parsed Page
    linked_from = defaultdict(set)
    queue = list(dict.fromkeys(sitemap or [base]))
    seen_pages = set()
    internal_other, external, src_links = defaultdict(set), defaultdict(set), defaultdict(set)

    while queue:
        url = queue.pop(0)
        n = norm(url)
        if n in seen_pages:
            continue
        seen_pages.add(n)
        st, final, ct, body = fetch(nopv(url))
        if st != 200:
            F["page-non-200"].append(f"{url} -> {st} (linked from: {', '.join(sorted(linked_from[n])[:3]) or 'sitemap'})")
            continue
        if "html" not in ct or body is None:
            continue
        fn = norm(final)
        if urllib.parse.urlsplit(final).netloc.lower() != host:
            continue
        if fn != n:  # internal link that redirects (e.g. /privacy -> /privacy/)
            F["link-redirects"].append(f"{url} -> {final} (linked from: {', '.join(sorted(linked_from[n])[:3]) or 'sitemap'})")
            linked_from[fn] |= linked_from[n]
            if fn in seen_pages:
                continue
            seen_pages.add(fn)
            url, n = final.split("?")[0], fn
        p = Page()
        try:
            p.feed(body)
        except Exception as e:
            F["parse"].append(f"{url}: {e}")
            continue
        pages[n] = (url, p, body)
        for href in p.links + p.assets:
            if href.startswith(("mailto:", "tel:", "javascript:", "data:", "#")):
                continue
            absu = urllib.parse.urljoin(final, href)
            sp = urllib.parse.urlsplit(absu)
            h = (sp.hostname or "").lower()
            if sp.scheme == "http" and OWN.search(h):
                F["http-own-host"].append(f"{url}: {href}")
            if sp.scheme not in ("http", "https"):
                continue
            if "src=" in sp.query and re.search(r"(install|invite|oauth2/authorize|/slack/|/auth/)", absu):
                src_links[absu].add(url)
            if h == host:
                path = sp.path or "/"
                looks_html = path.endswith("/") or path.endswith(".html") or "." not in path.rsplit("/", 1)[-1]
                if href in p.links and looks_html and not re.search(r"/(slack|auth|api|install|health|pv|unsubscribe|confirm)\b", path):
                    linked_from[norm(absu)].add(url)
                    if norm(absu) not in seen_pages:
                        queue.append(urllib.parse.urlunsplit((sp.scheme, sp.netloc, sp.path, "", "")))
                else:
                    internal_other[urllib.parse.urlunsplit((sp.scheme, sp.netloc, sp.path, sp.query, ""))].add(url)
            elif OWN.search(h):
                internal_other[urllib.parse.urlunsplit(("https", sp.netloc, sp.path, sp.query, ""))].add(url)
            else:
                external[urllib.parse.urlunsplit((sp.scheme, sp.netloc, sp.path, sp.query, ""))].add(url)

    # pages reachable by link but not in sitemap
    for n, (url, p, body) in pages.items():
        if sitemap and n not in sm_norm:
            F["not-in-sitemap"].append(f"{url} (linked from {', '.join(sorted(linked_from[n])[:3])})")

    # per-page HTML checks
    titles = defaultdict(list)
    beacon_pages = [n for n, (_, p, b) in pages.items() if "pv.js" in b]
    for n, (url, p, body) in sorted(pages.items()):
        if len(p.titles) != 1 or not p.titles[0]:
            F["title"].append(f"{url}: {len(p.titles)} <title>")
        else:
            titles[p.titles[0]].append(url)
        if not p.meta.get("description"):
            F["meta-description"].append(url)
        if not p.meta.get("viewport"):
            F["viewport"].append(url)
        if not p.canonical:
            F["canonical"].append(url)
        elif norm(urllib.parse.urljoin(url, p.canonical)) != n:
            F["canonical-mismatch"].append(f"{url}: canonical {p.canonical}")
        if p.h1 != 1:
            F["h1"].append(f"{url}: {p.h1} h1")
        if beacon_pages and "pv.js" not in body:
            F["no-beacon"].append(url)
    # funnel both ways (#656): product page -> /open with ?src=, /open page -> product host with ?src=
    for n, (url, p, body) in sorted(pages.items()):
        if host == "tuper.app":
            if urllib.parse.urlsplit(url).path.startswith("/open"):
                for href in p.links:
                    sp = urllib.parse.urlsplit(urllib.parse.urljoin(url, href))
                    if (sp.hostname or "").lower() in PRODUCT_HOSTS and "src=" not in sp.query:
                        F["funnel"].append(f"{url}: link to {href} has no ?src=")
        else:
            ok = False
            for href in p.links:
                sp = urllib.parse.urlsplit(urllib.parse.urljoin(url, href))
                if (sp.hostname or "").lower() == "tuper.app" and (sp.path.rstrip("/") == "/open") \
                        and re.search(r"(^|&)src=[\w.-]+", sp.query):
                    ok = True
                    break
            if not ok:
                F["funnel"].append(f"{url}: no link to tuper.app/open with ?src=")
    # hreflang (#454): targets 200, reciprocal, x-default or en, in sitemap, html lang
    alt_pages = {n: (u, p) for n, (u, p, b) in pages.items() if p.alts}
    alt_cache = {}

    def alt_set(target):
        tn = norm(target)
        if tn in alt_cache:
            return alt_cache[tn]
        if tn in pages:
            res = (200, {norm(urllib.parse.urljoin(target, h)) for _, h in pages[tn][1].alts})
        else:
            st_, _, ct_, b_ = fetch(nopv(target))
            pp = Page()
            if st_ == 200 and b_ and "html" in ct_:
                try:
                    pp.feed(b_)
                except Exception:
                    pass
            res = (st_, {norm(urllib.parse.urljoin(target, h)) for _, h in pp.alts})
        alt_cache[tn] = res
        return res

    for n, (url, p) in sorted(alt_pages.items()):
        codes = {c.lower(): urllib.parse.urljoin(url, h) for c, h in p.alts}
        if "x-default" not in codes and "en" not in codes:
            F["hreflang"].append(f"{url}: no x-default and no en alternate")
        if n not in {norm(urllib.parse.urljoin(url, h)) for _, h in p.alts}:
            F["hreflang"].append(f"{url}: no self-referencing hreflang")
        own = (p.lang or "").lower()
        if not own:
            F["hreflang"].append(f"{url}: <html> has no lang=")
        else:
            for c, h in p.alts:
                if norm(urllib.parse.urljoin(url, h)) == n and c.lower() not in ("x-default", own) \
                        and c.lower().split("-")[0] != own.split("-")[0]:
                    F["hreflang"].append(f"{url}: html lang={p.lang} but hreflang={c} points at itself")
        for c, h in p.alts:
            t = urllib.parse.urljoin(url, h)
            tn = norm(t)
            if urllib.parse.urlsplit(t).netloc.lower() != host:
                F["hreflang"].append(f"{url}: hreflang={c} target {t} is on another host")
                continue
            st_, back = alt_set(t)
            if st_ != 200:
                F["hreflang"].append(f"{url}: hreflang={c} target {t} -> {st_}")
                continue
            if tn != n and n not in back:
                F["hreflang"].append(f"{url}: hreflang={c} target {t} does not link back")
            if sitemap and tn not in sm_norm:
                F["hreflang"].append(f"{url}: hreflang={c} target {t} not in sitemap.xml")
            if tn in pages and not pages[tn][1].lang:
                F["hreflang"].append(f"{t}: <html> has no lang=")
    if F.get("hreflang"):
        F["hreflang"] = list(dict.fromkeys(F["hreflang"]))
    # share previews, structured data, llms.txt (#261)
    images = defaultdict(list)
    for n, (url, p, body) in sorted(pages.items()):
        img = p.props.get("og:image")
        if not img:
            F["og-image"].append(f"{url}: no og:image")
        else:
            images[urllib.parse.urljoin(url, img)].append(url)
        for msg in check_ld(url, p.canonical, p.ld):
            F["ld-json"].append(f"{url}: {msg}")
    for img, on in sorted(images.items()):
        st, ct, wh = png_size(img)
        if st != 200 or not ct.startswith("image/png") or wh != (1200, 630):
            F["og-image"].append(f"{img} -> {st} {ct or '-'} {wh} (want 200 image/png 1200x630; on {', '.join(on[:2])})")
    st, _, ct, body = fetch(base + "llms.txt")
    if st != 200 or not ct.startswith("text/plain") or not (body or "").startswith("# "):
        F["llms-txt"].append(f"{base}llms.txt -> {st} {ct or '-'}" + ("" if st != 200 else " (want text/plain starting with '# ')"))
    for t, us in titles.items():
        if len(us) > 1:
            F["duplicate-title"].append(f"'{t}': {', '.join(us)}")

    # assets and other internal targets, external links, ?src= links
    def chk(u, follow=True, timeout=15):
        st, *_ = fetch(nopv(u) if OWN.search(urllib.parse.urlsplit(u).hostname or "") else u, follow=follow, timeout=timeout)
        return u, st

    with ThreadPoolExecutor(8) as ex:
        for u, st in ex.map(lambda u: chk(u), sorted(internal_other)):
            if st != 200:
                F["internal-non-200"].append(f"{u} -> {st} (on {', '.join(sorted(internal_other[u])[:2])})")
        for u, st in ex.map(lambda u: chk(u, follow=False), sorted(src_links)):
            if st not in (200, 301, 302, 303, 307, 308):
                F["src-link"].append(f"{u} -> {st} (on {', '.join(sorted(src_links[u])[:2])})")
        if check_external:
            for u, st in ex.map(lambda u: chk(u, timeout=10), sorted(external)):
                if st in (401, 403, 429):  # bot walls (help.shopify.com, Cloudflare): not proof of a dead link
                    F["info-external-blocked"].append(f"{u} -> {st} (on {', '.join(sorted(external[u])[:2])})")
                elif st != 200:
                    F["external-non-200"].append(f"{u} -> {st} (on {', '.join(sorted(external[u])[:2])})")
    stats = {"sitemap": len(sitemap), "pages": len(pages), "internal_other": len(internal_other),
             "external": len(external), "src_links": len(src_links)}
    return stats, dict(F)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--site", action="append")
    ap.add_argument("--no-external", action="store_true")
    ap.add_argument("--json")
    a = ap.parse_args()
    out, total = {}, 0
    for host in a.site or SITES:
        stats, F = audit_site(host, not a.no_external)
        out[host] = {"stats": stats, "findings": F}
        n = sum(len(v) for k, v in F.items() if not k.startswith("info-"))
        total += n
        print(f"\n== {host}  {stats}  findings: {n}")
        for k, v in sorted(F.items()):
            for line in v:
                print(f"  [{k}] {line}")
    if a.json:
        json.dump(out, open(a.json, "w"), indent=1)
    print(f"\nTOTAL findings: {total}")
    sys.exit(1 if total else 0)


if __name__ == "__main__":
    main()