import re, json, gzip, datetime, urllib.request, urllib.error, urllib.robotparser
from concurrent.futures import ThreadPoolExecutor

SITES = ["https://www.marriott.com/","https://www.hilton.com/en/","https://www.booking.com/","https://www.melia.com/en/home",
"https://www.accor.com/en","https://www.ihg.com/","https://www.expedia.com/","https://www.nh-hotels.com/","https://www.hyatt.com/",
"https://www.radissonhotels.com/en-us/","https://www.wyndhamhotels.com/","https://www.choicehotels.com/","https://www.barcelo.com/en-us/",
"https://www.riu.com/en/","https://www.iberostar.com/en/","https://www.airbnb.com/","https://www.hotels.com/","https://www.agoda.com/",
"https://www.tripadvisor.com/","https://www.kayak.com/","https://www.trivago.com/","https://www.vrbo.com/",
"https://www.fourseasons.com/","https://www.shangri-la.com/","https://www.mandarinoriental.com/"]

UAS = {
 "browser":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
 "agent":"TxtFirstProbe/0.1 (+https://txtfirst.com/probe; research on agent readability)",
}

def strip(html):
    h = re.sub(r'(?is)<(script|style|noscript|svg|template|iframe)[^>]*>.*?</\1>',' ',html)
    h = re.sub(r'(?s)<!--.*?-->',' ',h); h = re.sub(r'(?s)<[^>]+>',' ',h)
    h = h.replace('&nbsp;',' ')
    return re.sub(r'\s+',' ',h).strip()

def allowed(url, ua):
    """Ask robots.txt first. A probe that argues for honest identification does not
    get to skip this. Failing to fetch robots.txt is treated as allowed, which is the
    conventional reading, and is recorded so the result is not silent."""
    p = urllib.robotparser.RobotFileParser()
    root = "/".join(url.split("/")[:3])
    p.set_url(root + "/robots.txt")
    try:
        p.read()
    except Exception:
        return True, "robots-unreachable"
    return p.can_fetch(ua, url), "robots-ok"


def probe(args):
    url, label = args
    # Always ask under the honest name and apply the answer to BOTH requests.
    # RobotFileParser matches the token before the first slash, so passing the
    # disguised Chrome string would consult rules for "Mozilla" and let a
    # "User-agent: TxtFirstProbe / Disallow: /" through on the disguised half.
    ok, note = allowed(url, "TxtFirstProbe")
    if not ok:
        return dict(url=url, ua=label, code="robots-disallow", html_kb=None,
                    text_kb=None, ratio=None, blocked=True, note=note)
    try:
        req = urllib.request.Request(url, headers={"User-Agent":UAS[label],"Accept":"text/html,*/*","Accept-Encoding":"gzip"})
        r = urllib.request.urlopen(req, timeout=25); raw = r.read()
        if r.headers.get("Content-Encoding")=="gzip": raw = gzip.decompress(raw)
        html = raw.decode("utf-8","replace"); t = strip(html)
        hb, tb = len(html.encode()), len(t.encode())
        return dict(url=url, ua=label, code=r.status, html_kb=round(hb/1024,1), text_kb=round(tb/1024,1),
                    ratio=round(hb/max(tb,1),1), blocked=tb<1500, note=note)
    except urllib.error.HTTPError as e:
        return dict(url=url, ua=label, code=e.code, html_kb=None, text_kb=None, ratio=None, blocked=True, note=note)
    except Exception as e:
        return dict(url=url, ua=label, code=type(e).__name__, html_kb=None, text_kb=None, ratio=None, blocked=True, note=note)

jobs=[(u,l) for u in SITES for l in UAS]
with ThreadPoolExecutor(12) as ex: res=list(ex.map(probe, jobs))
OUT = "probe-%s.json" % datetime.date.today().isoformat()
json.dump(res, open(OUT, "w"), indent=1)
print("wrote", OUT, "\n")

for l in UAS:
    sub=[r for r in res if r["ua"]==l]
    ok=[r for r in sub if not r["blocked"]]
    print(f"[{l:7}] leibles {len(ok):2}/{len(sub)}   bloqueados/vacios {len(sub)-len(ok)}")
    if ok:
        import statistics as _st
        m=lambda k: _st.median([r[k] for r in ok])
        print(f"          median html {m('html_kb'):.1f} KB | text {m('text_kb'):.1f} KB | ratio {m('ratio'):.1f}x")
        print("          (three independent order statistics, not one page)")
print()
byua = {l: {r["url"]: r for r in res if r["ua"] == l} for l in UAS}
flips = [u for u in SITES if byua["browser"][u]["blocked"] != byua["agent"][u]["blocked"]]
print("sites that flip between user agents:", len(flips))
for u in flips:
    print("  ", u, "browser", byua["browser"][u]["code"], "-> agent", byua["agent"][u]["code"])
print("\nn = 1 request per site per user agent. No repeats. Treat small differences as noise.")