#!/usr/bin/env node
// txtfirst: does this website answer a program?
//
// Zero dependencies, one file, Node 18+. It makes at most seven GET requests to one
// origin, identifies itself, and honours robots.txt. It never writes anything.
//
//   npx txtfirst check example.com
//   npx txtfirst check example.com --json
//   npx txtfirst check example.com --path /blog/some-post
//
// Two questions, in order. First: does the site answer a client that is not pretending
// to be a browser at all. Most do not, and nothing else matters until that is fixed.
// Second: does it meet txtfirst 0.1, which is R1 to R4 at https://txtfirst.com/spec
"use strict";

const UA = "txtfirst-cli/0.1 (+https://txtfirst.com/probe; conformance check, run by a site owner)";
const BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
const TIMEOUT = 12000;

const C = process.stdout.isTTY && !process.env.NO_COLOR
  ? { g: "\x1b[32m", r: "\x1b[31m", y: "\x1b[33m", d: "\x1b[2m", b: "\x1b[1m", x: "\x1b[0m" }
  : { g: "", r: "", y: "", d: "", b: "", x: "" };

async function get(url, { ua = UA, accept = "*/*" } = {}) {
  const ctl = new AbortController();
  const t = setTimeout(() => ctl.abort(), TIMEOUT);
  try {
    const res = await fetch(url, {
      headers: { "user-agent": ua, accept },
      redirect: "follow", signal: ctl.signal,
    });
    const body = await res.text();
    return {
      ok: true, status: res.status, body,
      bytes: Buffer.byteLength(body),
      type: (res.headers.get("content-type") || "").toLowerCase(),
      link: res.headers.get("link") || "",
      vary: (res.headers.get("vary") || "").toLowerCase(),
      url: res.url,
    };
  } catch (e) {
    return { ok: false, status: e.name === "AbortError" ? "timeout" : "error",
             body: "", bytes: 0, type: "", link: "", vary: "", error: String(e.message || e) };
  } finally { clearTimeout(t); }
}

// Text left once the markup is gone. The same stripping the study uses, so the numbers
// this prints can be compared with the ones published at txtfirst.com/measure.
function textOf(html) {
  return html
    .replace(/<(script|style|noscript|svg|template|iframe)[^>]*>[\s\S]*?<\/\1>/gi, " ")
    .replace(/<!--[\s\S]*?-->/g, " ")
    .replace(/<[^>]+>/g, " ")
    .replace(/&nbsp;/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

// Enough of RFC 9309 to honour a Disallow aimed at this client. Not a full parser.
function robotsAllows(txt, agent, path) {
  const groups = [];
  let cur = null;
  for (let line of txt.split(/\r?\n/)) {
    line = line.replace(/#.*/, "").trim();
    const m = line.match(/^([a-z-]+)\s*:\s*(.*)$/i);
    if (!m) continue;
    const k = m[1].toLowerCase(), v = m[2].trim();
    if (k === "user-agent") {
      if (!cur || cur.rules.length) { cur = { agents: [], rules: [] }; groups.push(cur); }
      cur.agents.push(v.toLowerCase());
    } else if (cur && (k === "allow" || k === "disallow")) cur.rules.push([k, v]);
  }
  const pick = groups.find(g => g.agents.some(a => a !== "*" && agent.includes(a)))
            || groups.find(g => g.agents.includes("*"));
  if (!pick) return true;
  let best = null;
  for (const [k, v] of pick.rules) {
    if (v === "") { if (k === "disallow" && best === null) best = ["allow", ""]; continue; }
    if (path.startsWith(v) && (best === null || v.length > best[1].length)) best = [k, v];
  }
  return best === null || best[0] === "allow";
}

// What robots.txt lets us do, as one decision with a name, so it can be tested without
// having to find a website willing to refuse us.
function robotsVerdict(status, body, path) {
  if (status === 401 || status === 403) return "blocked";
  if (status === 200 && !robotsAllows(body, "txtfirst-cli", path)) return "robots-disallow";
  return "proceed";
}

function normalise(input) {
  let s = String(input || "").trim().replace(/^https?:\/\//i, "").replace(/\/+$/, "");
  const slash = s.indexOf("/");
  const host = (slash === -1 ? s : s.slice(0, slash)).toLowerCase();
  const path = slash === -1 ? "" : s.slice(slash);
  if (!/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(host)) return null;
  return { host, path };
}

async function run(target, opts) {
  const t = normalise(target);
  if (!t) { console.error("Give a public hostname, like example.com or example.com/blog/post"); process.exit(2); }
  const origin = "https://" + t.host;
  const path = opts.path || t.path || "/";
  const out = { tool: "txtfirst-cli/0.1", spec: "0.1", host: t.host, path,
                checked: new Date().toISOString(), checks: [], notes: [] };
  const add = (id, level, pass, title, detail) => out.checks.push({ id, level, pass, title, detail });

  // Nothing is requested before robots.txt says it may be.
  const robots = await get(origin + "/robots.txt");
  // A 403 on robots.txt is not the owner declining, it is the origin refusing to speak
  // to a program at all. Both mean stop, and reporting them as the same thing tells a
  // site owner they made a choice they did not make. They are separated here.
  const gate = robotsVerdict(robots.status, robots.body, path);
  if (gate === "blocked") {
    out.verdict = "blocked";
    out.notes.push(`The request for /robots.txt was refused with ${robots.status}, so nothing else was requested. This is not a robots.txt preference: the origin will not answer a program at all. On Cloudflare it is usually Browser Integrity Check, which is on by default.`);
    return out;
  }
  if (gate === "robots-disallow") {
    out.verdict = "robots-disallow";
    out.notes.push("robots.txt disallows this client, so nothing was requested and nothing was measured. This is a choice the site made and it is respected.");
    return out;
  }

  // Layer 1, access. A site that refuses a plain client fails before any format matters.
  const plain = await get(origin + path);
  const asBrowser = await get(origin + path, { ua: BROWSER_UA, accept: "text/html,*/*" });
  const text = textOf(plain.body);
  const readable = plain.status === 200 && text.length >= 1500;
  add("A1", "access", plain.status === 200,
      "answers a plainly identified client",
      plain.status === 200 ? "HTTP 200"
        : `HTTP ${plain.status}` + (asBrowser.status === 200 ? ", while the same URL answers 200 to a browser string" : ""));
  add("A2", "access", readable,
      "the answer carries readable text",
      plain.status === 200
        ? `${text.length} bytes of text inside ${plain.bytes} bytes` +
          (text.length ? `, ${(plain.bytes / text.length).toFixed(1)} to 1` : "")
        : "not measured, the request was refused");
  if (plain.status !== 200 && asBrowser.status === 200)
    out.notes.push("This origin serves a browser string and refuses an honest one. On Cloudflare this is usually Browser Integrity Check, which is on by default.");

  // R2, the same URL in two shapes, from one source.
  const neg = await get(origin + path, { accept: "text/markdown" });
  const negIsMd = neg.type.includes("text/markdown");
  add("R2a", "spec", negIsMd, "Accept: text/markdown returns markdown",
      neg.status === 200 ? `HTTP 200, content-type ${neg.type || "(none)"}` : `HTTP ${neg.status}`);

  const mdPath = path === "/" ? "/index.md" : path.replace(/\/$/, "") + ".md";
  const suffix = await get(origin + mdPath);
  const suffixIsMd = suffix.status === 200 && suffix.type.includes("text/markdown");
  add("R2b", "spec", suffixIsMd, "the .md suffix returns the same source",
      `${mdPath} -> HTTP ${suffix.status}` + (suffix.type ? `, ${suffix.type}` : ""));

  const identical = negIsMd && suffixIsMd && neg.body === suffix.body && neg.body.length > 0;
  add("R2c", "spec", identical, "both routes return byte-identical bytes",
      negIsMd && suffixIsMd ? (identical ? `${neg.bytes} bytes both ways` : `${neg.bytes} vs ${suffix.bytes} bytes`)
                            : "not comparable, one of the two routes did not return markdown");

  const linkOk = /rel=("|')?alternate\1?/i.test(asBrowser.link) && /text\/markdown/i.test(asBrowser.link);
  add("R2d", "spec", linkOk, "a Link header points at the markdown",
      asBrowser.link ? asBrowser.link.slice(0, 120) : "no Link header");
  add("R2e", "spec", asBrowser.vary.includes("accept"), "Vary: Accept, so caches keep the shapes apart",
      asBrowser.vary || "no Vary header");
  const tagOk = /<link[^>]+rel=["']?alternate["']?[^>]*type=["']text\/markdown/i.test(asBrowser.body)
             || /<link[^>]+type=["']text\/markdown[^>]*rel=["']?alternate/i.test(asBrowser.body);
  add("R2f", "spec", tagOk, "the HTML declares its markdown alternate",
      tagOk ? "<link rel=alternate type=text/markdown> present" : "no alternate link element in the HTML");

  // R3, discovery. A machine that has never seen this site must be able to find the map.
  const wk = await get(origin + "/.well-known/txtfirst.json");
  let wkOk = false;
  try { wkOk = wk.status === 200 && !!JSON.parse(wk.body).txtfirst; } catch { wkOk = false; }
  add("R3a", "spec", wkOk, "/.well-known/txtfirst.json", wkOk ? "present and declares a version" : `HTTP ${wk.status}`);
  const llms = await get(origin + "/llms.txt");
  add("R3b", "spec", llms.status === 200 && llms.bytes > 50, "/llms.txt", `HTTP ${llms.status}, ${llms.bytes} bytes`);

  const spec = out.checks.filter(c => c.level === "spec");
  const access = out.checks.filter(c => c.level === "access");
  out.score = { spec_passed: spec.filter(c => c.pass).length, spec_total: spec.length,
                access_passed: access.filter(c => c.pass).length, access_total: access.length };
  out.verdict = !readable ? (plain.status === 200 ? "no content" : "blocked")
              : out.score.spec_passed === out.score.spec_total ? "conformant"
              : out.score.spec_passed === 0 ? "readable, not txtfirst" : "partial";
  return out;
}

function report(r) {
  const mark = (p) => p ? `${C.g}pass${C.x}` : `${C.r}fail${C.x}`;
  console.log(`\n${C.b}${r.host}${r.path === "/" ? "" : r.path}${C.x} ${C.d}txtfirst 0.1 conformance${C.x}\n`);
  if (r.verdict === "robots-disallow" || (r.verdict === "blocked" && !r.checks.length)) {
    for (const n of r.notes) console.log(`  ${C.y}${n}${C.x}`);
    console.log(`\n  verdict: ${C.y}${r.verdict}${C.x}\n`);
    return r.verdict === "robots-disallow" ? 3 : 1;
  }
  console.log(`  ${C.d}access, before any format matters${C.x}`);
  for (const c of r.checks.filter(c => c.level === "access"))
    console.log(`  ${mark(c.pass)}  ${c.title}\n        ${C.d}${c.detail}${C.x}`);
  console.log(`\n  ${C.d}txtfirst 0.1${C.x}`);
  for (const c of r.checks.filter(c => c.level === "spec"))
    console.log(`  ${mark(c.pass)}  ${c.id}  ${c.title}\n        ${C.d}${c.detail}${C.x}`);
  for (const n of r.notes) console.log(`\n  ${C.y}note${C.x}  ${n}`);
  const v = r.verdict === "conformant" ? `${C.g}${r.verdict}${C.x}` : `${C.y}${r.verdict}${C.x}`;
  console.log(`\n  verdict: ${v}   spec ${r.score.spec_passed}/${r.score.spec_total}   access ${r.score.access_passed}/${r.score.access_total}`);
  console.log(`  ${C.d}what each check means: https://txtfirst.com/spec${C.x}`);
  console.log(`  ${C.d}how to implement it: https://txtfirst.com/implement${C.x}\n`);
  return r.verdict === "conformant" ? 0 : 1;
}

const HELP = `txtfirst 0.1  does this website answer a program?

  curl -s https://txtfirst.com/cli/index.js > txtfirst.js

  node txtfirst.js check <host>            check the home page
  node txtfirst.js check <host> --path /p  check one page
  node txtfirst.js check <host> --json     machine readable, for CI

At most seven GET requests to one origin, identified as txtfirst-cli, after asking
robots.txt. Nothing is written and nothing is sent anywhere.

Exit codes
  0  conformant
  1  not conformant
  2  bad input
  3  not measured, because robots.txt disallows this client

The spec is public domain: https://txtfirst.com/spec`;

async function main() {
  const argv = process.argv.slice(2);
  const cmd = argv[0];
  if (!cmd || cmd === "-h" || cmd === "--help" || cmd === "help") { console.log(HELP); return 0; }
  if (cmd !== "check") { console.error(`Unknown command "${cmd}".\n\n${HELP}`); return 2; }
  const target = argv[1];
  if (!target) { console.error(`Give a hostname.\n\n${HELP}`); return 2; }
  const opts = { json: argv.includes("--json") };
  const pi = argv.indexOf("--path");
  if (pi !== -1) opts.path = argv[pi + 1];
  const r = await run(target, opts);
  // One table of exit codes for both modes. A site that refuses the checker must never
  // leave a deploy gate green: not measured is its own answer, not a pass.
  // 3 means not measured. A site that blocks the checker is measured and it failed,
  // so it is 1: only a robots.txt saying no leaves the question genuinely unanswered.
  const code = r.verdict === "robots-disallow" ? 3 : r.verdict === "conformant" ? 0 : 1;
  if (opts.json) { console.log(JSON.stringify({ ...r, exit_code: code }, null, 2)); return code; }
  report(r);
  return code;
}

main().then(c => process.exit(c)).catch(e => { console.error(String(e && e.message || e)); process.exit(2); });