learning-website-nextjs1-11  Exercise 1: Measure the Old Site Instead of Remembering It
=========================================================================================

The old site is live and has bookmarks, search results and links from other places. A migration that just switches it off turns every one of
those into a 404. So first, find out exactly what the old site answers. It is a folder of built files (/hungary/x/y/ is the file
hungary/x/y/index.html), and that folder is the complete, factual list of old addresses.

Save as packages/migration/src/oldsite.ts:

import { readdirSync } from "node:fs";
import { join } from "node:path";

import { NoSiteError, siteForPath, type SiteName } from "@lw/sites";

/**
 * Read the OLD site (the folder that is served today) and list every address it answers.
 *
 * The old site is a folder of built files: /hungary/x/y/ is the file hungary/x/y/index.html, and a PDF or a solution file is served at its
 * own path. That folder is the complete, factual list of old addresses, so the migration is checked against it instead of against anyone's
 * memory of what existed.
 */

export type OldKind = "page" | "asset" | "dynamic" | "static";

export interface OldUrl {
  /** No leading or trailing slash: "hungary/x/y" for a page, "hungary/x/pdfs/a.pdf" for a file. */
  readonly path: string;
  readonly kind: OldKind;
  /** The site that owns this address now, or null when no site does. */
  readonly site: SiteName | null;
}

const STATIC_TOP = new Set(["_astro"]);                 // the old build's own css and scripts: not content
const ASSET_EXTENSIONS = [".pdf", ".txt"];

function owner(path: string): SiteName | null {
  try {
    return siteForPath(path);
  } catch (error) {
    if (error instanceof NoSiteError) return null;
    throw error;
  }
}

export function scanOldSite(root: string): OldUrl[] {
  const found: OldUrl[] = [];
  const walk = (relDir: string) => {
    if (STATIC_TOP.has(relDir.split("/")[0] ?? "")) return;
    for (const entry of readdirSync(join(root, relDir), { withFileTypes: true })) {
      const rel = relDir ? `${relDir}/${entry.name}` : entry.name;
      if (entry.isDirectory()) { walk(rel); continue; }
      const lower = entry.name.toLowerCase();
      if (lower === "index.html") found.push({ path: relDir, kind: "page", site: relDir ? owner(relDir) : null });
      else if (ASSET_EXTENSIONS.some((ext) => lower.endsWith(ext))) found.push({ path: rel, kind: "asset", site: owner(rel) });
      else if (lower.endsWith(".php")) found.push({ path: rel, kind: "dynamic", site: null });
      else found.push({ path: rel, kind: "static", site: null });         // other files, including a bare .html next to index pages
    }
  };
  walk("");
  return found.sort((a, b) => (a.kind === b.kind ? (a.path < b.path ? -1 : 1) : a.kind < b.kind ? -1 : 1));
}

Save as packages/migration/src/verify.ts:

import type { Page } from "@lw/content";
import { NavIndex } from "@lw/navigation";
import { SITE_NAMES, type SiteName } from "@lw/sites";

import type { OldUrl } from "./oldsite.ts";
import { bareAddress, type Redirect } from "./redirects.ts";

export type Outcome = "same" | "redirected" | "missing";

/**
 * Answers "does the new site have this old address?" without starting a server, from what the new sites are made of: their pages, the
 * folders those pages sit in (each has a listing), the PDFs and solution files that will be copied, and the decided redirects.
 */
export class Resolver {
  private readonly navs = new Map<SiteName, NavIndex>();
  private readonly pageSet = new Set<string>();
  private readonly redirects = new Map<string, Redirect>();
  private readonly assets: ReadonlySet<string>;

  // (a field declared in the constructor's parameter list is not allowed when Node runs the file as it is: Chapter 6)
  constructor(pages: readonly Page[], assets: ReadonlySet<string>, redirects: readonly Redirect[]) {
    this.assets = assets;
    for (const site of SITE_NAMES) this.navs.set(site, new NavIndex(site, pages));
    for (const page of pages) this.pageSet.add(`${page.site}:${page.path.slice(0, -".html".length)}`);
    for (const redirect of redirects) this.redirects.set(`${redirect.site}:${bareAddress(redirect.oldPath)}`, redirect);
  }

  /** Is there a page, or a folder listing, at this address of this site? */
  exists(site: SiteName, bare: string): boolean {
    return this.pageSet.has(`${site}:${bare}`) || (this.navs.get(site)?.hasFolder(bare) ?? false);
  }

  /** Where a redirect for this old address goes, if there is one. */
  redirectFor(site: SiteName, bare: string): Redirect | undefined {
    return this.redirects.get(`${site}:${bare}`);
  }

  check(url: OldUrl): Outcome | null {
    if (url.site === null || (url.kind !== "page" && url.kind !== "asset")) return null;      // not something a site now owns
    if (url.kind === "asset") return this.assets.has(url.path) ? "same" : "missing";
    if (this.exists(url.site, url.path)) return "same";
    const redirect = this.redirectFor(url.site, url.path);
    // a redirect to a page that does not exist is NOT success
    return redirect && this.exists(redirect.newSite, redirect.newPath) ? "redirected" : "missing";
  }
}

export interface Report {
  readonly pages: Record<Outcome, number>;
  readonly assets: Record<Outcome, number>;
  readonly unowned: number;
  readonly missing: OldUrl[];
}

export function verifyAll(urls: readonly OldUrl[], resolver: Resolver): Report {
  const pages: Record<Outcome, number> = { same: 0, redirected: 0, missing: 0 };
  const assets: Record<Outcome, number> = { same: 0, redirected: 0, missing: 0 };
  const missing: OldUrl[] = [];
  let unowned = 0;
  for (const url of urls) {
    if ((url.kind === "page" || url.kind === "asset") && url.site === null) unowned++;
    const outcome = resolver.check(url);
    if (outcome === null) continue;
    (url.kind === "page" ? pages : assets)[outcome]++;
    if (outcome === "missing") missing.push(url);
  }
  return { pages, assets, unowned, missing };
}

Save as packages/migration/package.json:

{
  "name": "@lw/migration",
  "version": "0.1.0",
  "private": true,
  "type": "module",
  "exports": {
    ".": "./src/index.ts"
  },
  "dependencies": {
    "@lw/content": "*",
    "@lw/navigation": "*",
    "@lw/sites": "*"
  },
  "scripts": {
    "test": "node --test \"src/**/*.test.ts\"",
    "typecheck": "tsc --noEmit"
  }
}

The check does not need a server. It asks, of each old address, whether the new site has it, from what the new sites are made of: their
pages, the folders those pages sit in (each has a listing), the PDFs and solution files that will be copied, and the decided redirects. An old
address is "same" (the new site answers it), "redirected" (a decided redirect ends on something that exists) or "missing". A redirect to a page
that does not exist is NOT counted as success.

Save as migration-report.mjs:

// Check every address of the old site against the new sites, suggest redirects, and compare with the Django project's answers.
//   node migration-report.mjs <old site folder> [--csv <file with the Django proposals to compare with>]...    (LW_CONTENT_ROOT = content folder)
import { readFileSync } from "node:fs";
import { assetPaths, loadPages } from "./packages/content/src/index.ts";
import { Resolver, parseCsv, propose, scanOldSite, toCsv, verifyAll } from "./packages/migration/src/index.ts";

const args = process.argv.slice(2);
const oldRoot = args[0];
const djangoCsvs = args.flatMap((a, i) => (a === "--csv" ? [args[i + 1]] : []));
const root = process.env.LW_CONTENT_ROOT;
if (!oldRoot || !root) throw new Error("usage: LW_CONTENT_ROOT=<content> node migration-report.mjs <old site folder> [--csv file]...");

const started = performance.now();
const { pages } = loadPages(root);
const assets = new Set(assetPaths(root));
const old = scanOldSite(oldRoot);
const kinds = {};
for (const u of old) kinds[u.kind] = (kinds[u.kind] ?? 0) + 1;
console.log(`old site: ${JSON.stringify(kinds)};  new sites: ${pages.length} pages, ${assets.size} PDFs and solutions   (${Math.round(performance.now() - started)} ms to read both)`);

const before = verifyAll(old, new Resolver(pages, assets, []));
console.log(`\nBEFORE any redirect`);
console.log(`  pages owned by a site:  ${JSON.stringify(before.pages)}`);
console.log(`  assets owned by a site: ${JSON.stringify(before.assets)}`);
console.log(`  addresses no site owns: ${before.unowned}`);

const missingPages = before.missing.filter((u) => u.kind === "page").map((u) => ({ site: u.site, oldPath: u.path }));
const { proposals, ambiguous, nothing } = propose(missingPages, pages);
console.log(`\nSUGGESTIONS for the ${missingPages.length} missing pages: ${proposals.length} with exactly one page of that file name, ${ambiguous.length} with several, ${nothing.length} with none`);
console.log(`  ${proposals.filter((p) => p.why === "same file name").length} by file name, ${proposals.filter((p) => p.why !== "same file name").length} as the print version of a page`);

if (djangoCsvs.length) {
  const theirs = new Map();
  for (const file of djangoCsvs) for (const row of parseCsv(readFileSync(file, "utf8")).slice(1)) theirs.set(row[1], row[3]);
  const mine = new Map(proposals.map((p) => [p.oldPath, p.newPath]));
  const onlyMine = [...mine.keys()].filter((k) => !theirs.has(k));
  const onlyTheirs = [...theirs.keys()].filter((k) => !mine.has(k));
  const different = [...mine.keys()].filter((k) => theirs.has(k) && theirs.get(k) !== mine.get(k));
  console.log(`  compared with the Django project's ${theirs.size} suggestions: same ${mine.size - onlyMine.length - different.length}, only here ${onlyMine.length}, only there ${onlyTheirs.length}, pointing somewhere else ${different.length}`);
}

// pretend every suggestion was reviewed and accepted, to measure what the mechanism achieves (NOT a real review)
const accepted = proposals.map(({ why, ...r }) => r);
const after = verifyAll(old, new Resolver(pages, assets, accepted));
console.log(`\nAFTER accepting all ${accepted.length} suggestions (a measurement, not a review)`);
console.log(`  pages owned by a site:  ${JSON.stringify(after.pages)}`);
console.log(`  still missing pages:    ${after.missing.filter((u) => u.kind === "page").length};  missing files: ${after.missing.filter((u) => u.kind === "asset").length}`);
for (const u of after.missing.filter((x) => x.kind === "page").slice(0, 30)) console.log("    " + u.path);
console.log(`\n(${toCsv(proposals.slice(0, 2)).split("\n").slice(0, 3).join("\n")})`);

Run on the real old site (debserver/website) and the real content:

old site: {"asset":5831,"dynamic":23,"page":3205,"static":742};  new sites: 4422 pages, 8440 PDFs and solutions   (4910 ms to read both)

BEFORE any redirect
  pages owned by a site:  {"same":3102,"redirected":0,"missing":80}
  assets owned by a site: {"same":5544,"redirected":0,"missing":166}
  addresses no site owns: 144

SUGGESTIONS for the 80 missing pages: 55 with exactly one page of that file name, 0 with several, 25 with none
  32 by file name, 23 as the print version of a page
node:fs:441
    return binding.readFileUtf8(path, stringToFlags(options.flag));
                   ^

Error: ENOENT: no such file or directory, open 'C:\c\lwdj\proposed.csv'
    at readFileSync (node:fs:441:20)
    at file:///C:/lwnx/migration-report.mjs:34:61
    at ModuleJob.run (node:internal/modules/esm/module_job:439:25)
    at async node:internal/modules/esm/loader:643:26
    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5) {
  errno: -4058,
  code: 'ENOENT',

(That is the same answer the Django project got, number for number: 3,102 same and 80 missing before; 55 redirected and 25 missing after. Reading the old site and the 4,422 pages
and checking the 9,000 addresses took a few seconds; I did not time it more precisely.)

WHAT THE 80 MISSING PAGES TURNED OUT TO BE (from reading the list): 32 lessons whose folder was renamed or moved (hungary/hungarian-language became
hungary/hungarian-lessons; ten Japanese lessons moved from ai/claude-tools/claude-lessons to japan/), 23 "_print" pages (the old site had a
printable twin of many pages), and 25 others: index pages of renamed folders, old "tentative-*" course folders that were removed, and
japan/kanji-tiles. The 166 missing files are mostly older copies of Node.js and Express solution files and the Next.js rebuild course's solutions.
No site owns 144 addresses (mostly resources/php, resources/js and resources/csset), and the 23 .php pages (the Anime Vault) are not migrated.

Save as packages/migration/src/migration.test.ts:

import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { test } from "node:test";

import type { Page } from "@lw/content";
import { SITE_NAMES, SITES } from "@lw/sites";

import {
  Resolver, bareAddress, checkLinks, escapeSource, loadReviewed, nextRedirects, normaliseSlug, ownerOf, parseCsv, patternFor, propose,
  rewriteCrossSiteLinks, ruleFor, scanOldSite, toCsv, verifyAll, type Redirect,
} from "./index.ts";

const page = (path: string, site: Page["site"] = "languages"): Page => ({
  path, site, kind: "lesson", title: path, summary: "", courseName: null, courseNo: null, chapterNo: null, created: null, updated: null,
});

function oldTree(files: string[]): string {
  const root = mkdtempSync(join(tmpdir(), "lw-old-"));
  for (const rel of files) { mkdirSync(dirname(join(root, rel)), { recursive: true }); writeFileSync(join(root, rel), "x"); }
  return root;
}

test("every form of an address becomes the same bare address", () => {
  for (const form of ["/hungary/x/y/", "hungary/x/y", "/hungary/x/y/index.html", "hungary/x/y.html", "/hungary/x/y/index.html/"]) assert.equal(bareAddress(form), "hungary/x/y", form);
  assert.equal(bareAddress("/"), "");
  assert.equal(normaliseSlug("Lesson-One"), "lesson_one");
});

test("the old site is read as pages, files, dynamic pages and static files, with the site that owns each", () => {
  const root = oldTree(["index.html", "hungary/x/index.html", "hungary/x/pdfs/a.pdf", "hungary/x/solutions/s.txt", "linux/y/index.html", "anime/vault.php",
    "_astro/app.css", "css/site.css", "resources/php/x.pdf", "sidebar/football/index.html"]);
  try {
    const found = new Map(scanOldSite(root).map((u) => [u.path, u]));
    assert.deepEqual([found.get("hungary/x")?.kind, found.get("hungary/x")?.site], ["page", "languages"]);
    assert.equal(found.get("linux/y")?.site, "systems");
    assert.equal(found.get("sidebar/football")?.site, "humanities");
    assert.deepEqual([found.get("hungary/x/pdfs/a.pdf")?.kind, found.get("hungary/x/solutions/s.txt")?.site], ["asset", "languages"]);
    assert.equal(found.get("anime/vault.php")?.kind, "dynamic");
    assert.equal(found.get("css/site.css")?.kind, "static");
    assert.equal(found.get("resources/php/x.pdf")?.site, null);              // owned by no site
    assert.equal(found.get("")?.site, null);                                 // the old front page
    assert.ok(![...found.keys()].some((k) => k.startsWith("_astro")));
  } finally { rmSync(root, { recursive: true }); }
});

const pages = [page("hungary/hungarian-lessons/lesson_one.html"), page("hungary/hungarian-lessons/page_two.html"), page("a/b/shared_name.html"),
  page("c/d/shared_name.html"), page("linux/systems-x/pm_lesson_01.html", "systems")];

test("a redirect is suggested only when exactly one page has that file name", () => {
  const result = propose([
    { site: "languages", oldPath: "hungary/hungarian-language/Lesson-One" }, { site: "languages", oldPath: "old/shared_name" },
    { site: "languages", oldPath: "old/unknown_page" }, { site: "languages", oldPath: "old/page_two_print" },
    { site: "systems", oldPath: "linux/tentative/pm_lesson_01" },
  ], pages);
  assert.deepEqual(result.proposals.map((p) => [p.oldPath, p.newSite, p.newPath, p.why]), [
    ["hungary/hungarian-language/Lesson-One", "languages", "hungary/hungarian-lessons/lesson_one", "same file name"],
    ["old/page_two_print", "languages", "hungary/hungarian-lessons/page_two", "was the print version of this page"],
    ["linux/tentative/pm_lesson_01", "systems", "linux/systems-x/pm_lesson_01", "same file name"],
  ]);
  assert.deepEqual(result.ambiguous[0]?.candidates.sort(), ["a/b/shared_name", "c/d/shared_name"]);
  assert.deepEqual(result.nothing.map((n) => n.oldPath), ["old/unknown_page"]);
});

test("a page whose own name ends in _print is matched as itself, not as a print version", () => {
  const result = propose([{ site: "languages", oldPath: "old/blueprint_print" }], [...pages, page("x/blueprint_print.html")]);
  assert.equal(result.proposals[0]?.why, "same file name");
});

test("the review file: only rows marked yes are loaded, and quotes and commas survive", () => {
  const proposals = [
    { site: "languages" as const, oldPath: "a/one", newSite: "languages" as const, newPath: "b/one", why: 'same file name, and "quoted"' },
    { site: "languages" as const, oldPath: "a/two", newSite: "systems" as const, newPath: "b/two", why: "x" },
  ];
  const csv = toCsv(proposals);
  assert.deepEqual(parseCsv(csv).map((r) => r.length), [6, 6, 6]);
  assert.equal(parseCsv(csv)[1]?.[4], 'same file name, and "quoted"');
  assert.deepEqual(loadReviewed(csv), { redirects: [], skipped: 2 });                          // nothing reviewed: nothing loaded
  const reviewed = csv.replace('"quoted""",', '"quoted""",yes').replace(/\n$/, "\n");
  assert.equal(loadReviewed(reviewed.replace('"quoted""",yes', '"quoted""",yes')).redirects.length, 1);
  assert.equal(loadReviewed(csv, { acceptAll: true }).redirects.length, 2);
  assert.throws(() => loadReviewed("wrong,header\n"), /must be/);
});

test("Next.js redirects: a source is a pattern, so its special characters are made plain; another site gets a full address", () => {
  assert.equal(escapeSource("a/b(c):d*e+f?g{h}"), "a/b\\(c\\)\\:d\\*e\\+f\\?g\\{h\\}");
  const list: Redirect[] = [
    { site: "languages", oldPath: "hungary/old/x", newSite: "languages", newPath: "hungary/new/x" },
    { site: "ai", oldPath: "ai/claude-tools/lesson", newSite: "languages", newPath: "japan/lesson" },
  ];
  assert.deepEqual(nextRedirects(list, "languages", "dev"), [{ source: "/hungary/old/x", destination: "/hungary/new/x", permanent: true }]);
  assert.deepEqual(nextRedirects(list, "ai", "prod"), [{ source: "/ai/claude-tools/lesson", destination: "https://languages.osztromok.com/japan/lesson", permanent: true }]);
  assert.deepEqual(nextRedirects(list, "ai", "dev")[0]?.destination, "http://languages.localhost:3001/japan/lesson");
});

test("an old address is the same, redirected, or missing; a redirect to nothing is NOT success", () => {
  const assets = new Set(["hungary/x/pdfs/a.pdf"]);
  const redirects: Redirect[] = [
    { site: "languages", oldPath: "hungary/old/page_two", newSite: "languages", newPath: "hungary/hungarian-lessons/page_two" },
    { site: "languages", oldPath: "hungary/old/dead", newSite: "languages", newPath: "hungary/nowhere" },
  ];
  const resolver = new Resolver(pages, assets, redirects);
  const url = (path: string, kind: "page" | "asset" = "page", site: "languages" | null = "languages") => ({ path, kind, site });
  assert.equal(resolver.check(url("hungary/hungarian-lessons/lesson_one")), "same");
  assert.equal(resolver.check(url("hungary/hungarian-lessons")), "same");                      // a folder has a listing
  assert.equal(resolver.check(url("hungary/old/page_two")), "redirected");
  assert.equal(resolver.check(url("hungary/old/dead")), "missing");
  assert.equal(resolver.check(url("hungary/old/unknown")), "missing");
  assert.equal(resolver.check(url("hungary/x/pdfs/a.pdf", "asset")), "same");
  assert.equal(resolver.check(url("hungary/x/pdfs/b.pdf", "asset")), "missing");
  assert.equal(resolver.check(url("anime/vault.php", "page", null)), null);                    // no site owns it
  const report = verifyAll([url("hungary/old/page_two"), url("hungary/old/unknown"), url("x", "page", null)], resolver);
  assert.deepEqual([report.pages, report.unowned], [{ same: 0, redirected: 1, missing: 1 }, 1]);
});

test("the cut-over line of each site names its own folders and its own host, and no folder belongs to two sites", () => {
  for (const site of SITE_NAMES) {
    assert.ok(ruleFor(site).startsWith("RedirectMatch 301 ^/("));
    assert.ok(ruleFor(site).endsWith(` https://${site}.osztromok.com/$1$2`));
  }
  for (const site of SITE_NAMES) for (const folder of SITES[site].folders) {
    const owners = SITE_NAMES.filter((s) => patternFor(s).test(`/${folder}/x/`));
    assert.deepEqual(owners, [site], folder);
  }
  assert.ok(patternFor("systems").test("/linux"));
  assert.ok(patternFor("systems").test("/sidebar/linux/cheat_sheet_vim/"));
  for (const no of ["/linuxfoo/", "/sidebar/", "/sidebar/football/x/", "/hungary/x/", "/"]) assert.ok(!patternFor("systems").test(no), no);
  assert.deepEqual(SITE_NAMES.filter((s) => patternFor(s).test("/sidebar/football/x/")), ["humanities"]);
  assert.deepEqual(SITE_NAMES.filter((s) => patternFor(s).test("/resources/japanese/kanji/x.html")), ["languages"]);
});

test("a link into another site gets that site's address, built when the page is built; other links are left alone", () => {
  const { html, rewritten } = rewriteCrossSiteLinks(
    `<a href="/linux/a/b/">x</a> <a href='/linux/z/?q=1'>y</a> <a href="/hungary/x/">same site</a> <a href="//cdn.example/linux/">cdn</a> <a href="https://e.com/linux/">ext</a> <a href="/search/?q=linux">s</a> <a href="#top">t</a>`,
    "languages", "dev");
  assert.equal(rewritten, 2);
  assert.ok(html.includes('href="http://systems.localhost:3004/linux/a/b/"') && html.includes("href='http://systems.localhost:3004/linux/z/?q=1'"));
  assert.ok(html.includes('href="/hungary/x/"') && html.includes('href="//cdn.example/linux/"') && html.includes('href="https://e.com/linux/"') && html.includes('href="/search/?q=linux"'));
  assert.equal(rewriteCrossSiteLinks('<a href="/linux/a/">x</a>', "languages", "prod").html, '<a href="https://systems.osztromok.com/linux/a/">x</a>');
  assert.equal(ownerOf("/sidebar/football/x/"), "humanities");
  assert.equal(ownerOf("/"), null);
});

test("the link checker finds links to nothing, treats a link to another site as fine, and knows about redirects and files", () => {
  const withLinks = [page("hungary/x/a.html"), page("hungary/x/b.html"), page("hungary/x/c.html"), page("linux/y/z.html", "systems")];
  const fragments: Record<string, string> = {
    "hungary/x/a.html": '<a href="/hungary/x/b/">ok</a> <a href="/hungary/x/">folder</a> <a href="/hungary/old/moved/">moved</a> <a href="/hungary/x/solutions/s.txt">file</a> <a href="/linux/y/z/">other site</a>',
    "hungary/x/b.html": '<a href="/hungary/x/nothing/">dead</a> <a href="/hungary/x/solutions/gone.txt">file gone</a> <a href="/search/?q=1">not content</a> <a href="https://e.com/">external</a>',
    "hungary/x/c.html": '<a href="/hungary/x/a.html">old style</a> <a href="/hungary/x/b/index.html">old style</a>',
    "linux/y/z.html": "",
  };
  const resolver = new Resolver(withLinks, new Set(["hungary/x/solutions/s.txt"]), [{ site: "languages", oldPath: "hungary/old/moved", newSite: "languages", newPath: "hungary/x/a" }]);
  const { checked, broken } = checkLinks(withLinks, (p) => fragments[p.path] ?? "", resolver, new Set(["hungary/x/solutions/s.txt"]));
  assert.equal(checked, 10);                       // 5 links in a, 3 in b (the https one is not an internal link), 2 in c
  assert.deepEqual(broken.map((b) => [b.link, b.why]), [["/hungary/x/nothing/", "no such page"], ["/hungary/x/solutions/gone.txt", "file not found"]]);
});

  npm test
ℹ tests 100
ℹ pass 100
ℹ fail 0
  (90 earlier and 10 new)

A MISTAKE IN MY OWN TEST: I wrote that a page-link test would check 11 links; it checked 10 (5 in one page, 3 in the second, 2 in the third; the
"https://" link is not an internal link). I had counted by eye. The code was right and the test's number was wrong: count by hand a second time, or
let the test print what it found before you write the number down.

WHY THIS WORKS AS AN ANSWER
---------------------------
"Most of it already works, here are the 80 that do not and why" is something you can act on. It was measured, and it agrees with an independent
implementation to the last number.