learning-website-nextjs1-11  Exercise 2: Redirects You Review First, Kept in the Configuration
=======================================================================================================

An old address that moved needs a permanent (308) redirect, or every old bookmark becomes a 404. But a browser remembers a permanent redirect
very hard, so a WRONG one is hard to undo. So: suggest, then review, then build.

Save as packages/migration/src/redirects.ts:

import type { Page } from "@lw/content";
import { siteUrl, type Environment, type SiteName } from "@lw/sites";

/** A decided redirect: an old address, and where it went. */
export interface Redirect {
  /** The site that gets the old request, and the old address ("hungary/hungarian-language/lesson_01"). */
  readonly site: SiteName;
  readonly oldPath: string;
  readonly newSite: SiteName;
  readonly newPath: string;
}

/** Any form of an address becomes "hungary/x/y": no slashes at the ends, no index.html, no .html. */
export function bareAddress(path: string): string {
  return path.replace(/^\/+|\/+$/g, "").replace(/(^|\/)index\.html$/, "").replace(/\.html$/, "").replace(/^\/+|\/+$/g, "");
}

/** Compare file names loosely: case, and - against _, are not differences that matter here. */
export function normaliseSlug(slug: string): string {
  return slug.toLowerCase().replaceAll("-", "_");
}

export interface Proposal extends Redirect { readonly why: string }
export interface Proposals {
  /** Exactly ONE page has that file name. */
  readonly proposals: Proposal[];
  /** Several pages have it: a person must choose. */
  readonly ambiguous: { site: SiteName; oldPath: string; candidates: string[] }[];
  /** No page has it. */
  readonly nothing: { site: SiteName; oldPath: string }[];
}

/**
 * For old page addresses that no longer exist, suggest where they went. Suggestions are only suggestions: a file name is not proof that it
 * is the same lesson, so each is reviewed before it becomes a redirect.
 */
export function propose(missing: readonly { site: SiteName; oldPath: string }[], pages: readonly Page[]): Proposals {
  const bySlug = new Map<string, Page[]>();
  for (const page of pages) {
    const slug = normaliseSlug(page.path.slice(0, -".html".length).split("/").at(-1) ?? "");
    bySlug.set(slug, [...(bySlug.get(slug) ?? []), page]);
  }
  const out: Proposals = { proposals: [], ambiguous: [], nothing: [] };
  for (const { site, oldPath } of missing) {
    let slug = normaliseSlug(oldPath.split("/").at(-1) ?? "");
    let found = bySlug.get(slug) ?? [];
    let why = "same file name";
    if (found.length === 0 && slug.endsWith("_print")) {
      // the old site had a printable twin of many pages; the page itself is the closest thing now
      slug = slug.slice(0, -"_print".length);
      found = bySlug.get(slug) ?? [];
      why = "was the print version of this page";
    }
    if (found.length === 1) {
      const page = found[0] as Page;
      out.proposals.push({ site, oldPath, newSite: page.site, newPath: page.path.slice(0, -".html".length), why });
    } else if (found.length > 1) {
      out.ambiguous.push({ site, oldPath, candidates: found.map((p) => p.path.slice(0, -".html".length)) });
    } else {
      out.nothing.push({ site, oldPath });
    }
  }
  return out;
}

// ---- the file you review

const COLUMNS = ["site", "old_path", "new_site", "new_path", "why", "decision"] as const;
const quote = (field: string) => (/[",\n]/.test(field) ? `"${field.replaceAll('"', '""')}"` : field);

/** The decision column is left EMPTY: you write "yes" against each row you accept. */
export function toCsv(proposals: readonly Proposal[]): string {
  const lines = [COLUMNS.join(",")];
  for (const p of proposals) lines.push([p.site, p.oldPath, p.newSite, p.newPath, p.why, ""].map(quote).join(","));
  return lines.join("\n") + "\n";
}

/** Read the file back: one record per row (quoted fields may hold commas, quotes and line breaks). */
export function parseCsv(text: string): string[][] {
  const rows: string[][] = [];
  let row: string[] = [];
  let field = "";
  let quoted = false;
  for (let i = 0; i < text.length; i++) {
    const c = text[i] as string;
    if (quoted) {
      if (c === '"' && text[i + 1] === '"') { field += '"'; i++; }
      else if (c === '"') quoted = false;
      else field += c;
    } else if (c === '"') quoted = true;
    else if (c === ",") { row.push(field); field = ""; }
    else if (c === "\n" || c === "\r") {
      if (c === "\r" && text[i + 1] === "\n") i++;
      row.push(field); field = "";
      if (row.length > 1 || row[0] !== "") rows.push(row);
      row = [];
    } else field += c;
  }
  if (field !== "" || row.length > 0) { row.push(field); rows.push(row); }
  return rows;
}

export interface Loaded { readonly redirects: Redirect[]; readonly skipped: number }

/** Only rows whose decision says yes become redirects. */
export function loadReviewed(csv: string, options: { acceptAll?: boolean } = {}): Loaded {
  const [header, ...rows] = parseCsv(csv);
  if (!header || COLUMNS.some((name, i) => header[i] !== name)) throw new Error(`the first row must be: ${COLUMNS.join(",")}`);
  const redirects: Redirect[] = [];
  let skipped = 0;
  for (const row of rows) {
    const [site, oldPath, newSite, newPath, , decision] = row as [string, string, string, string, string, string?];
    if (!options.acceptAll && (decision ?? "").trim().toLowerCase() !== "yes") { skipped++; continue; }
    redirects.push({ site: site as SiteName, oldPath, newSite: newSite as SiteName, newPath });
  }
  return { redirects, skipped };
}

// ---- Next.js

/** Next reads a redirect's source as a pattern, in which ( ) : * + ? { } mean something. Take the meaning away from them. */
export function escapeSource(path: string): string {
  return path.replace(/[(){}:*+?\\]/g, "\\$&");
}

export interface NextRedirect { source: string; destination: string; permanent: true }

/** The redirects of ONE site as Next.js wants them. A page that moved to ANOTHER site gets a full address. */
export function nextRedirects(redirects: readonly Redirect[], site: SiteName, env: Environment): NextRedirect[] {
  return redirects.filter((r) => r.site === site).map((r) => ({
    source: "/" + escapeSource(bareAddress(r.oldPath)),
    destination: r.newSite === site ? "/" + r.newPath : siteUrl(r.newSite, env, "/" + r.newPath),
    permanent: true as const,
  }));
}

Save as propose-redirects.mjs:

// Write the suggestions for the old pages that no longer exist, as a file to REVIEW (the decision column is empty).
//   LW_CONTENT_ROOT=<content> node propose-redirects.mjs <old site folder> <csv to write>
import { writeFileSync } from "node:fs";
import { assetPaths, loadPages } from "./packages/content/src/index.ts";
import { Resolver, propose, scanOldSite, toCsv, verifyAll } from "./packages/migration/src/index.ts";

const [oldRoot, out] = process.argv.slice(2);
const root = process.env.LW_CONTENT_ROOT;
if (!oldRoot || !out || !root) throw new Error("usage: LW_CONTENT_ROOT=<content> node propose-redirects.mjs <old site folder> <csv to write>");
const { pages } = loadPages(root);
const report = verifyAll(scanOldSite(oldRoot), new Resolver(pages, new Set(assetPaths(root)), []));
const missing = report.missing.filter((u) => u.kind === "page").map((u) => ({ site: u.site, oldPath: u.path }));
const { proposals, ambiguous, nothing } = propose(missing, pages);
writeFileSync(out, toCsv(proposals));
console.log(`${missing.length} old pages do not work: ${proposals.length} suggested, ${ambiguous.length} ambiguous, ${nothing.length} with nothing to suggest`);
console.log(`written to ${out}: put "yes" in the decision column of each row you accept, then run load-redirects.mjs`);
if (nothing.length) console.log("needs a person:\n" + nothing.map((n) => "  " + n.oldPath).join("\n"));

Save as load-redirects.mjs:

// Turn a REVIEWED redirect file into the lists the apps read: one JSON file per site, holding only the rows you marked yes.
//   node load-redirects.mjs <csv> [--accept-all] [--out <folder>]
// --accept-all is for tests: it accepts every row WITHOUT a review, and says so.
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { loadReviewed } from "./packages/migration/src/index.ts";
import { SITE_NAMES } from "./packages/sites/src/index.ts";

const args = process.argv.slice(2);
const csv = args[0];
const acceptAll = args.includes("--accept-all");
const out = args.includes("--out") ? args[args.indexOf("--out") + 1] : join(import.meta.dirname, "redirects");
if (!csv) throw new Error("usage: node load-redirects.mjs <csv> [--accept-all] [--out <folder>]");

const { redirects, skipped } = loadReviewed(readFileSync(csv, "utf8"), { acceptAll });
mkdirSync(out, { recursive: true });
for (const site of SITE_NAMES) {
  const list = redirects.filter((r) => r.site === site);
  writeFileSync(join(out, `${site}.json`), JSON.stringify(list, null, 1) + "\n");
}
console.log(`${redirects.length} redirects written to ${out} (${skipped} rows skipped: not marked yes)${acceptAll ? "  [--accept-all: NOT reviewed]" : ""}`);
for (const site of SITE_NAMES) { const n = redirects.filter((r) => r.site === site).length; if (n) console.log(`  ${site}: ${n}`); }

The suggestion rule is deliberately modest: if exactly ONE new page has the same file name (ignoring capitals and - against _), propose it; if
several do, list them for a person; if none, say so; a name ending in _print falls back to the page without it. A file name is not proof that it
is the same lesson, so every row of the file starts with an EMPTY decision column and only rows where you write "yes" are loaded.

  node propose-redirects.mjs <old site> redirects/proposed.csv
  80 old pages do not work: 55 suggested, 0 ambiguous, 25 with nothing to suggest

  node load-redirects.mjs redirects/proposed.csv            (nothing reviewed yet)
  0 redirects written to redirects (55 rows skipped: not marked yes)

So the lists that are COMMITTED (redirects/<site>.json) are all empty, and redirects/proposed.csv holds the 55 suggestions waiting for you. To
measure what the mechanism does, I made a scratch copy with every row accepted (--accept-all, which says so: "NOT reviewed"): 19 for languages, 8
programming, 11 systems, 11 AI and 6 humanities. That scratch copy is not committed.

Where the redirects live: in the app's configuration. next.config.ts reads the site's list and Next answers the redirects itself (the config can
import the workspace package like any other code):

  async redirects() {
    const file = join(process.env["LW_REDIRECTS_DIR"] ?? join(process.cwd(), "..", "..", "redirects"), "languages.json");
    return existsSync(file) ? nextRedirects(JSON.parse(readFileSync(file, "utf8")) as Redirect[], "languages", environment()) : [];
  },

A redirect's "source" is a PATTERN in Next, in which ( ) : * + ? { } mean something, so those characters are made plain in each old address
(tested). A page that moved to ANOTHER site gets a full address (https://languages.osztromok.com/... in production), which is why a list is kept
for the OLD owner of each address.

THE REAL TEST: build with the scratch list, start the app, and ask it over HTTP about EVERY old address the languages site owns:

Save as verify-http.mjs:

// Ask the RUNNING languages app about every old address the languages site owns, over real HTTP, and compare with the logic-level check.
//   LW_CONTENT_ROOT=<content> LW_REDIRECTS_DIR=<folder used by the build> node verify-http.mjs <old site folder> <app url>
import { readFileSync, existsSync } from "node:fs";
import { join } from "node:path";
import { assetPaths, loadPages } from "./packages/content/src/index.ts";
import { Resolver, scanOldSite } from "./packages/migration/src/index.ts";

const [oldRoot, base = "http://localhost:3001"] = process.argv.slice(2);
const root = process.env.LW_CONTENT_ROOT;
const file = join(process.env.LW_REDIRECTS_DIR ?? join(import.meta.dirname, "redirects"), "languages.json");
const redirects = existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : [];
const { pages } = loadPages(root);
const resolver = new Resolver(pages, new Set(assetPaths(root)), redirects);
const mine = scanOldSite(oldRoot).filter((u) => u.site === "languages" && (u.kind === "page" || u.kind === "asset"));

/** Follow up to 3 redirects by hand, so each hop can be seen. */
async function get(url) {
  const hops = [];
  for (let i = 0; i < 4; i++) {
    const response = await fetch(url, { redirect: "manual" });
    await response.arrayBuffer();
    if (response.status >= 300 && response.status < 400) { hops.push(response.status); url = new URL(response.headers.get("location"), url).href; continue; }
    return { status: response.status, hops, url };
  }
  return { status: 0, hops, url };
}

const tally = { page: { same: 0, redirected: 0, missing: 0 }, asset: { same: 0, redirected: 0, missing: 0 } };
const disagree = [];
const permanent = new Set();
for (const u of mine) {
  const address = `${base}/${u.path}${u.kind === "page" ? "/" : ""}`;
  const result = await get(address);
  // the trailing-slash 308 is not a MOVE: it is the site's own tidy-up. A real redirect has a hop after it.
  const real = result.hops.length > (u.kind === "page" ? 1 : 0);
  const http = result.status === 200 ? (real ? "redirected" : "same") : "missing";
  tally[u.kind][http]++;
  result.hops.forEach((h) => permanent.add(h));
  const logic = resolver.check(u);
  if (logic !== http) disagree.push({ path: u.path, http, logic, hops: result.hops, status: result.status });
}
console.log(`old addresses the languages site owns: ${mine.filter((u) => u.kind === "page").length} pages and ${mine.filter((u) => u.kind === "asset").length} files, asked over HTTP`);
console.log("pages: ", JSON.stringify(tally.page));
console.log("files: ", JSON.stringify(tally.asset));
console.log("redirect statuses seen:", [...permanent].join(", "));
console.log(`HTTP and the logic-level check disagree on ${disagree.length}`);
for (const d of disagree.slice(0, 5)) console.log("  ", JSON.stringify(d));

  old addresses the languages site owns: 319 pages and 10 files, asked over HTTP
  pages:  {"same":297,"redirected":19,"missing":3}
  files:  {"same":4,"redirected":0,"missing":6}
  redirect statuses seen: 308
  HTTP and the logic-level check disagree on 0

The three missing pages are france/french-language and hungary/hungarian-language (the index pages of renamed folders) and japan/kanji-tiles; the six
missing files are in resources/hungarian and in the old hungarian-language/pdfs folder. They need a person (website_checks.md). The HTTP answers and
the check in Exercise 1 agree on all 329 addresses.

What an old address does, by hand:
  /hungary/hungarian-language/hungarian_lesson_01/    308 -> /hungary/hungarian-language/hungarian_lesson_01     (the site's own trailing-slash tidy-up)
  /hungary/hungarian-language/hungarian_lesson_01     308 -> /hungary/hungarian-lessons/hungarian_lesson_01      (the real redirect)
That is TWO hops for an old link with a slash. It works, and each hop is cheap, but it is not one. (With no reviewed list, the first hop is
followed by a 404.)

Can it fail? Taking out the print-version rule in the suggester, and running the Exercise 1 report again: "32 suggestions, compared with the Django
project's 55: same 32, only there 23". Put back.

WHY THIS WORKS AS AN ANSWER
---------------------------
Nothing becomes a permanent redirect without a person writing yes, the mechanism is proved on the real addresses over real HTTP, and the two ways
of asking (logic and HTTP) agree.