T

learning-website-nextjs1-4 Exercise 1: Prepare a Fragment, and Prove It Matches Django =============================================================================================== A content file is not yet what a site shows. Three steps (the same ones the Django importer takes) turn it into a fragment: 1. a complete HTML document gives its body, preceded by the styles from its head; a fragment loses the banner comment at its top; 2. every .txt link is pointed at the course's own solutions folder, whatever the old link said; 3. a site may add one change of its own: the languages site marks each line in the language being taught with a lang attribute (lang="hu", lang="ja", ...), so screen readers and browsers treat it correctly. Save as packages/content/src/fragment.ts: import { posix } from "node:path"; import type { SiteName } from "@lw/sites"; import { addLanguageAttributes } from "./languages.ts"; import { ContentParseError } from "./parse.ts"; /** * Turn a raw content file into the fragment a site shows. The same steps as the Django importer: * a complete document gives its body (plus the styles from its head), a fragment loses its banner comment, * every .txt link is pointed at the course's own solutions folder, and a site may add its own change. */ const BODY_OPEN = /]*>/i; const BODY_CLOSE = /<\/body>/i; const HEAD = /]*>([\s\S]*?)<\/head>/i; const STYLE_ONE = /]*>[\s\S]*?<\/style>/i; const STYLE_ALL = /]*>[\s\S]*?<\/style>/gi; const LEADING_COMMENT = /^\s*\s*/; const TXT_HREF = /href="([^"]*?)([^"/]+\.txt)"/gi; export class MalformedDocument extends ContentParseError { constructor(message: string) { super(message); this.name = "MalformedDocument"; } } export function extractFragment(raw: string): string { const opened = BODY_OPEN.exec(raw); if (!opened) return raw.replace(LEADING_COMMENT, ""); // a fragment loses its banner comment const closed = BODY_CLOSE.exec(raw); if (!closed || closed.index < opened.index) throw new MalformedDocument(" without a matching "); const inner = raw.slice(opened.index + opened[0].length, closed.index); const head = HEAD.exec(raw); const headText = head?.[1] ?? ""; const styles = STYLE_ONE.test(headText) ? (headText.match(STYLE_ALL) ?? []).join("\n") + "\n" : ""; return styles + inner; } /** Percent-decoding like Python's unquote: bad sequences become U+FFFD, they never throw. */ export function unquote(text: string): string { return text.replace(/(?:%[0-9a-fA-F]{2})+/g, (run) => new TextDecoder("utf-8").decode(Uint8Array.from(run.slice(1).split("%").map((hex) => parseInt(hex, 16))))); } /** Every .txt link is a solution file: point it at /solutions/, whatever it said before. */ export function rewriteSolutionLinks(fragment: string, courseUrlPath: string): string { const prefix = courseUrlPath.startsWith("/") ? courseUrlPath : "/" + courseUrlPath; return fragment.replace(TXT_HREF, (_whole, _before: string, file: string) => `href="${prefix}/solutions/${unquote(file)}"`); } export type Transform = (fragment: string) => string; /** Changes that only one site wants. */ export const SITE_TRANSFORMS: Partial> = { languages: [addLanguageAttributes], }; export function prepareFragment(raw: string, path: string, site: SiteName): string { const courseUrlPath = path.includes("/") ? "/" + posix.dirname(path) : ""; let fragment = rewriteSolutionLinks(extractFragment(raw), courseUrlPath); for (const transform of SITE_TRANSFORMS[site] ?? []) fragment = transform(fragment); return fragment; } Save as packages/content/src/languages.ts: /** * Mark the text in the language being taught with a lang attribute. * * Lesson fragments wrap each line in the language being taught in , and so on. * Adding lang="hu" / lang="ja" helps a screen reader pronounce it, makes the browser pick the right glyph forms for * Japanese, and gives spell checking the right language. It is safe to run twice. */ /** the class a lesson uses for a line in the language being taught -> the HTML lang code for that text */ export const CLASS_TO_LANG: Readonly> = { hu: "hu", de: "de", fr: "fr", jp: "ja" }; const TAG = /<([a-zA-Z][a-zA-Z0-9]*)(\s[^<>]*?)?>/g; const CLASS = /\sclass\s*=\s*"([^"]*)"/; export function addLanguageAttributes(fragment: string): string { return fragment.replace(TAG, (whole: string, tag: string, attrs: string | undefined) => { const attributes = attrs ?? ""; if (attributes.includes("lang=")) return whole; // already marked: leave it alone const found = CLASS.exec(attributes); if (!found) return whole; for (const token of (found[1] ?? "").split(/\s+/)) { // whole class names only: "hu", not "hu-lesson" const code = Object.hasOwn(CLASS_TO_LANG, token) ? CLASS_TO_LANG[token] : undefined; if (code !== undefined) return `<${tag} lang="${code}"${attributes}>`; } return whole; }); } Save as packages/content/src/index.ts: export { parseBanner } from "./banner.ts"; export { ContentParseError, bestTitle, checkedDate, decodeEntities, humanize, numbersFromPath, parsePage, } from "./parse.ts"; export { contentPaths, groupCourses, loadPages, readContentFile } from "./scan.ts"; export type { LoadOptions, LoadResult } from "./scan.ts"; export { MalformedDocument, SITE_TRANSFORMS, extractFragment, prepareFragment, rewriteSolutionLinks, unquote, } from "./fragment.ts"; export type { Transform } from "./fragment.ts"; export { CLASS_TO_LANG, addLanguageAttributes } from "./languages.ts"; export type { Banner, ContentError, Course, IsoDate, Kind, Page } from "./types.ts"; The loader also gained an option, loadPages(root, { site }): the site is decided from the PATH alone, so another site's files are never read (the languages site reads 561 files, not 4,414): export interface LoadOptions { readonly site?: SiteName } Save as packages/content/src/fragment.test.ts: import assert from "node:assert/strict"; import { test } from "node:test"; import { MalformedDocument, addLanguageAttributes, extractFragment, prepareFragment, rewriteSolutionLinks, unquote, } from "./index.ts"; test("a fragment loses its banner comment and nothing else", () => { assert.equal(extractFragment("\n

"), "

"); assert.equal(extractFragment("

no comment

"), "

no comment

"); }); test("a complete document gives its body, preceded by the styles from its head", () => { const raw = "T" + "

"; assert.equal(extractFragment(raw), "\n\n

"); assert.equal(extractFragment("

"), "

"); }); test("a body that never closes is refused, not guessed at", () => { assert.throws(() => extractFragment("

"), MalformedDocument); assert.throws(() => extractFragment("x"), MalformedDocument); }); test("percent-decoding never throws and understands UTF-8", () => { assert.equal(unquote("a%20b"), "a b"); assert.equal(unquote("%E3%81%82.txt"), "あ.txt"); assert.equal(unquote("100%"), "100%"); assert.equal(unquote("%E3%81"), "�"); // a cut-off sequence is replaced, not an error }); test("every .txt link is pointed at the course's solutions folder", () => { const html = 'a b c'; assert.equal( rewriteSolutionLinks(html, "/web/js"), 'a b c'); assert.equal(rewriteSolutionLinks('', "web/js"), ''); }); test("language marking adds lang to whole class names only", () => { assert.equal(addLanguageAttributes('x'), 'x'); assert.equal(addLanguageAttributes('

'), '

'); assert.equal(addLanguageAttributes('

'), '

'); assert.equal(addLanguageAttributes('x'), 'x'); assert.equal(addLanguageAttributes('x'), 'x'); // not a property lookup assert.equal(addLanguageAttributes("

"), "

"); }); test("language marking is safe to run twice", () => { const once = addLanguageAttributes('x'); assert.equal(addLanguageAttributes(once), once); }); test("preparing a page applies the shared steps, then the site's own", () => { const raw = '\noui e'; assert.equal( prepareFragment(raw, "france/french-basic-1/f_1_1.html", "languages"), 'ouie'); assert.equal( prepareFragment(raw, "web-development/x/p.html", "webdevelopment"), 'ouie'); // no marking on another site assert.equal(prepareFragment("

", "top.html", "languages"), "

"); // a page with no folder }); Run: npm test ℹ tests 34 ℹ pass 34 ℹ fail 0 (the 17 tests of Chapter 3, 9 of the site package, and 8 new ones) Now the proof. The Django project stored the prepared fragment of every page; its dumphashes.py writes a SHA-256 of each. compare-fragments.mjs prepares every page here and compares the hashes: Save as compare-fragments.mjs: // Prepare every page's fragment with @lw/content and compare its hash with the one Django stored. // node compare-fragments.mjs import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { loadPages, prepareFragment, readContentFile } from "./packages/content/src/index.ts"; const [root, hashesPath] = process.argv.slice(2); const django = JSON.parse(readFileSync(hashesPath, "utf8")); const started = performance.now(); const { pages } = loadPages(root); let same = 0; const different = []; const missing = []; const failed = []; for (const page of pages) { const expected = django[page.path]; if (expected === undefined) { missing.push(page.path); continue; } let fragment; try { fragment = prepareFragment(readContentFile(root, page.path), page.path, page.site); } catch (error) { failed.push(`${page.path}: ${error.message}`); continue; } const hash = createHash("sha256").update(fragment, "utf8").digest("hex"); if (hash === expected) same++; else different.push(page.path); } const ms = Math.round(performance.now() - started); console.log(`${pages.length} pages prepared in ${ms} ms`); console.log(`identical to Django's stored fragment: ${same}`); console.log(`different: ${different.length}`, different.slice(0, 5)); console.log(`not in Django: ${missing.length} failed to prepare: ${failed.length}`, failed.slice(0, 3)); console.log(`Django pages not parsed here: ${Object.keys(django).filter((p) => !pages.some((x) => x.path === p)).length}`); python dumphashes.py django-hashes.json (in the Django project) 4414 fragment hashes node compare-fragments.mjs django-hashes.json 4415 pages prepared in 9078 ms identical to Django's stored fragment: 4414 different: 0 [] not in Django: 1 failed to prepare: 0 [] Django pages not parsed here: 0 (4,414 identical; the 4,415th page is this chapter's own new file, which the Django database has not imported yet.) Can this comparison fail? Three deliberate breakages of the TypeScript code, each undone afterwards: language marking switched off identical 4253, different 161 file names in solution links not percent-decoded different 0 decoded with the wrong text encoding different 0 The first shows the comparison works: 161 pages carry language marks. The other two show something honest: NO real page has a percent-encoded .txt file name, so that decoding (and its UTF-8 handling) is checked only by its unit tests, not by the real content. WHY THIS WORKS AS AN ANSWER --------------------------- "The same as Django" is shown byte for byte on 4,414 real pages, with a note of which parts of the code the real data does not exercise.