learning-website-django1-11 Exercise 1: Measure the Old Site Instead of Remembering It ================================================================================== Django 6.1.2. Before moving anything, list every address the old site answers and check each one against the new sites. The old site is a folder of built files (/hungary/x/y/ is the file hungary/x/y/index.html), so that folder is the complete, factual list. Save as apps/migration/oldsite.py: """Read the OLD site (the folder that is deployed today) and list every address it answers. The old site is a folder of built files: /hungary/x/y/ is the file hungary/x/y/index.html, and a PDF or a solution file is served at its own path. That folder is the complete, factual list of old addresses, so the migration is checked against it instead of against anyone's memory of what existed.""" import os from dataclasses import dataclass from config.sites_config import NoSiteError, site_for_path STATIC_TOP = {"_astro"} # the old build's own css and scripts: not content ASSET_EXTENSIONS = (".pdf", ".txt") STATIC_EXTENSIONS = (".css", ".js", ".jpg", ".png", ".svg", ".ico", ".md", "") @dataclass(frozen=True) class OldUrl: path: str # no leading or trailing slash: "hungary/x/y" for a page, "hungary/x/pdfs/a.pdf" for a file kind: str # "page", "asset", "dynamic" (.php) or "static" site: str | None # the new site that now owns the folder, or None when no site does def _owner(path): try: return site_for_path(path + "/x.html" if "." not in path.rsplit("/", 1)[-1] else path) except NoSiteError: return None def scan_old_site(root): """Every old address, as a list of OldUrl, sorted by path.""" found = [] for dirpath, dirnames, filenames in os.walk(root): rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/") rel_dir = "" if rel_dir == "." else rel_dir if rel_dir.split("/")[0] in STATIC_TOP: dirnames[:] = [] continue for name in filenames: rel = f"{rel_dir}/{name}" if rel_dir else name lower = name.lower() if lower == "index.html": found.append(OldUrl(rel_dir, "page", _owner(rel_dir) if rel_dir else None)) elif lower.endswith(ASSET_EXTENSIONS): found.append(OldUrl(rel, "asset", _owner(rel))) elif lower.endswith(".php"): found.append(OldUrl(rel, "dynamic", None)) elif lower.endswith(".html"): found.append(OldUrl(rel, "static", None)) # a bare .html next to index pages: not an address of a page else: found.append(OldUrl(rel, "static", None)) return sorted(found, key=lambda u: (u.kind, u.path)) Save as apps/migration/verify.py: """Check every OLD address against the NEW sites.""" from collections import Counter from pathlib import Path from django.conf import settings from django.test import Client from config.sites_config import host_map def _host(site): return next(host for host, name in host_map("dev").items() if name == site) def check_page(client, site, path): """-> "same" (the old address still works), "redirected" (a redirect that ends on a page) or "missing".""" response = client.get("/" + path + "/" if path else "/", HTTP_HOST=_host(site)) if response.status_code == 200: return "same" if response.status_code in (301, 302): target = response["Location"] if target.startswith("http"): # another site: check it there from urllib.parse import urlsplit host, parts = urlsplit(target).netloc.split(":")[0], urlsplit(target).path follow = client.get(parts, HTTP_HOST=host) else: follow = client.get(target, HTTP_HOST=_host(site)) return "redirected" if follow.status_code == 200 else "missing" return "missing" def check_asset(site, path, asset_root=None): return "same" if (Path(asset_root or settings.ASSET_ROOT) / site / path).is_file() else "missing" def verify(old_urls, asset_root=None): """Run every old page and asset address that a site now owns. Returns (counts, missing) where counts is {kind: Counter(result)} and missing is a list of (kind, site, path).""" client = Client() counts = {"page": Counter(), "asset": Counter()} missing = [] for url in old_urls: if url.kind not in counts or url.site is None: continue result = check_page(client, url.site, url.path) if url.kind == "page" else check_asset(url.site, url.path, asset_root) counts[url.kind][result] += 1 if result == "missing": missing.append((url.kind, url.site, url.path)) return counts, missing Save as apps/migration/management/commands/verify_old_urls.py: from django.core.management.base import BaseCommand, CommandError from apps.migration.oldsite import scan_old_site from apps.migration.verify import verify class Command(BaseCommand): help = "Check every address of the old site (a folder of built files) against the new sites." def add_arguments(self, parser): parser.add_argument("old_root", help="the folder the old site is served from") parser.add_argument("--list", metavar="FILE", help="write every missing address to this file") def handle(self, old_root, list=None, **options): urls = scan_old_site(old_root) if not urls: raise CommandError(f"no files found under {old_root}") by_kind = {} for url in urls: by_kind.setdefault(url.kind, []).append(url) self.stdout.write("Old site: " + ", ".join(f"{len(v)} {k}" for k, v in sorted(by_kind.items()))) unowned = [u for u in urls if u.kind in ("page", "asset") and u.site is None] self.stdout.write(f"No new site owns {len(unowned)} page/asset addresses (decide: keep on the old host, or drop).") counts, missing = verify(urls) for kind, counter in counts.items(): total = sum(counter.values()) self.stdout.write(f"{kind}s owned by a site: {total} " + ", ".join(f"{n} {r}" for r, n in sorted(counter.items()))) if list: with open(list, "w", encoding="utf-8") as handle: for kind, site, path in missing: handle.write(f"{kind}\t{site}\t{path}\n") self.stdout.write(f"Missing addresses written to {list}") if missing: raise CommandError(f"{len(missing)} old addresses do not work on the new sites") self.stdout.write(self.style.SUCCESS("Every old address works.")) A page counts as "same" when the old address answers 200 on the site that now owns its folder, "redirected" when it answers a redirect that ends on a page that exists (a redirect to a page that does not exist is NOT counted as success), and "missing" otherwise. A PDF or solution file counts when the file exists in that site's asset folder. Run on the real old site (debserver/website) and the real new database (4,410 pages, assets mirrored with collect_assets): python manage.py verify_old_urls "" --list missing.tsv Old site: 5831 asset, 23 dynamic, 3205 page, 742 static No new site owns 144 page/asset addresses (decide: keep on the old host, or drop). pages owned by a site: 3182 80 missing, 3102 same assets owned by a site: 5710 166 missing, 5544 same So before any redirect: 3,102 of 3,182 old pages (97.5%) already have the SAME address on the new sites, and 80 do not. Of the 5,710 files, 5,544 are in place and 166 are not. What the 80 missing pages turned out to be (read from the list, not guessed): - 32 old lesson addresses whose folder was renamed or moved, for example hungary/hungarian-language/... is now hungary/hungarian-lessons/..., and ten Japanese lessons that used to sit under ai/claude-tools/claude-lessons/ are now under japan/ - 23 "_print" pages: the old site had a printable twin of many pages - 25 others: folder index pages of renamed folders, old "tentative-*" course folders that were removed, and japan/kanji-tiles What the 166 missing files turned out to be: - 120 are older copies of Node.js and Express solution files in text-files/ folders (the courses now keep them in solutions/) and 36 are in projects/nextjs-rebuild/solutions - the rest are a handful of PDFs in renamed folders and three in resources/hungarian What no site owns (144 addresses, not counted as missing): mostly resources/php (83 files), resources/js (30), resources/csset (19), the old front page, /admin/ and /sidebar/. The old site's 23 .php files (the Anime Vault) are dynamic pages and are not migrated at all. WHY THIS WORKS AS AN ANSWER --------------------------- The migration is measured, not estimated: "97.5% already work, here are the 80 that do not, and here is why" is something you can act on, and the command fails (exit code 1) until the list is empty, so it can be run again after every change.