Re-implement wcag.json output (#4301)

- Re-implements `wcag.json` generation using TypeScript (reusing code
already involved in the Eleventy build process where possible)
- Always generates based on published guidelines (not ED); also relies
on built Understanding docs (i.e. it needs to run after Eleventy)
- Can generate for both 2.1 and 2.2 (since multiple consumers rely on
parts of both)
- Adds a `WCAG_JSON` environment variable recognized by the Eleventy
build process to also build `wcag.json` at the end
- Updates the `w3c-publish` scripts to include the new environment
variable
- Outputs `wcag.json` under `_site` (the build output folder), since it
is an output of the build process; there may be some discussion as to
the precise behavior of this
- Updates behavior of some fields in the JSON to be easier to work with
by their respective consumers, e.g.:
- Any fields containing HTML have links modified to point to absolute
URLs
- `techniques` is replaced by `techniquesHtml`, as Quickref's version of
the former had been manually modified to the point where it would be
unrealistic to automate, while the latter can be output as-is
- Removes old XML build targets and XSLT code that this supersedes
- Removes wcag.json from the guidelines folder, as it is now part of
build output intended to be published

Author: kfranqueiro

Diff for: 11ty/README.md

@@ -92,6 +92,12 @@ Possible values:
 - `editors` - Sets base URLs appropriate for `gh-pages` publishing; used by deploy action
 - `publication` - Sets base URLs appropriate for WAI site publishing; used by `publish-w3c` script
 
+### `WCAG_JSON`
+
+Generates `_site/wcag.json`. (This is not done by default, as it adds to build time.)
+
+**Default:** Unset by default; `publish-w3c` scripts set this to a non-empty value.
+
 ### `GITHUB_REPOSITORY`
 
 **Usage context:** Automatically set during GitHub workflows; should not need to be set manually

Diff for: 11ty/cheerio.ts

@@ -1,12 +1,14 @@
-import { load, type CheerioOptions } from "cheerio";
+import { load, type CheerioAPI, type CheerioOptions } from "cheerio";
 import { readFileSync } from "fs";
 import { readFile } from "fs/promises";
 import { dirname, resolve } from "path";
 
 export { load } from "cheerio";
 
-/** Superset of the type returned by any Cheerio $() call. */
-export type CheerioAnyNode = ReturnType<ReturnType<typeof load>>;
+/** Superset of the type returned by any Cheerio $() call */
+export type CheerioAnyNode = ReturnType<CheerioAPI>;
+/** Type returned by e.g. $(...).find() */
+export type CheerioElement = ReturnType<CheerioAnyNode["find"]>;
 
 /** Convenience function that combines readFile and load. */
 export const loadFromFile = async (

Diff for: 11ty/common.ts

@@ -2,7 +2,7 @@
 
 import { AxiosError, type AxiosResponse } from "axios";
 
-import type { Guideline, Principle, SuccessCriterion } from "./guidelines";
+import type { WcagItem } from "./guidelines";
 
 /** Generates an ID for heading permalinks. Equivalent to wcag:generate-id in base.xslt. */
 export function generateId(title: string) {
@@ -18,8 +18,8 @@ export const resolveDecimalVersion = (version: `${number}`) => version.split("")
 
 /** Sort function for ordering WCAG principle/guideline/SC numbers ascending */
 export function wcagSort(
-  a: Principle | Guideline | SuccessCriterion,
-  b: Principle | Guideline | SuccessCriterion
+  a: WcagItem,
+  b: WcagItem
 ) {
   const aParts = a.num.split(".").map((n) => +n);
   const bParts = b.num.split(".").map((n) => +n);

Diff for: 11ty/cp-cvs.ts

@@ -52,6 +52,10 @@ for (const [srcDir, destDir] of Object.entries(dirs)) {
   }
 }
 
+try {
+  await copyFile(join(outputBase, "wcag.json"), join(wcagBase, "wcag.json"));
+} catch (error) {}
+
 await mkdirp(join(wcagBase, "errata"));
 await copyFile(
   join(outputBase, "errata", `${wcagVersion}.html`),

Diff for: 11ty/guidelines.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 import type { CheerioAPI } from "cheerio";
 import { glob } from "glob";
+import pick from "lodash-es/pick";
 
 import { readFile } from "fs/promises";
 import { basename, join } from "path";
@@ -34,22 +35,28 @@ export const actRules = (
   JSON.parse(await readFile("guidelines/act-mapping.json", "utf8")) as ActMapping
 )["act-rules"];
 
+/** Version-dependent overrides of SC shortcodes for older versions */
+export const scSlugOverrides: Record<string, (version: WcagVersion) => string> = {
+  "target-size-enhanced": (version) => (version < "22" ? "target-size" : "target-size-enhanced"),
+};
+
 /**
  * Flattened object hash, mapping each WCAG 2 SC slug to the earliest WCAG version it applies to.
  * (Functionally equivalent to "guidelines-versions" target in build.xml; structurally inverted)
  */
-const scVersions = await (async function () {
+async function resolveScVersions(version: WcagVersion) {
   const paths = await glob("*/*.html", { cwd: "understanding" });
   const map: Record<string, WcagVersion> = {};
 
   for (const path of paths) {
     const [fileVersion, filename] = path.split("/");
     assertIsWcagVersion(fileVersion);
-    map[basename(filename, ".html")] = fileVersion;
+    const slug = basename(filename, ".html");
+    map[slug in scSlugOverrides ? scSlugOverrides[slug](version) : slug] = fileVersion;
   }
 
   return map;
-})();
+}
 
 export interface DocNode {
   id: string;
@@ -83,15 +90,12 @@ export interface SuccessCriterion extends DocNode {
   type: "SC";
 }
 
+export type WcagItem = Principle | Guideline | SuccessCriterion;
+
 export function isSuccessCriterion(criterion: any): criterion is SuccessCriterion {
   return !!(criterion?.type === "SC" && "level" in criterion);
 }
 
-/** Version-dependent overrides of SC shortcodes for older versions */
-export const scSlugOverrides: Record<string, (version: WcagVersion) => string> = {
-  "target-size-enhanced": (version) => (version < "22" ? "target-size" : "target-size-enhanced"),
-};
-
 /** Selectors ignored when capturing content of each Principle / Guideline / SC */
 const contentIgnores = [
   "h1, h2, h3, h4, h5, h6",
@@ -115,7 +119,12 @@ const getContentHtml = ($el: CheerioAnyNode) => {
 };
 
 /** Performs processing common across WCAG versions */
-function processPrinciples($: CheerioAPI) {
+async function processPrinciples($: CheerioAPI) {
+  // Auto-detect version from end of title
+  const version = $("title").text().trim().split(" ").pop()!.replace(".", "");
+  assertIsWcagVersion(version);
+  const scVersions = await resolveScVersions(version);
+
   const principles: Principle[] = [];
   $(".principle").each((i, el) => {
     const guidelines: Guideline[] = [];
@@ -175,7 +184,7 @@ export const getPrinciples = async (path = "guidelines/index.html") =>
  * Returns a flattened object hash, mapping shortcodes to each principle/guideline/SC.
  */
 export function getFlatGuidelines(principles: Principle[]) {
-  const map: Record<string, Principle | Guideline | SuccessCriterion> = {};
+  const map: Record<string, WcagItem> = {};
   for (const principle of principles) {
     map[principle.id] = principle;
     for (const guideline of principle.guidelines) {
@@ -199,7 +208,7 @@ interface Term {
 }
 export type TermsMap = Record<string, Term>;
 
-function processTermsMap($: CheerioAPI) {
+function processTermsMap($: CheerioAPI, includeSynonyms = true) {
   const terms: TermsMap = {};
 
   $("dfn").each((_, el) => {
@@ -213,12 +222,16 @@ function processTermsMap($: CheerioAPI) {
       trId: el.attribs.id,
     };
 
-    // Include both original and all-lowercase version to simplify lookups
-    // (since most synonyms are lowercase) while preserving case in name
-    const names = [term.name, term.name.toLowerCase()].concat(
-      (el.attribs["data-lt"] || "").toLowerCase().split("|")
-    );
-    for (const name of names) terms[name] = term;
+    if (includeSynonyms) {
+      // Include both original and all-lowercase version to simplify lookups
+      // (since most synonyms are lowercase) while preserving case in name
+      const names = [term.name, term.name.toLowerCase()].concat(
+        (el.attribs["data-lt"] || "").toLowerCase().split("|")
+      );
+      for (const name of names) terms[name] = term;
+    } else {
+      terms[term.name] = term;
+    }
   });
 
   return terms;
@@ -230,7 +243,7 @@ function processTermsMap($: CheerioAPI) {
  * comparable to the term elements in wcag.xml from the guidelines-xml Ant task.
  */
 export const getTermsMap = async (path = "guidelines/index.html") =>
-  processTermsMap(await flattenDomFromFile(path));
+  processTermsMap(await flattenDomFromFile(path), true);
 
 // Version-specific APIs
 
@@ -245,6 +258,11 @@ const loadRemoteGuidelines = async (version: WcagVersion, stripRespec = true) =>
     ).data);
 
   const $ = load(html);
+
+  // Remove extra markup from headings, regardless of stripRespec setting,
+  // so that names parse consistently
+  $("bdi").remove();
+
   if (!stripRespec) return $;
 
   // Re-collapse definition links and notes, to be processed by this build system
@@ -271,9 +289,6 @@ const loadRemoteGuidelines = async (version: WcagVersion, stripRespec = true) =>
   $(".example[id]").removeAttr("id");
   $(".example > .marker").remove();
 
-  // Remove extra markup from headings so they can be parsed for names
-  $("bdi").remove();
-
   // Remove abbr elements which exist only in TR, not in informative docs
   $("#acknowledgements li abbr, #glossary abbr").each((_, abbrEl) => {
     $(abbrEl).replaceWith($(abbrEl).text());
@@ -300,15 +315,30 @@ export const getAcknowledgementsForVersion = async (version: WcagVersion) => {
 /**
  * Retrieves and processes a pinned WCAG version using published guidelines.
  */
-export const getPrinciplesForVersion = async (version: WcagVersion) =>
-  processPrinciples(await loadRemoteGuidelines(version));
+export const getPrinciplesForVersion = async (version: WcagVersion, stripRespec?: boolean) =>
+  processPrinciples(await loadRemoteGuidelines(version, stripRespec));
+
+interface GetTermsMapForVersionOptions {
+  /**
+   * Whether to populate additional map keys based on synonyms defined in data-lt;
+   * this should typically be true for informative docs, but may be false for other purposes
+   */
+  includeSynonyms?: boolean;
+  /**
+   * Whether to strip respec-generated content and attributes;
+   * this should typically be true for informative docs, but may be false for other purposes
+   */
+  stripRespec?: boolean;
+}
 
 /**
  * Resolves term definitions from a WCAG 2.x publication,
  * organized for lookup by name.
  */
-export const getTermsMapForVersion = async (version: WcagVersion) =>
-  processTermsMap(await loadRemoteGuidelines(version));
+export const getTermsMapForVersion = async (
+  version: WcagVersion,
+  { includeSynonyms, stripRespec }: GetTermsMapForVersionOptions = {}
+) => processTermsMap(await loadRemoteGuidelines(version, stripRespec), includeSynonyms);
 
 /** Parses errata items from the errata document for the specified WCAG version. */
 export const getErrataForVersion = async (version: WcagVersion) => {