Add improvements to llmstxt.ts to generate links with pages of different langs

GregHolmes · GregHolmes · commit 99f84e56ee2b · 2025-08-14T16:13:01.000+01:00
diff --git a/bin/validate-llms.txt.ts b/bin/validate-llms.txt.ts
@@ -25,29 +25,60 @@ function countHtmlFiles(): number {
   return contentFiles.length;
 }
 
-function countLlmsLines(): number {
+function countLlmsPages(): { totalLines: number; uniquePages: number } {
   const content = fs.readFileSync(LLMS_FILE, 'utf-8');
-  return content.split('\n').filter((line) => line.trim().length > 0).length;
+  const lines = content.split('\n').filter((line) => line.trim().length > 0);
+
+  // Extract unique page URLs (without ?lang= parameters)
+  const uniqueUrls = new Set<string>();
+
+  lines.forEach((line) => {
+    // Skip the header line
+    if (line.startsWith('#')) {
+      return;
+    }
+
+    // Extract URL from markdown link format: - [Title](URL): Description
+    const urlMatch = line.match(/\[.*?\]\((.*?)\)/);
+    if (urlMatch) {
+      const url = urlMatch[1];
+      // Remove lang parameter to get base URL
+      const baseUrl = url.split('?')[0];
+      uniqueUrls.add(baseUrl);
+    }
+  });
+
+  return {
+    totalLines: lines.length,
+    uniquePages: uniqueUrls.size,
+  };
 }
 
-function validateCounts(htmlCount: number, llmsCount: number): boolean {
-  const difference = Math.abs(htmlCount - llmsCount);
-  const maxAllowedDifference = Math.ceil(htmlCount * 0.8); // 80% of html count
-  return difference <= maxAllowedDifference;
+function validateCounts(htmlCount: number, uniquePageCount: number, totalLineCount: number): boolean {
+  // With language-specific URLs, we expect to have a reasonable coverage of the HTML files
+  // The unique page count should be at least 50% of the HTML count (allowing for pages not captured by our GraphQL queries)
+  // and at most 100% (we shouldn't have more unique pages than HTML files)
+  const coverage = uniquePageCount / htmlCount;
+  const minCoverage = 0.5; // At least 50% coverage
+  const maxCoverage = 1.0; // At most 100% coverage
+
+  return coverage >= minCoverage && coverage <= maxCoverage;
 }
 
 function main() {
   try {
     const htmlCount = countHtmlFiles();
-    const llmsCount = countLlmsLines();
-    const isValid = validateCounts(htmlCount, llmsCount);
+    const { totalLines, uniquePages } = countLlmsPages();
+    const isValid = validateCounts(htmlCount, uniquePages, totalLines);
 
     console.log(`HTML files found: ${htmlCount}`);
-    console.log(`Lines in llms.txt: ${llmsCount}`);
+    console.log(`Total lines in llms.txt: ${totalLines}`);
+    console.log(`Unique pages in llms.txt: ${uniquePages}`);
+    console.log(`Coverage: ${Math.round((uniquePages / htmlCount) * 100)}%`);
 
     if (!isValid) {
       console.error(
-        `Error: The number of lines in llms.txt (${llmsCount}) is not within 80% of the number of HTML files (${htmlCount})`,
+        `Error: The coverage of unique pages in llms.txt (${Math.round((uniquePages / htmlCount) * 100)}%) is not within the acceptable range of 50-100% of HTML files (${htmlCount})`,
       );
       process.exit(1);
     }
diff --git a/data/onPostBuild/llmstxt.ts b/data/onPostBuild/llmstxt.ts
@@ -1,6 +1,7 @@
 import { GatsbyNode } from 'gatsby';
 import * as path from 'path';
 import * as fs from 'fs';
+import languageInfo from '../../src/data/languages/languageInfo';
 
 /**
  * This script is used to create a file called llms.txt that contains a list of all the pages in the site.
@@ -11,6 +12,26 @@ const LLMS_TXT_PREAMBLE = `# https://ably.com/docs llms.txt\n`;
 
 const REPORTER_PREFIX = 'onPostBuild:';
 
+// Valid languages for URL generation (matching your requirements)
+const VALID_LANGUAGES = [
+  'javascript',
+  'nodejs',
+  'csharp',
+  'flutter',
+  'java',
+  'objc',
+  'php',
+  'python',
+  'ruby',
+  'swift',
+  'go',
+];
+
+// Function to get the display label for a language
+const getLanguageLabel = (languageKey: string): string => {
+  return languageInfo[languageKey as keyof typeof languageInfo]?.label || languageKey;
+};
+
 interface DocumentQueryResult {
   site: {
     siteMetadata: {
@@ -24,6 +45,7 @@ interface DocumentQueryResult {
         meta: {
           title: string;
           meta_description: string;
+          languages?: string[];
         };
       };
     }[];
@@ -38,6 +60,9 @@ interface DocumentQueryResult {
         title?: string;
         meta_description?: string;
       };
+      internal: {
+        contentFilePath?: string;
+      };
     }[];
   };
 }
@@ -53,6 +78,34 @@ const escapeMarkdown = (text: string) => {
   return text.replace(/([\\`*_{}[\]()#+!])/g, '\\$1');
 };
 
+// Function to extract code element classes from an MDX file
+const extractCodeLanguages = async (filePath: string): Promise<Set<string>> => {
+  try {
+    // Check if the file exists
+    if (!fs.existsSync(filePath)) {
+      return new Set();
+    }
+
+    // Read the file content
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+
+    // Find all instances of code blocks with language specifiers (```language)
+    const codeBlockRegex = /```(\w+)/g;
+    let match;
+    const languages = new Set<string>();
+
+    while ((match = codeBlockRegex.exec(fileContent)) !== null) {
+      if (match[1] && match[1].trim()) {
+        languages.add(match[1].trim());
+      }
+    }
+    return languages;
+  } catch (error) {
+    console.error(`Error extracting code element classes from ${filePath}:`, error);
+    return new Set();
+  }
+};
+
 export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql, reporter, basePath }) => {
   const query = `
     query {
@@ -62,13 +115,14 @@ export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql, reporter
         }
       }
 
-      allFileHtml(filter: { articleType: { in: ["document", "apiReference"] } }) {
+      allFileHtml {
         edges {
           node {
             slug
             meta {
               title
               meta_description
+              languages
             }
           }
         }
@@ -86,6 +140,9 @@ export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql, reporter
             title
             meta_description
           }
+          internal {
+            contentFilePath
+          }
         }
       }
     }
@@ -109,30 +166,50 @@ export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql, reporter
     throw new Error('Site URL not found.');
   }
 
-  // Process textile-based pages (allFileHtml)
-  const textilePages = queryRecords.allFileHtml.edges.map((edge) => edge.node);
-
-  // Process MDX pages (allMdx)
-  const mdxPages = queryRecords.allMdx.nodes
-    .filter((node) => {
-      // Only include pages from docs directory that have the required frontmatter
-      return (
-        node.parent.relativeDirectory.startsWith('docs') &&
-        node.frontmatter?.title &&
-        node.frontmatter?.meta_description
-      );
-    })
-    .map((node) => ({
-      // Create slug from parent file info - remove 'docs/' prefix since it's already in relativeDirectory
-      slug: (node.parent.relativeDirectory + (node.parent.name === 'index' ? '' : `/${node.parent.name}`)).replace(
-        /^docs\//,
-        '',
-      ),
-      meta: {
-        title: node.frontmatter.title!,
-        meta_description: node.frontmatter.meta_description!,
-      },
-    }));
+  // Process textile-based pages (allFileHtml) and extract languages
+  const textilePages = queryRecords.allFileHtml.edges.map((edge) => {
+    // Extract valid languages from the meta.languages field
+    const metaLanguages = edge.node.meta.languages || [];
+    const languages = metaLanguages.filter((lang) => VALID_LANGUAGES.includes(lang));
+
+    return {
+      ...edge.node,
+      languages,
+    };
+  });
+
+  // Process MDX pages (allMdx) and extract languages from files
+  const mdxPages = await Promise.all(
+    queryRecords.allMdx.nodes
+      .filter((node) => {
+        // Only include pages from docs directory that have the required frontmatter
+        return (
+          node.parent.relativeDirectory.startsWith('docs') &&
+          node.frontmatter?.title &&
+          node.frontmatter?.meta_description
+        );
+      })
+      .map(async (node) => {
+        // Create slug from parent file info - remove 'docs/' prefix since it's already in relativeDirectory
+        const slug = (
+          node.parent.relativeDirectory + (node.parent.name === 'index' ? '' : `/${node.parent.name}`)
+        ).replace(/^docs\//, '');
+
+        // Extract valid languages from the file content
+        const filePath = node.internal.contentFilePath || '';
+        const detectedLanguages = await extractCodeLanguages(filePath);
+        const languages = Array.from(detectedLanguages).filter((lang) => VALID_LANGUAGES.includes(lang));
+
+        return {
+          slug,
+          meta: {
+            title: node.frontmatter.title!,
+            meta_description: node.frontmatter.meta_description!,
+          },
+          languages,
+        };
+      }),
+  );
 
   const allPages = [...textilePages, ...mdxPages];
 
@@ -143,15 +220,27 @@ export const onPostBuild: GatsbyNode['onPostBuild'] = async ({ graphql, reporter
   const serializedPages = [LLMS_TXT_PREAMBLE];
 
   for (const page of allPages) {
-    const { slug, meta } = page;
+    const { slug, meta, languages } = page;
     const { title, meta_description } = meta;
 
     try {
-      const url = prefixPath({ url: `/docs/${slug}`, siteUrl, pathPrefix: basePath });
+      const baseUrl = prefixPath({ url: `/docs/${slug}`, siteUrl, pathPrefix: basePath });
       const safeTitle = escapeMarkdown(title);
-      const link = `[${safeTitle}](${url})`;
-      const line = `- ${[link, meta_description].join(': ')}`;
-      serializedPages.push(line);
+
+      // Generate base page entry (without language parameter)
+      const baseLink = `[${safeTitle}](${baseUrl})`;
+      const baseLine = `- ${[baseLink, meta_description].join(': ')}`;
+      serializedPages.push(baseLine);
+
+      // Generate language-specific entries if the page has languages
+      if (languages && languages.length > 0) {
+        for (const language of languages) {
+          const langUrl = `${baseUrl}?lang=${language}`;
+          const langLink = `[${safeTitle} (${getLanguageLabel(language)})](${langUrl})`;
+          const langLine = `- ${[langLink, meta_description].join(': ')}`;
+          serializedPages.push(langLine);
+        }
+      }
     } catch (err) {
       reporter.panic(`${REPORTER_PREFIX} Error serializing pages`, err as Error);
     }
