feat(nx-dev): improve link text for ai docs

nrwl · Aug 31, 2023 · 68684de · 68684de
1 parent 1abe35c
commit 68684de
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 9 deletions.
diff --git a/nx-dev/util-ai/src/lib/chat-utils.ts b/nx-dev/util-ai/src/lib/chat-utils.ts
@@ -54,7 +54,7 @@ export function getMessageFromResponse(
 
 export function getListOfSources(
   pageSections: PageSection[]
-): { heading: string; url: string }[] {
+): { heading: string; url: string; longer_heading: string }[] {
   const uniqueUrlPartials = new Set<string | null>();
   const result = pageSections
     .filter((section) => {
@@ -72,6 +72,7 @@ export function getListOfSources(
       }
       return {
         heading: section.heading,
+        longer_heading: section.longer_heading,
         url: url.toString(),
       };
     });
@@ -90,11 +91,40 @@ ${sourcesMarkdown}
 }
 
 export function toMarkdownList(
-  sections: { heading: string; url: string }[]
+  sections: { heading: string; url: string; longer_heading: string }[]
 ): string {
-  return sections
+  const sectionsWithLongerHeadings: {
+    heading: string;
+    url: string;
+    longer_heading: string;
+  }[] = [];
+
+  const headings = new Set<string>();
+  const sectionsWithUniqueHeadings = sections.filter((section) => {
+    if (headings.has(section.heading)) {
+      sectionsWithLongerHeadings.push(section);
+      return false;
+    } else {
+      headings.add(section.heading);
+      return true;
+    }
+  });
+
+  const finalSections = sectionsWithUniqueHeadings
     .map((section) => `- [${section.heading}](${section.url})`)
-    .join('\n');
+    .join('\n')
+    .concat('\n')
+    .concat(
+      sectionsWithLongerHeadings
+        .map(
+          (section, index) =>
+            `- [${
+              section.longer_heading ?? section.heading + ' ' + (index + 1)
+            }](${section.url})`
+        )
+        .join('\n')
+    );
+  return finalSections;
 }
 
 export function extractLinksFromSourcesSection(markdown: string): string[] {

diff --git a/nx-dev/util-ai/src/lib/utils.ts b/nx-dev/util-ai/src/lib/utils.ts
@@ -65,6 +65,7 @@ export interface PageSection {
   page_id: number;
   content: string;
   heading: string;
+  longer_heading: string;
   similarity: number;
   slug: string;
   url_partial: string | null;

diff --git a/tools/documentation/create-embeddings/src/main.mts b/tools/documentation/create-embeddings/src/main.mts
@@ -155,17 +155,17 @@ async function generateEmbeddings() {
     type: 'boolean',
   }).argv;
 
-  const shouldRefresh = argv.refresh;
+  const shouldRefresh = argv.refresh ?? true;
 
   if (!process.env.NX_NEXT_PUBLIC_SUPABASE_URL) {
     throw new Error(
       'Environment variable NX_NEXT_PUBLIC_SUPABASE_URL is required: skipping embeddings generation'
     );
   }
 
-  if (!process.env.NX_SUPABASE_SERVICE_ROLE_KEY) {
+  if (!process.env.NX_SUPABASE_SERVICE_ROLE_KEY_ACTUAL) {
     throw new Error(
-      'Environment variable NX_SUPABASE_SERVICE_ROLE_KEY is required: skipping embeddings generation'
+      'Environment variable NX_SUPABASE_SERVICE_ROLE_KEY_ACTUAL is required: skipping embeddings generation'
     );
   }
 
@@ -177,7 +177,7 @@ async function generateEmbeddings() {
 
   const supabaseClient = createClient(
     process.env.NX_NEXT_PUBLIC_SUPABASE_URL,
-    process.env.NX_SUPABASE_SERVICE_ROLE_KEY,
+    process.env.NX_SUPABASE_SERVICE_ROLE_KEY_ACTUAL,
     {
       auth: {
         persistSession: false,
@@ -307,13 +307,19 @@ async function generateEmbeddings() {
 
           const [responseData] = embeddingResponse.data;
 
+          const longer_heading = createLongerHeading(heading, url_partial);
+
           const { error: insertPageSectionError, data: pageSection } =
             await supabaseClient
               .from('nods_page_section')
               .insert({
                 page_id: page.id,
                 slug,
-                heading,
+                heading:
+                  heading?.length && heading !== null && heading !== 'null'
+                    ? heading
+                    : longer_heading,
+                longer_heading,
                 content,
                 url_partial,
                 token_count: embeddingResponse.usage.total_tokens,
@@ -433,6 +439,29 @@ function getAllFilesWithItemList(data): WalkEntry[] {
   return files;
 }
 
+function createLongerHeading(
+  heading?: string | null,
+  url_partial?: string
+): string | undefined {
+  if (url_partial?.length) {
+    if (heading?.length && heading !== null && heading !== 'null') {
+      return `${heading}${` - ${
+        url_partial.split('/')?.[1]?.[0].toUpperCase() +
+        url_partial.split('/')?.[1]?.slice(1)
+      }`}`;
+    } else {
+      return url_partial
+        .split('#')[0]
+        .split('/')
+        .map((part) =>
+          part?.length ? part[0].toUpperCase() + part.slice(1) + ' - ' : ''
+        )
+        .join('')
+        .slice(0, -3);
+    }
+  }
+}
+
 async function main() {
   await generateEmbeddings();
 }