Skip to content

Commit

Permalink
馃帀 Add intersphinx and markdown links myst:
Browse files Browse the repository at this point in the history
  • Loading branch information
rowanc1 committed Sep 7, 2022
1 parent 0cef659 commit 5460169
Show file tree
Hide file tree
Showing 7 changed files with 422 additions and 127 deletions.
6 changes: 6 additions & 0 deletions .changeset/weak-geese-hope.md
@@ -0,0 +1,6 @@
---
'curvenote': patch
'myst-transforms': patch
---

Add intersphinx interoperability (read) as well as markdown links syntax for referencing.
86 changes: 65 additions & 21 deletions apps/cli/src/store/local/actions.ts
Expand Up @@ -10,14 +10,14 @@ import {
basicTransformationsPlugin,
htmlPlugin,
footnotesPlugin,
keysPlugin,
ReferenceState,
MultiPageReferenceState,
resolveReferencesTransform,
mathPlugin,
codePlugin,
enumerateTargetsPlugin,
getFrontmatter,
keysTransform,
} from 'myst-transforms';
import { dirname, extname, join } from 'path';
import chalk from 'chalk';
Expand Down Expand Up @@ -61,11 +61,13 @@ import { processNotebook } from './notebook';
import { watch } from './reducers';
import { warnings } from '../build';
import { selectFileWarnings } from '../build/selectors';
import { Inventory } from './intersphinx';

type ISessionWithCache = ISession & {
$citationRenderers: Record<string, CitationRenderer>; // keyed on path
$doiRenderers: Record<string, SingleCitationRenderer>; // keyed on doi
$references: Record<string, ReferenceState>; // keyed on path
$intersphinx: Record<string, Inventory>; // keyed on id
$mdast: Record<string, { pre: PreRendererData; post?: RendererData }>; // keyed on path
};

Expand All @@ -90,6 +92,7 @@ function castSession(session: ISession): ISessionWithCache {
if (!cache.$doiRenderers) cache.$doiRenderers = {};
if (!cache.$references) cache.$references = {};
if (!cache.$mdast) cache.$mdast = {};
if (!cache.$intersphinx) cache.$intersphinx = {};
return cache;
}

Expand Down Expand Up @@ -123,6 +126,45 @@ async function loadCitations(session: ISession, path: string) {
return renderer;
}

async function loadInterspinx(
session: ISession,
opts: { projectPath: string; force?: boolean },
): Promise<Inventory[]> {
const projectConfig = selectors.selectProjectConfig(session.store.getState(), opts.projectPath);
const cache = castSession(session);
if (!projectConfig?.intersphinx) return [];
const intersphinx = Object.entries(projectConfig.intersphinx)
.filter(([key, object]) => {
if (isUrl(object.url)) return true;
session.log.error(`鈿狅笍 ${key} intersphinx is not a valid url: "${object.url}"`);
return false;
})
.map(([key, object]) => {
if (!cache.$intersphinx[key] || opts.force) {
cache.$intersphinx[key] = new Inventory({ name: key, path: object.url });
}
return cache.$intersphinx[key];
})
.filter((exists) => !!exists);
await Promise.all(
intersphinx.map(async (loader) => {
if (loader._loaded) return;
const toc = tic();
try {
await loader.load();
} catch (error) {
session.log.debug(`\n\n${(error as Error)?.stack}\n\n`);
session.log.error(`Problem fetching intersphinx entry: ${loader.name} (${loader.path})`);
return null;
}
session.log.info(
toc(`馃彨 Read ${loader.numEntries} intersphinx links for "${loader.name}" in %s.`),
);
}),
);
return intersphinx;
}

function combineCitationRenderers(cache: ISessionWithCache, ...files: string[]) {
const combined: CitationRenderer = {};
files.forEach((file) => {
Expand Down Expand Up @@ -282,7 +324,6 @@ export async function transformMdast(
await unified()
.use(codePlugin, { lang: frontmatter?.kernelspec?.language })
.use(footnotesPlugin, { references }) // Needs to happen nead the end
.use(keysPlugin) // Keys should be the last major transform
.run(mdast, vfile);
await transformImages(session, file, mdast, { localExport });
// Note, the thumbnail transform must be **after** images, as it may read the images
Expand Down Expand Up @@ -327,12 +368,12 @@ export async function postProcessMdast(
session: ISession,
{
file,
strict,
projectPath,
checkLinks,
pageReferenceStates,
}: {
file: string;
strict?: boolean;
projectPath: string;
checkLinks?: boolean;
pageReferenceStates: PageReferenceStates;
},
Expand All @@ -341,14 +382,17 @@ export async function postProcessMdast(
const { log } = session;
const cache = castSession(session);
const mdastPost = selectFile(session, file);
const intersphinx = await loadInterspinx(session, { projectPath });
// NOTE: This is doing things in place, we should potentially make this a different state?
const linkLookup = transformLinks(session, file, mdastPost.mdast, { checkLinks });
await transformLinks(session, file, mdastPost.mdast, {
checkLinks,
intersphinx,
});
const state = cache.$references[file];
const projectState = new MultiPageReferenceState(pageReferenceStates, file);
resolveReferencesTransform(mdastPost.mdast, { state: projectState });
if (strict || checkLinks) {
await linkLookup;
}
// Ensure there are keys on every node
keysTransform(mdastPost.mdast);
logMessagesFromVFile(session, state.file);
log.debug(toc(`Transformed mdast cross references and links for "${file}" in %s`));
}
Expand All @@ -361,7 +405,7 @@ export function selectFile(session: ISession, file: string) {
return mdastPost;
}

export async function writeFile(
export function writeFile(
session: ISession,
{ file, pageSlug, projectSlug }: { file: string; projectSlug: string; pageSlug: string },
) {
Expand Down Expand Up @@ -418,8 +462,8 @@ export async function fastProcessFile(
await transformMdast(session, { file, projectPath, projectSlug, pageSlug, watchMode: true });
const { pages } = loadProject(session, projectPath);
const pageReferenceStates = selectPageReferenceStates(session, pages);
await postProcessMdast(session, { file, pageReferenceStates });
await writeFile(session, { file, pageSlug, projectSlug });
await postProcessMdast(session, { file, projectPath, pageReferenceStates });
writeFile(session, { file, pageSlug, projectSlug });
session.log.info(toc(`馃摉 Built ${file} in %s.`));
await writeSiteManifest(session);
}
Expand All @@ -439,6 +483,8 @@ export async function processProject(
...project.bibliography.map((path) => loadFile(session, path, '.bib')),
// Load all content (.md and .ipynb)
...pages.map((page) => loadFile(session, page.file)),
// Load up all the intersphinx references
loadInterspinx(session, { projectPath: siteProject.path }) as Promise<any>,
]);
}
// Consolidate all citations onto single project citation renderer
Expand All @@ -461,22 +507,20 @@ export async function processProject(
pages.map((page) =>
postProcessMdast(session, {
file: page.file,
strict: opts?.strict,
checkLinks: opts?.checkLinks,
projectPath: project.path,
checkLinks: opts?.checkLinks || opts?.strict,
pageReferenceStates,
}),
),
);
// Write all pages
if (writeFiles) {
await Promise.all(
pages.map((page) =>
writeFile(session, {
file: page.file,
projectSlug: siteProject.slug,
pageSlug: page.slug,
}),
),
pages.map((page) =>
writeFile(session, {
file: page.file,
projectSlug: siteProject.slug,
pageSlug: page.slug,
}),
);
}
log.info(toc(`馃摎 Built ${pages.length} pages for ${siteProject.slug} in %s.`));
Expand Down
135 changes: 135 additions & 0 deletions apps/cli/src/store/local/intersphinx.ts
@@ -0,0 +1,135 @@
import fs from 'fs';
import zlib from 'zlib';
import fetch from 'node-fetch';
import { isUrl } from '../../utils';

export type InventoryItem = { location: string; display?: string };
export type InventoryData = Record<string, Record<string, InventoryItem>>;

const DEFAULT_INV_NAME = 'objects.inv';

export class Inventory {
path: string;

/**
* Place to look for the inv file, by default it is "objects.inv"
*/
invName = DEFAULT_INV_NAME;
name: string;
project?: string;
version?: string;

numEntries?: number;
data: InventoryData;

constructor(opts: { name: string; path: string; invName?: string }) {
this.name = opts.name;
if (isUrl(opts.path)) {
this.path = opts.path.replace(/\/$/, ''); // Remove any trailing slash
this.invName = opts.invName || DEFAULT_INV_NAME;
} else {
this.path = opts.path; // Local path
}
this.data = {};
}

write(path: string) {
const header = [
`# Sphinx inventory version 2`,
`# Project: ${this.project}`,
`# Version: ${this.version}`,
`# The remainder of this file is compressed using zlib.`,
].join('\n');

// https://github.com/sphinx-doc/sphinx/blob/5e9550c78e3421dd7dcab037021d996841178f67/sphinx/util/inventory.py#L154
const data = zlib.deflateSync(
[
'start:install std:label -1 start/overview.html#start-install Install Jupyter Book\n', // needs trailing new line
'my-fig-ref std:label -1 content/references.html#$ My figure title.\n', // needs trailing new line
].join(''),
);
fs.writeFileSync(path, `${header}\r\n${data.toString('binary')}`, { encoding: 'binary' });
}

_loaded = false;

async load() {
if (this._loaded) return;
let buffer: Buffer;
if (isUrl(this.path)) {
const url = `${this.path}/${this.invName || DEFAULT_INV_NAME}`;
const res = await fetch(url);
if (!res.ok) {
throw new Error(
`Error fetching intersphinx from "${url}": ${res.status} ${res.statusText}`,
);
}
buffer = await res.buffer();
} else {
buffer = fs.readFileSync(this.path);
}
const str = buffer.toString('binary');
const [header, projectInfo, versionInfo, zlibInfo, ...rest] = str.split('\n');

if (!header.includes('version 2')) {
throw new Error('Can only read version 2 inv files');
}
if (!zlibInfo.includes('compressed using zlib')) {
throw new Error('Last line of header must include: "compressed using zlib"');
}
this.project = projectInfo.slice(11).trim();
this.version = versionInfo.slice(11).trim();
const compressed = Buffer.from(rest.join('\n'), 'binary');
const re = zlib.inflateSync(compressed);

let entries = 0;
re.toString()
.split('\n')
.forEach((s) => {
const pattern = /(.+?)\s+(\S+)\s+(-?\d+)\s+?(\S*)\s+(.*)/;
const match = s.match(pattern);
if (!match) return;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [, name, type, priority, location, display] = match;
if (!type.includes(':')) {
// wrong type value. type should be in the form of "{domain}:{objtype}"
return;
}
if (type === 'py:module' && this.getEntry({ type, name })) {
// due to a bug in 1.1 and below,
// two inventory entries are created
// for Python modules, and the first
// one is correct
return;
}
entries++;
this.setEntry(type, name, location, display);
});
this.numEntries = entries;
this._loaded = true;
}

setEntry(type: string, name: string, location: string, display: string) {
if (!this.data[type]) this.data[type] = {};
let resolvedLocation = location;
if (location.endsWith('$')) {
resolvedLocation = location.slice(0, -1) + name.toLowerCase().replace(/\s+/g, '-');
}
const resolvedDisplay = display.trim() === '-' ? undefined : display.trim();
this.data[type][name] = { location: resolvedLocation, display: resolvedDisplay };
}

getEntry(opts: {
type?: string;
name: string;
}): { location: string; display?: string } | undefined {
if (!opts.type) {
const type = Object.keys(this.data).find((t) => !!this.data[t][opts.name]);
if (!type) return undefined;
return this.getEntry({ type, name: opts.name });
}
const entry = this.data[opts.type]?.[opts.name];
if (!entry) return undefined;
return { location: `${this.path}/${entry.location}`, display: entry.display };
}
}

0 comments on commit 5460169

Please sign in to comment.