Skip to content

Commit

Permalink
Add scaladoc feature for preprocessing markdown files and publishing …
Browse files Browse the repository at this point in the history
…docs.scala-lang
  • Loading branch information
BarkingBad committed Aug 11, 2021
1 parent 3109226 commit 8ec44e0
Show file tree
Hide file tree
Showing 12 changed files with 312 additions and 145 deletions.
17 changes: 14 additions & 3 deletions project/Build.scala
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import sbtbuildinfo.BuildInfoPlugin.autoImport._
import scala.util.Properties.isJavaAtLeast

import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._
import org.scalajs.linker.interface.ModuleInitializer

object DottyJSPlugin extends AutoPlugin {
import Build._
Expand Down Expand Up @@ -329,7 +330,6 @@ object Build {
".*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/",
"-skip-by-regex:.+\\.internal($|\\..+)",
"-skip-by-regex:.+\\.impl($|\\..+)",
"-format", "md",
"-project-logo", "docs/logo.svg",
"-social-links:" +
"github::https://github.com/lampepfl/dotty," +
Expand Down Expand Up @@ -1245,6 +1245,7 @@ object Build {
// Note: the two tasks below should be one, but a bug in Tasty prevents that
val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
val generateTestcasesDocumentation = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
val copyTheScaladocJsOutput = inputKey[Unit]("Copy the output of the scaladoc js files")

lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")).
dependsOn(`scala3-compiler-bootstrapped`).
Expand All @@ -1253,8 +1254,12 @@ object Build {
enablePlugins(DottyJSPlugin).
dependsOn(`scala3-library-bootstrappedJS`).
settings(
Compile / scalaJSMainModuleInitializer := (sys.env.get("scaladoc.projectFormat") match {
case Some("md") => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "markdownMain"))
case _ => Some(ModuleInitializer.mainMethod("dotty.tools.scaladoc.Main", "main"))
}),
Test / fork := false,
scalaJSUseMainModuleInitializer := true,
Compile / scalaJSUseMainModuleInitializer := true,
libraryDependencies += ("org.scala-js" %%% "scalajs-dom" % "1.1.0").cross(CrossVersion.for3Use2_13)
)

Expand Down Expand Up @@ -1285,7 +1290,6 @@ object Build {
scalaSrcLink(stdLibVersion, srcManaged(dottyNonBootstrappedVersion, "scala") + "="),
dottySrcLink(referenceVersion, srcManaged(dottyNonBootstrappedVersion, "dotty") + "=", "#library/src"),
dottySrcLink(referenceVersion),
"-Ygenerate-inkuire",
) ++ scalacOptionsDocSettings ++ revision ++ params ++ targets
import _root_.scala.sys.process._
val escapedCmd = cmd.map(arg => if(arg.contains(" ")) s""""$arg"""" else arg)
Expand Down Expand Up @@ -1398,6 +1402,13 @@ object Build {
)
}.value,

copyTheScaladocJsOutput := Def.inputTask {
val extraArgs = spaceDelimited("<arg>").parsed
val dest = extraArgs.lift(0).getOrElse("output")
val jsDestinationFile: File = Paths.get(dest).toFile
sbt.IO.copyFile((`scaladoc-js` / Compile / fullOptJS).value.data, jsDestinationFile)
}.evaluated,

Test / buildInfoKeys := Seq[BuildInfoKey](
(Test / Build.testcasesOutputDir),
(Test / Build.testcasesSourceRoot),
Expand Down
31 changes: 31 additions & 0 deletions project/scripts/genDocsScalaLang
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/usr/bin/env bash

set -e
shopt -s extglob # needed for rm everything but x
echo "Working directory: $PWD"

GENDOC_EXTRA_ARGS=$@
GIT_HEAD=$(git rev-parse HEAD) # save current head for commit message in gh-pages
PREVIOUS_SNAPSHOTS_DIR="$PWD/../prev_snapshots"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)"
SITE_OUT_DIR="$PWD/docs/_site"

DOCS_SCALA_LANG_DIR="$PWD/docsScalaLang"

rm -rf $DOCS_SCALA_LANG_DIR
mkdir -pv $DOCS_SCALA_LANG_DIR
git clone "https://github.com/scala/docs.scala-lang.git" $DOCS_SCALA_LANG_DIR

SBT="$SCRIPT_DIR/sbt"
mkdir -pv $SITE_OUT_DIR
env "scaladoc.projectFormat=md" "$SBT" "scaladoc/copyTheScaladocJsOutput $DOCS_SCALA_LANG_DIR/scripts/searchbar.js"
"dist/target/pack/bin/scaladoc" "-d" "$SITE_OUT_DIR" "-format" "md" "-siteroot" "docs" "/dev/null"

if [ ! -d "$SITE_OUT_DIR" ]; then
echo "Output directory did not exist: $SITE_OUT_DIR" 1>&2
exit 1
fi

cp -rf "$SITE_OUT_DIR/docs/reference"/* "$DOCS_SCALA_LANG_DIR/_scala3-reference"
cp -f "$SITE_OUT_DIR/resources/my.css" "$DOCS_SCALA_LANG_DIR/scripts/my.css"
cp -f "$SITE_OUT_DIR/resources/footer.html" "$DOCS_SCALA_LANG_DIR/_includes/footer.html"
27 changes: 20 additions & 7 deletions scaladoc-js/src/Main.scala
Original file line number Diff line number Diff line change
@@ -1,9 +1,22 @@
package dotty.tools.scaladoc

object Main extends App {
Searchbar()
SocialLinks()
CodeSnippets()
DropdownHandler()
Ux()
}
object Main:

private def common(): Unit =
CodeSnippets()

def main(): Unit =
Searchbar()
SocialLinks()
DropdownHandler()
Ux()
common()

/**
* This main is conditionally enabled by system env variable `scaladoc.projectFormat=md`
* passed in ./projects/scripts/genDocsScalaLang
* The reason why we have to pass the condition by env variable is because js is build before scaladoc,
* so we cannot access its args
*/
def markdownMain(): Unit =
common()
6 changes: 5 additions & 1 deletion scaladoc/src/dotty/tools/scaladoc/Scaladoc.scala
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,10 @@ object Scaladoc:
given docContext: DocContext = new DocContext(args, ctx)
val module = ScalaModuleProvider.mkModule()

new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members).render()
val renderer = args.projectFormat match
case "html" => new dotty.tools.scaladoc.renderers.HtmlRenderer(module.rootPackage, module.members)
case "md" => new dotty.tools.scaladoc.renderers.MarkdownRenderer(module.rootPackage, module.members)

renderer.render()
report.inform("generation completed successfully")
docContext
141 changes: 14 additions & 127 deletions scaladoc/src/dotty/tools/scaladoc/renderers/HtmlRenderer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,122 +14,21 @@ import java.nio.file.Files
import java.nio.file.FileVisitOption
import java.io.File

case class Page(link: Link, content: Member | ResolvedTemplate | String, children: Seq[Page]):
def withNewChildren(newChildren: Seq[Page]) = copy(children = children ++ newChildren)

def withTitle(newTitle: String) = copy(link = link.copy(name = newTitle))

def hasFrame = content match
case t: ResolvedTemplate => t.hasFrame
case _ => true

class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx: DocContext)
extends SiteRenderer, Resources, Locations, Writer:
private val args = summon[DocContext].args
val staticSite = summon[DocContext].staticSiteContext

val effectiveMembers = members

private def memberPage(member: Member): Page =
val childrenPages = member.members.filter(_.needsOwnPage)
Page(Link(member.name, member.dri), member, childrenPages.map(memberPage))

val navigablePage: Page =
val rootPckPage = memberPage(rootPackage)
staticSite match
case None => rootPckPage.withTitle(args.name)
case Some(siteContext) =>
val (indexes, templates) = siteContext.templates.partition(f =>
f.templateFile.isIndexPage() && f.file.toPath.getParent() == siteContext.docsPath)
if (indexes.size > 1)
val msg = s"ERROR: Multiple index pages for doc found ${indexes.map(_.file)}"
report.error(msg)

val templatePages = templates.map(templateToPage(_, siteContext))

indexes.headOption match
case None if templatePages.isEmpty=>
rootPckPage.withTitle(args.name)
case None =>
Page(Link(args.name, docsRootDRI),"", templatePages :+ rootPckPage.withTitle("API"))
case Some(indexPage) =>
val newChildren = templatePages :+ rootPckPage.withTitle("API")
templateToPage(indexPage, siteContext).withNewChildren(newChildren)

val hiddenPages: Seq[Page] =
staticSite match
case None =>
Seq(navigablePage.copy( // Add index page that is a copy of api/index.html
link = navigablePage.link.copy(dri = docsRootDRI),
children = Nil
))
case Some(siteContext) =>
// In case that we do not have an index page and we do not have any API entries
// we want to create empty index page, so there is one
val actualIndexTemplate = siteContext.indexTemplate() match {
case None if effectiveMembers.isEmpty => Seq(siteContext.emptyIndexTemplate)
case templates => templates.toSeq
}

(siteContext.orphanedTemplates ++ actualIndexTemplate).map(templateToPage(_, siteContext))

/**
* Here we have to retrive index pages from hidden pages and replace fake index pages in navigable page tree.
*/
val allPages: Seq[Page] =
def traversePages(page: Page): (Page, Seq[Page]) =
val (newChildren, newPagesToRemove): (Seq[Page], Seq[Page]) = page.children.map(traversePages(_)).foldLeft((Seq[Page](), Seq[Page]())) {
case ((pAcc, ptrAcc), (p, ptr)) => (pAcc :+ p, ptrAcc ++ ptr)
}
hiddenPages.find(_.link == page.link) match
case None =>
(page.copy(children = newChildren), newPagesToRemove)
case Some(newPage) =>
(newPage.copy(children = newChildren), newPagesToRemove :+ newPage)

val (newNavigablePage, pagesToRemove) = traversePages(navigablePage)

val all = newNavigablePage +: hiddenPages.filterNot(pagesToRemove.contains)
// We need to check for conflicts only if we have top-level member called blog or docs
val hasPotentialConflict =
rootPackage.members.exists(m => m.name.startsWith("docs") || m.name.startsWith("blog"))

if hasPotentialConflict then
def walk(page: Page): Unit =
if page.link.dri.isStaticFile then
val dest = absolutePath(page.link.dri)
if apiPaths.contains(dest) then
report.error(s"Conflict between static page and API member for $dest. $pathsConflictResoultionMsg")
page.children.foreach(walk)

all.foreach (walk)

all

def renderContent(page: Page) = page.content match
case m: Member =>
val signatureRenderer = new SignatureRenderer:
def currentDri: DRI = page.link.dri
def link(dri: DRI): Option[String] =
Some(pathToPage(currentDri, dri)).filter(_ != UnresolvedLocationLink)

MemberRenderer(signatureRenderer).fullMember(m)
case t: ResolvedTemplate => siteContent(page.link.dri, t)
case a: String => raw(a)


def renderPage(page: Page, parents: Vector[Link]): Seq[String] =
val newParents = parents :+ page.link
val content = ctx.args.projectFormat match
case "html" => html(
mkHead(page),
body(
if !page.hasFrame then renderContent(page)
else mkFrame(page.link, newParents, renderContent(page))
)
class HtmlRenderer(rootPackage: Member, members: Map[DRI, Member])(using ctx: DocContext)
extends Renderer(rootPackage, members, extension = "html"):

override def pageContent(page: Page, parents: Vector[Link]): AppliedTag =
html(
mkHead(page),
body(
if !page.hasFrame then renderContent(page)
else mkFrame(page.link, parents, renderContent(page))
)
case "md" => renderContent(page)
write(page.link.dri, content) +: page.children.flatMap(renderPage(_, newParents))
)

override def render(): Unit =
val renderedResources = renderResources()
super.render()

private def specificResources(page: Page): Set[String] =
page.children.toSet.flatMap(specificResources) ++ (page.content match
Expand Down Expand Up @@ -159,10 +58,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx
val resources = siteResourcesPaths.toSeq.map(pathToResource) ++ allResources(allPages) ++ onlyRenderedResources
resources.flatMap(renderResource)

def render(): Unit =
val renderedResources = renderResources()
val sites = allPages.map(renderPage(_, Vector.empty))

def mkHead(page: Page): AppliedTag =
val resources = page.content match
case t: ResolvedTemplate =>
Expand Down Expand Up @@ -226,14 +121,6 @@ class HtmlRenderer(rootPackage: Member, val members: Map[DRI, Member])(using ctx

renderNested(navigablePage, toplevel = true)._2

private def canonicalUrl(l: String): AppliedTag | String =
val canon = args.docCanonicalBaseUrl
if !canon.isEmpty then
val canonicalUrl = if canon.endsWith("/") then canon else canon + "/"
link(rel := "canonical", href := canonicalUrl + l)
else
"" // return empty tag

private def hasSocialLinks = !args.socialLinks.isEmpty

private def socialLinks(whiteIcon: Boolean = true) =
Expand Down
4 changes: 2 additions & 2 deletions scaladoc/src/dotty/tools/scaladoc/renderers/Locations.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ trait Locations(using ctx: DocContext):

// We generate this collection only if there may be a conflict with resources.
// Potentially can be quite big.
lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath)
lazy val apiPaths = effectiveMembers.keySet.filterNot(_.isStaticFile).map(absolutePath(_))

var cache = new JHashMap[DRI, Seq[String]]()

Expand Down Expand Up @@ -80,7 +80,7 @@ trait Locations(using ctx: DocContext):
pathToRaw(from, to.split("/").toList)

def resolveRoot(dri: DRI, path: String): String = resolveRoot(rawLocation(dri), path)
def absolutePath(dri: DRI): String = rawLocation(dri).mkString("", "/", ".html")
def absolutePath(dri: DRI, extension: String = "html"): String = rawLocation(dri).mkString("", "/", s".$extension")

def resolveLink(dri: DRI, url: String): String =
if URI(url).isAbsolute then url else resolveRoot(dri, url)
Expand Down
48 changes: 48 additions & 0 deletions scaladoc/src/dotty/tools/scaladoc/renderers/MarkdownRenderer.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package dotty.tools.scaladoc
package renderers

import util.HTML._
import collection.JavaConverters._
import java.net.URI
import java.net.URL
import dotty.tools.scaladoc.site._
import scala.util.Try
import org.jsoup.Jsoup
import java.nio.file.Paths
import java.nio.file.Path
import java.nio.file.Files
import java.nio.file.FileVisitOption
import java.io.File

class MarkdownRenderer(rootPackage: Member, members: Map[DRI, Member])(using ctx: DocContext)
extends Renderer(rootPackage, members, extension = "md"):

override def render(): Unit =
renderResources()
super.render()

override def pageContent(page: Page, parents: Vector[Link]): AppliedTag =
renderContent(page)

private def renderResources(): Seq[String] =

// TODO REMOVE THIS CODE AND RESOURCES UNDER docs/resources
// when we sort out the css classes
def siteRoot = staticSite.get.root.toPath
def pathToResource(p: String) = Resource.File(p, siteRoot.resolve(p))

def harvestResources(path: String) =
val siteImgPath = siteRoot.resolve(path)
if !Files.exists(siteImgPath) then Nil
else
val allPaths = Files.walk(siteImgPath, FileVisitOption.FOLLOW_LINKS)
val files = allPaths.filter(Files.isRegularFile(_)).iterator().asScala
files.map(p => siteRoot.relativize(p).toString).toList

val staticResources = staticSite.toSeq.flatMap { _ =>
harvestResources("images") ++ harvestResources("resources")
}
staticResources.map(pathToResource).flatMap(renderResource)
// END TODO REMOVE THIS CODE AND RESOURCES UNDER docs/resources

allResources(Nil).flatMap(renderResource)

0 comments on commit 8ec44e0

Please sign in to comment.