From 8f95ca626725a78437870150856e5cf95a32cc2c Mon Sep 17 00:00:00 2001 From: vmishenev Date: Tue, 5 Apr 2022 01:25:43 +0300 Subject: [PATCH 1/3] Fix `@` symbol using inside code block --- .../base/src/main/kotlin/parsers/Parser.kt | 43 +++++++++++- .../src/test/kotlin/markdown/ParserTest.kt | 2 + ...uleAndPackageDocumentationFragmentsTest.kt | 67 +++++++++++++++++-- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/plugins/base/src/main/kotlin/parsers/Parser.kt b/plugins/base/src/main/kotlin/parsers/Parser.kt index 960d7a649a..898a898b2e 100644 --- a/plugins/base/src/main/kotlin/parsers/Parser.kt +++ b/plugins/base/src/main/kotlin/parsers/Parser.kt @@ -1,6 +1,8 @@ package org.jetbrains.dokka.base.parsers import org.jetbrains.dokka.model.doc.* +import org.jetbrains.dokka.model.doc.Deprecated +import org.jetbrains.dokka.model.doc.Suppress abstract class Parser { @@ -49,10 +51,49 @@ abstract class Parser { private fun jkdocToListOfPairs(javadoc: String): List> = "description $javadoc" - .split("\n@") + .splitIgnoredInsideBackticks("\n@") .map { content -> val contentWithEscapedAts = content.replace("\\@", "@") val (tag, body) = contentWithEscapedAts.split(" ", limit = 2) tag to body } + + private fun CharSequence.splitIgnoredInsideBackticks(delimiter: String): List { + var countOfBackticks = 0 + var countOfBackticksInOpeningFence = 0 + + var isInCode = false + val result = mutableListOf() + val buf = StringBuilder() + var currentOffset = 0 + while (currentOffset < length) { + + if (get(currentOffset) == '`') { + countOfBackticks++ + } else { + if (isInCode) { + // The closing code fence must be at least as long as the opening fence + isInCode = countOfBackticks < countOfBackticksInOpeningFence + } else { + if (countOfBackticks > 0) { + isInCode = true + countOfBackticksInOpeningFence = countOfBackticks + } + } + countOfBackticks = 0 + } + if (!isInCode && startsWith(delimiter, currentOffset)) { + result.add(buf.toString()) + buf.clear() + currentOffset += delimiter.length + continue + } + + buf.append(get(currentOffset)) + ++currentOffset + } + result.add(buf.toString()) + return result + } + } diff --git a/plugins/base/src/test/kotlin/markdown/ParserTest.kt b/plugins/base/src/test/kotlin/markdown/ParserTest.kt index a9e1eee429..3498f73f1b 100644 --- a/plugins/base/src/test/kotlin/markdown/ParserTest.kt +++ b/plugins/base/src/test/kotlin/markdown/ParserTest.kt @@ -951,6 +951,7 @@ class ParserTest : KDocTest() { fun `Multilined Code Block`() { val kdoc = """ | ```kotlin + | @Suppress("UNUSED_VARIABLE") | val x: Int = 0 | val y: String = "Text" | @@ -968,6 +969,7 @@ class ParserTest : KDocTest() { listOf( CodeBlock( listOf( + Text("@Suppress(\"UNUSED_VARIABLE\")"), Br, Text("val x: Int = 0"), Br, Text("val y: String = \"Text\""), Br, Br, Text(" val z: Boolean = true"), Br, diff --git a/plugins/base/src/test/kotlin/parsers/ParseModuleAndPackageDocumentationFragmentsTest.kt b/plugins/base/src/test/kotlin/parsers/ParseModuleAndPackageDocumentationFragmentsTest.kt index 4560c53271..b6f9307fc0 100644 --- a/plugins/base/src/test/kotlin/parsers/ParseModuleAndPackageDocumentationFragmentsTest.kt +++ b/plugins/base/src/test/kotlin/parsers/ParseModuleAndPackageDocumentationFragmentsTest.kt @@ -1,12 +1,11 @@ package parsers -import org.jetbrains.dokka.base.parsers.moduleAndPackage.IllegalModuleAndPackageDocumentation +import org.intellij.markdown.MarkdownElementTypes +import org.jetbrains.dokka.base.parsers.moduleAndPackage.* import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentation.Classifier.Module import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentation.Classifier.Package -import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentationFile -import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentationFragment -import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumentationSource -import org.jetbrains.dokka.base.parsers.moduleAndPackage.parseModuleAndPackageDocumentationFragments +import org.jetbrains.dokka.model.doc.* +import org.jetbrains.dokka.utilities.DokkaLogger import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -216,6 +215,64 @@ class ParseModuleAndPackageDocumentationFragmentsTest { ) } + @Test + fun `at in code block is supported`() { + val fragment = parseModuleAndPackageDocumentationFragments( + source( + """ + # Module My Module + ``` + @Smth + ``` + @author Smb + """.trimIndent() + ) + ) + + assertEquals( + "```\n" + + "@Smth\n" + + "```\n" + + "@author Smb", fragment.single().documentation, + "Expected documentation being available" + ) + + val parsingContext = ModuleAndPackageDocumentationParsingContext(object : DokkaLogger { + override var warningsCount: Int = 0 + override var errorsCount: Int = 0 + override fun debug(message: String) {} + override fun info(message: String) {} + override fun progress(message: String) {} + override fun warn(message: String) {} + override fun error(message: String) {} + }) + val parsedFragment = parseModuleAndPackageDocumentation(parsingContext, fragment.single()) + val expectedDocumentationNode = DocumentationNode( + listOf( + Description( + CustomDocTag( + listOf( + CodeBlock( + listOf( + Text("@Smth") + ) + ) + ), name = MarkdownElementTypes.MARKDOWN_FILE.name + ) + ), + Author( + CustomDocTag( + listOf( + P(listOf(Text("Smb"))) + ), name = MarkdownElementTypes.MARKDOWN_FILE.name + ) + ) + ) + ) + assertEquals( + expectedDocumentationNode, parsedFragment.documentation + ) + } private fun source(documentation: String): ModuleAndPackageDocumentationSource = object : ModuleAndPackageDocumentationSource() { From 0dca3df7d37e63da6d40f57d40e0310128ce3a9f Mon Sep 17 00:00:00 2001 From: vmishenev Date: Wed, 6 Apr 2022 00:59:50 +0300 Subject: [PATCH 2/3] Replace StringBuilder with Range --- plugins/base/src/main/kotlin/parsers/Parser.kt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/base/src/main/kotlin/parsers/Parser.kt b/plugins/base/src/main/kotlin/parsers/Parser.kt index 898a898b2e..ec55c321af 100644 --- a/plugins/base/src/main/kotlin/parsers/Parser.kt +++ b/plugins/base/src/main/kotlin/parsers/Parser.kt @@ -64,7 +64,8 @@ abstract class Parser { var isInCode = false val result = mutableListOf() - val buf = StringBuilder() + var rangeStart = 0 + var rangeEnd = 0 var currentOffset = 0 while (currentOffset < length) { @@ -83,16 +84,17 @@ abstract class Parser { countOfBackticks = 0 } if (!isInCode && startsWith(delimiter, currentOffset)) { - result.add(buf.toString()) - buf.clear() + result.add(substring(rangeStart, rangeEnd)) currentOffset += delimiter.length + rangeStart = currentOffset + rangeEnd = currentOffset continue } - buf.append(get(currentOffset)) + ++rangeEnd ++currentOffset } - result.add(buf.toString()) + result.add(substring(rangeStart, rangeEnd)) return result } From b7874c82c8244ced6e5daef5f6028d627c67e939 Mon Sep 17 00:00:00 2001 From: vmishenev Date: Mon, 11 Apr 2022 04:43:41 +0300 Subject: [PATCH 3/3] Support code fence of tildes --- .../base/src/main/kotlin/parsers/Parser.kt | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/plugins/base/src/main/kotlin/parsers/Parser.kt b/plugins/base/src/main/kotlin/parsers/Parser.kt index ec55c321af..af07ec5321 100644 --- a/plugins/base/src/main/kotlin/parsers/Parser.kt +++ b/plugins/base/src/main/kotlin/parsers/Parser.kt @@ -11,7 +11,7 @@ abstract class Parser { abstract fun preparse(text: String): String open fun parse(text: String): DocumentationNode = - DocumentationNode(jkdocToListOfPairs(preparse(text)).map { (tag, content) -> parseTagWithBody(tag, content) }) + DocumentationNode(extractTagsToListOfPairs(preparse(text)).map { (tag, content) -> parseTagWithBody(tag, content) }) open fun parseTagWithBody(tagName: String, content: String): TagWrapper = when (tagName) { @@ -49,18 +49,29 @@ abstract class Parser { else -> CustomTagWrapper(parseStringToDocNode(content), tagName) } - private fun jkdocToListOfPairs(javadoc: String): List> = - "description $javadoc" - .splitIgnoredInsideBackticks("\n@") + /** + * KDoc parser from Kotlin compiler relies on a comment asterisk + * So there is a mini parser here + * TODO: at least to adapt [org.jetbrains.kotlin.kdoc.lexer.KDocLexer] to analyze KDoc without the asterisks and use it here + */ + private fun extractTagsToListOfPairs(text: String): List> = + "description $text" + .extractKDocSections() .map { content -> val contentWithEscapedAts = content.replace("\\@", "@") val (tag, body) = contentWithEscapedAts.split(" ", limit = 2) tag to body } - private fun CharSequence.splitIgnoredInsideBackticks(delimiter: String): List { + /** + * Ignore a doc tag inside code spans and blocks + * @see org.jetbrains.kotlin.kdoc.psi.impl.KDocSection + */ + private fun CharSequence.extractKDocSections(delimiter: String = "\n@"): List { var countOfBackticks = 0 + var countOfTildes = 0 var countOfBackticksInOpeningFence = 0 + var countOfTildesInOpeningFence = 0 var isInCode = false val result = mutableListOf() @@ -69,19 +80,38 @@ abstract class Parser { var currentOffset = 0 while (currentOffset < length) { - if (get(currentOffset) == '`') { - countOfBackticks++ - } else { - if (isInCode) { - // The closing code fence must be at least as long as the opening fence - isInCode = countOfBackticks < countOfBackticksInOpeningFence - } else { - if (countOfBackticks > 0) { - isInCode = true - countOfBackticksInOpeningFence = countOfBackticks + when (get(currentOffset)) { + '`' -> { + countOfBackticks++ + countOfTildes = 0 + } + '~' -> { + countOfTildes++ + countOfBackticks = 0 + } + else -> { + if (isInCode) { + // The closing code fence must be at least as long as the opening fence + if(countOfBackticks >= countOfBackticksInOpeningFence + || countOfTildes >= countOfTildesInOpeningFence) + isInCode = false + } else { + // as per CommonMark spec, there can be any number of backticks for a code span, not only one or three + if (countOfBackticks > 0) { + isInCode = true + countOfBackticksInOpeningFence = countOfBackticks + countOfTildesInOpeningFence = Int.MAX_VALUE + } + // tildes are only for a code block, not code span + if (countOfTildes >= 3) { + isInCode = true + countOfTildesInOpeningFence = countOfTildes + countOfBackticksInOpeningFence = Int.MAX_VALUE + } } + countOfTildes = 0 + countOfBackticks = 0 } - countOfBackticks = 0 } if (!isInCode && startsWith(delimiter, currentOffset)) { result.add(substring(rangeStart, rangeEnd))