Skip to content

Commit

Permalink
Fix traversing the directory hierarchy on WindowsOS / Ant-style path …
Browse files Browse the repository at this point in the history 
…matching (#1615)

Globs always use a "/" as directory separator on all OS's. Input patterns containing
a "\" on Windows OS are transformed to "/" as users on Windows more likely would
assume that the "\" may be used.

On WindowsOS, transform "\" in the filepath to "/" before comparing the filename with
the regular expression (of the glob) which always uses "/" as separator.

Refactor all logic which create globs based on an input path.
- If a path (absolute or relative) point to a directory, that path is expanded
  to the default globs (*.kt, *.kts) in that specific directory or any of its
  subdirectories.
- If a path (absolute or relative) does not point to a directory, e.g. it
  points to a file, or it is a pattern. See "**" replacement below.
- On Windows OS patters containing a "*" (or "**") can not be resolved with
  default Paths utilities. In such case the given input pattern is handled as
  is. See "**" replacement below.

Patterns that contain one or more occurrence of a "**" are split into multiple
patterns so that files on that specific path and subdirectories will be matched.
 - For example, for path "some/path/**/*.kt" an additional pattern
   "some/path/*.kt" is generated to make sure that not only the "*.kt" files in
   a subdirectory of "some/path/" are found but also the "*.kt" in directory
   "some/path" as well. This is in sync with the "**" notation in a glob which
   should be interpreted as having zero or more intermediate subdirectories.
 - For example, for path "some/**/path/**/*.kt", multiple additional patterns
   are generated. As it contains two "**" patterns, 2 x 2 patterns are needed
   to match all possible combinations:
   - "some/**/path/**/*.kt"
   - "some/**/path/*.kt"
   - "some/path/**/*.kt"
   - "some/path/*.kt"

Finally, on Windows OS more fixes are needed as the resulting globs may not
contain any drive destinations as the start of the path. Such a drive
destination is replaced with a "**". So "D:/some/path/*.kt" becomes
"/some/path/*.kt". Note that the last glob representation is less strict than
the original pattern as it could match on other drives that "D:/" as well.

Extend trace logging.

Closes #1600
Closes #1601
  • Loading branch information
@paul-dingemans
paul-dingemans committed Sep 3, 2022
1 parent e5ff031 commit 3c71f71
Show file tree
Hide file tree
Showing 3 changed files with 352 additions and 122 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Expand Up @@ -10,6 +10,9 @@ This project adheres to [Semantic Versioning](https://semver.org/).

Do not show deprecation warning about property "disabled_rules" when using CLi-parameter `--disabled-rules` ([#1599](https://github.com/pinterest/ktlint/issues/1599))

* Traversing directory hierarchy at Windows ([#1600](https://github.com/pinterest/ktlint/issues/1600))
* Ant-style path pattern support ([#1601](https://github.com/pinterest/ktlint/issues/1601))

### Added

### Changed
Expand Down
223 changes: 181 additions & 42 deletions ktlint/src/main/kotlin/com/pinterest/ktlint/internal/FileUtils.kt
View file
Expand Up @@ -15,7 +15,11 @@ import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.Deque
import java.util.LinkedList
import kotlin.io.path.absolutePathString
import kotlin.io.path.isDirectory
import kotlin.io.path.pathString
import kotlin.system.exitProcess
import kotlin.system.measureTimeMillis
import mu.KotlinLogging
Expand All @@ -28,11 +32,10 @@ internal val workDir: String = File(".").canonicalPath
private val tildeRegex = Regex("^(!)?~")
private const val NEGATION_PREFIX = "!"

private val os = System.getProperty("os.name")
private val userHome = System.getProperty("user.home")

private val defaultKotlinFileExtensions = setOf("kt", "kts")
internal val defaultPatterns = defaultKotlinFileExtensions.map { "**$globSeparator*.$it" }
internal val defaultPatterns = defaultKotlinFileExtensions.map { "**/*.$it" }

/**
* Transform the [patterns] to a sequence of files. Each element in [patterns] can be a glob, a file or directory path
Expand Down Expand Up @@ -84,7 +87,7 @@ internal fun FileSystem.fileSequence(
Start walkFileTree for rootDir: '$rootDir'
include:
${pathMatchers.map { " - $it" }}
exlcude:
exclude:
${negatedPathMatchers.map { " - $it" }}
""".trimIndent()
}
Expand All @@ -96,13 +99,27 @@ internal fun FileSystem.fileSequence(
filePath: Path,
fileAttrs: BasicFileAttributes,
): FileVisitResult {
if (negatedPathMatchers.none { it.matches(filePath) } &&
pathMatchers.any { it.matches(filePath) }
val path =
if (onWindowsOS) {
Paths.get(
filePath
.absolutePathString()
.replace(File.separatorChar, '/'),
).also {
if (it != filePath) {
logger.trace { "On WindowsOS transform '$filePath' to '$it'" }
}
}
} else {
filePath
}
if (negatedPathMatchers.none { it.matches(path) } &&
pathMatchers.any { it.matches(path) }
) {
logger.trace { "- File: $filePath: Include" }
result.add(filePath)
logger.trace { "- File: $path: Include" }
result.add(path)
} else {
logger.trace { "- File: $filePath: Ignore" }
logger.trace { "- File: $path: Ignore" }
}
return FileVisitResult.CONTINUE
}
Expand All @@ -127,14 +144,32 @@ internal fun FileSystem.fileSequence(
return result.asSequence()
}

private fun FileSystem.expand(
internal fun FileSystem.expand(
patterns: List<String>,
rootDir: Path,
) =
patterns
.map { it.expandTildeToFullPath() }
.map { it.replace(File.separator, globSeparator) }
.flatMap { path -> toGlob(path, rootDir) }
.mapNotNull {
if (onWindowsOS) {
it.normalizeWindowsPattern()
} else {
it
}
}.map { it.expandTildeToFullPath() }
.map {
if (onWindowsOS) {
// By definition the globs should use "/" as separator. Out of courtesy replace "\" with "/"
it
.replace(File.separator, "/")
.also { transformedPath ->
if (it != transformedPath) {
logger.trace { "On WindowsOS transform '$it' to '$transformedPath'" }
}
}
} else {
it
}
}.flatMap { path -> toGlob(path, rootDir) }

private fun FileSystem.toGlob(
path: String,
Expand All @@ -145,42 +180,135 @@ private fun FileSystem.toGlob(
} else {
""
}
val pathWithoutNegationPrefix = path.removePrefix(NEGATION_PREFIX)
val resolvedPath = try {
rootDir.resolve(pathWithoutNegationPrefix)
val pathWithoutNegationPrefix =
path
.removePrefix(NEGATION_PREFIX)
val expandedPatterns = try {
val resolvedPath =
rootDir
.resolve(pathWithoutNegationPrefix)
.normalize()
if (resolvedPath.isDirectory()) {
resolvedPath
.expandPathToDefaultPatterns()
.also {
logger.trace { "Expanding resolved directory path '$resolvedPath' to patterns: [$it]" }
}
} else {
resolvedPath
.pathString
.expandDoubleStarPatterns()
.also {
logger.trace { "Expanding resolved path '$resolvedPath` to patterns: [$it]" }
}
}
} catch (e: InvalidPathException) {
// Windows throws an exception when you pass a glob to Path#resolve.
null
}
val expandedGlobs = if (resolvedPath != null && resolvedPath.isDirectory()) {
getDefaultPatternsForPath(resolvedPath)
} else if (isGlobAbsolutePath(pathWithoutNegationPrefix)) {
listOf(pathWithoutNegationPrefix)
} else {
listOf(pathWithoutNegationPrefix.prefixIfNot("**$globSeparator"))
if (onWindowsOS) {
// Windows throws an exception when passing a wildcard (*) to Path#resolve.
pathWithoutNegationPrefix
.expandDoubleStarPatterns()
.also {
logger.trace { "On WindowsOS: expanding unresolved path '$pathWithoutNegationPrefix` to patterns: [$it]" }
}
} else {
emptyList()
}
}
return expandedGlobs.map { "${negation}glob:$it" }
}

private fun getDefaultPatternsForPath(path: Path?) = defaultKotlinFileExtensions
.flatMap {
listOf(
"$path$globSeparator*.$it",
"$path$globSeparator**$globSeparator*.$it",
)
}
return expandedPatterns
.map { originalPattern ->
if (onWindowsOS) {
originalPattern
// Replace "\" with "/"
.replace(this.separator, "/")
// Remove drive letter (and colon) from path as this will lead to invalid globs and replace it with a double
// star pattern. Technically this is not functionally identical as the pattern could match on multiple drives.
.substringAfter(":")
.removePrefix("/")
.prefixIfNot("**/")
.also { transformedPattern ->
if (transformedPattern != originalPattern) {
logger.trace { "On WindowsOS, transform '$originalPattern' to '$transformedPattern'" }
}
}
} else {
originalPattern
}
}.map { "${negation}glob:$it" }
}

private fun FileSystem.isGlobAbsolutePath(glob: String) =
rootDirectories
.map { it.toString() }
.any { glob.startsWith(it) }
/**
* For each double star pattern in the path, create and additional path in which the double start pattern is removed.
* In this way a pattern like some-directory/**/*.kt will match wile files in some-directory or any of its
* subdirectories.
*/
private fun String?.expandDoubleStarPatterns(): Set<String> {
val paths = mutableSetOf(this)
val parts = this?.split("/").orEmpty()
parts
.filter { it == "**" }
.forEach { doubleStarPart ->
run {
val expandedPath =
parts
.filter { it !== doubleStarPart }
.joinToString(separator = "/")
// The original path can contain multiple double star patterns. Replace only one double start pattern
// with an additional path patter and call recursively for remain double star patterns
paths.addAll(expandedPath.expandDoubleStarPatterns())
}
}
return paths.filterNotNull().toSet()
}

private val globSeparator: String get() =
when {
os.startsWith("windows", ignoreCase = true) -> "\\\\"
else -> "/"
private fun String?.normalizeWindowsPattern() =
if (onWindowsOS) {
val parts: Deque<String> = LinkedList()
// Replace "\" with "/"
this
?.replace("\\", "/")
?.split("/")
?.filterNot {
// Reference to current directory can simply be ignored
it == "."
}?.forEach {
if (it == "..") {
// Whenever the parent directory reference follows a part not containing a wildcard, then the parent
// reference and the preceding element can be ignored. In other cases, the result pattern can not be
// cleaned. If that pattern would be transformed to a glob then the result regular expression of
// that glob results in a pattern that will never be matched as the ".." reference will not occur in
// the filepath that is being checked with the regular expression.
if (parts.isEmpty()) {
logger.warn {
"On WindowsOS the pattern '$this' can not be used as it refers to a path outside of the current directory"
}
return@normalizeWindowsPattern null
} else if (parts.peekLast().contains('*')) {
logger.warn {
"On WindowsOS the pattern '$this' can not be used as '/..' follows the wildcard pattern ${parts.peekLast()}"
}
return@normalizeWindowsPattern null
} else {
parts.removeLast()
}
} else {
parts.addLast(it)
}
}
parts.joinToString(separator = "/")
} else {
this
}

private fun Path.expandPathToDefaultPatterns() =
defaultKotlinFileExtensions
.flatMap {
listOf(
"$this/*.$it",
"$this/**/*.$it",
)
}

/**
* List of paths to Java `jar` files.
*/
Expand All @@ -198,13 +326,24 @@ internal fun JarFiles.toFilesURIList() = map {
// a complete solution would be to implement https://www.gnu.org/software/bash/manual/html_node/Tilde-Expansion.html
// this implementation takes care only of the most commonly used case (~/)
internal fun String.expandTildeToFullPath(): String =
if (os.startsWith("windows", true)) {
if (onWindowsOS) {
// Windows sometimes inserts `~` into paths when using short directory names notation, e.g. `C:\Users\USERNA~1\Documents
this
} else {
replaceFirst(tildeRegex, userHome)
.also {
if (it != this) {
logger.trace { "On non-WindowsOS expand '$this' to '$it'" }
}
}
}

private val onWindowsOS
get() =
System
.getProperty("os.name")
.startsWith("windows", true)

internal fun File.location(
relative: Boolean,
) = if (relative) this.toRelativeString(File(workDir)) else this.path
Expand Down

0 comments on commit 3c71f71

Please sign in to comment.