Skip to content

Commit

Permalink
Remove support for the String Templates preview feature
Browse files Browse the repository at this point in the history 
Rollback of b5feefe, 38de9c4, e946e82

See https://bugs.openjdk.org/browse/JDK-8329949

PiperOrigin-RevId: 629182379
  • Loading branch information
@cushon
cushon authored and google-java-format Team committed Apr 29, 2024
1 parent 6e3b852 commit fdf4b29
Show file tree
Hide file tree
Showing 11 changed files with 26 additions and 253 deletions.
13 changes: 5 additions & 8 deletions core/src/main/java/com/google/googlejavaformat/java/JavaInput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,14 +387,7 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
final boolean isNumbered; // Is this tok numbered? (tokens and comments)
String extraNewline = null; // Extra newline at end?
List<String> strings = new ArrayList<>();
if (tokText.startsWith("'")
|| tokText.startsWith("\"")
|| JavacTokens.isStringFragment(t.kind())) {
// Perform this check first, STRINGFRAGMENT tokens can start with arbitrary characters.
isToken = true;
isNumbered = true;
strings.add(originalTokText);
} else if (Character.isWhitespace(tokText0)) {
if (Character.isWhitespace(tokText0)) {
isToken = false;
isNumbered = false;
Iterator<String> it = Newlines.lineIterator(originalTokText);
Expand All @@ -411,6 +404,10 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
strings.add(line);
}
}
} else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
isToken = true;
isNumbered = true;
strings.add(originalTokText);
} else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
// For compatibility with an earlier lexer, the newline after a // comment is its own tok.
if (tokText.startsWith("//")
Expand Down
145 changes: 19 additions & 126 deletions core/src/main/java/com/google/googlejavaformat/java/JavacTokens.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
package com.google.googlejavaformat.java;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkElementIndex;
import static java.util.Arrays.stream;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
Expand All @@ -29,15 +27,7 @@
import com.sun.tools.javac.parser.Tokens.TokenKind;
import com.sun.tools.javac.parser.UnicodeReader;
import com.sun.tools.javac.util.Context;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import org.jspecify.annotations.Nullable;

/** A wrapper around javac's lexer. */
final class JavacTokens {
Expand All @@ -54,8 +44,6 @@ static class RawTok {
private final int endPos;

RawTok(String stringVal, TokenKind kind, int pos, int endPos) {
checkElementIndex(pos, endPos, "pos");
checkArgument(pos < endPos, "expected pos (%s) < endPos (%s)", pos, endPos);
this.stringVal = stringVal;
this.kind = kind;
this.pos = pos;
Expand Down Expand Up @@ -83,88 +71,30 @@ public String stringVal() {
}
}

private static final TokenKind STRINGFRAGMENT =
stream(TokenKind.values())
.filter(t -> t.name().contentEquals("STRINGFRAGMENT"))
.findFirst()
.orElse(null);

static boolean isStringFragment(TokenKind kind) {
return STRINGFRAGMENT != null && Objects.equals(kind, STRINGFRAGMENT);
}

private static ImmutableList<Token> readAllTokens(
String source, Context context, Set<Integer> nonTerminalStringFragments) {
/** Lex the input and return a list of {@link RawTok}s. */
public static ImmutableList<RawTok> getTokens(
String source, Context context, Set<TokenKind> stopTokens) {
if (source == null) {
return ImmutableList.of();
}
ScannerFactory fac = ScannerFactory.instance(context);
char[] buffer = (source + EOF_COMMENT).toCharArray();
Scanner scanner =
new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length));
List<Token> tokens = new ArrayList<>();
do {
scanner.nextToken();
tokens.add(scanner.token());
} while (scanner.token().kind != TokenKind.EOF);
for (int i = 0; i < tokens.size(); i++) {
if (isStringFragment(tokens.get(i).kind)) {
int start = i;
while (isStringFragment(tokens.get(i).kind)) {
i++;
}
for (int j = start; j < i - 1; j++) {
nonTerminalStringFragments.add(tokens.get(j).pos);
}
}
}
// A string template is tokenized as a series of STRINGFRAGMENT tokens containing the string
// literal values, followed by the tokens for the template arguments. For the formatter, we
// want the stream of tokens to appear in order by their start position.
if (Runtime.version().feature() >= 21) {
Collections.sort(tokens, Comparator.comparingInt(t -> t.pos));
}
return ImmutableList.copyOf(tokens);
}

/** Lex the input and return a list of {@link RawTok}s. */
public static ImmutableList<RawTok> getTokens(
String source, Context context, Set<TokenKind> stopTokens) {
if (source == null) {
return ImmutableList.of();
}
Set<Integer> nonTerminalStringFragments = new HashSet<>();
ImmutableList<Token> javacTokens = readAllTokens(source, context, nonTerminalStringFragments);

ImmutableList.Builder<RawTok> tokens = ImmutableList.builder();
int end = source.length();
int last = 0;
for (Token t : javacTokens) {
do {
scanner.nextToken();
Token t = scanner.token();
if (t.comments != null) {
// javac accumulates comments in reverse order
for (Comment c : Lists.reverse(t.comments)) {
int pos = c.getSourcePos(0);
int length;
if (pos == -1) {
// We've found a comment whose position hasn't been recorded. Deduce its position as the
// first `/` character after the end of the previous token.
//
// javac creates a new JavaTokenizer to process string template arguments, so
// CommentSavingTokenizer doesn't get a chance to preprocess those comments and save
// their text and positions.
//
// TODO: consider always using this approach once the minimum supported JDK is 16 and
// we can assume BasicComment#getRawCharacters is always available.
pos = source.indexOf('/', last);
length = CommentSavingTokenizer.commentLength(c);
} else {
length = c.getText().length();
if (last < c.getSourcePos(0)) {
tokens.add(new RawTok(null, null, last, c.getSourcePos(0)));
}
if (last < pos) {
tokens.add(new RawTok(null, null, last, pos));
}
tokens.add(new RawTok(null, null, pos, pos + length));
last = pos + length;
tokens.add(
new RawTok(null, null, c.getSourcePos(0), c.getSourcePos(0) + c.getText().length()));
last = c.getSourcePos(0) + c.getText().length();
}
}
if (stopTokens.contains(t.kind)) {
Expand All @@ -176,25 +106,14 @@ public static ImmutableList<RawTok> getTokens(
if (last < t.pos) {
tokens.add(new RawTok(null, null, last, t.pos));
}
if (isStringFragment(t.kind)) {
int endPos = t.endPos;
int pos = t.pos;
if (nonTerminalStringFragments.contains(t.pos)) {
// Include the \ escape from \{...} in the preceding string fragment
endPos++;
}
tokens.add(new RawTok(source.substring(pos, endPos), t.kind, pos, endPos));
last = endPos;
} else {
tokens.add(
new RawTok(
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
t.kind,
t.pos,
t.endPos));
last = t.endPos;
}
}
tokens.add(
new RawTok(
t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
t.kind,
t.pos,
t.endPos));
last = t.endPos;
} while (scanner.token().kind != TokenKind.EOF);
if (last < end) {
tokens.add(new RawTok(null, null, last, end));
}
Expand All @@ -203,32 +122,6 @@ public static ImmutableList<RawTok> getTokens(

/** A {@link JavaTokenizer} that saves comments. */
static class CommentSavingTokenizer extends JavaTokenizer {

private static final Method GET_RAW_CHARACTERS_METHOD = getRawCharactersMethod();

private static @Nullable Method getRawCharactersMethod() {
try {
// This is a method in PositionTrackingReader, but that class is not public.
return BasicComment.class.getMethod("getRawCharacters");
} catch (NoSuchMethodException e) {
return null;
}
}

static int commentLength(Comment comment) {
if (comment instanceof BasicComment && GET_RAW_CHARACTERS_METHOD != null) {
// If we've seen a BasicComment instead of a CommentWithTextAndPosition, getText() will
// be null, so we deduce the length using getRawCharacters. See also the comment at the
// usage of this method in getTokens.
try {
return ((char[]) GET_RAW_CHARACTERS_METHOD.invoke(((BasicComment) comment))).length;
} catch (ReflectiveOperationException e) {
throw new LinkageError(e.getMessage(), e);
}
}
return comment.getText().length();
}

CommentSavingTokenizer(ScannerFactory fac, char[] buffer, int length) {
super(fac, buffer, length);
}
Expand Down
23 changes: 1 addition & 22 deletions core/src/main/java/com/google/googlejavaformat/java/java21/Java21InputAstVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import com.sun.source.tree.ExpressionTree;
import com.sun.source.tree.PatternCaseLabelTree;
import com.sun.source.tree.PatternTree;
import com.sun.source.tree.StringTemplateTree;
import com.sun.source.tree.Tree;
import com.sun.tools.javac.tree.JCTree;
import javax.lang.model.element.Name;
Expand Down Expand Up @@ -63,7 +62,6 @@ public Void visitConstantCaseLabel(ConstantCaseLabelTree node, Void aVoid) {

@Override
public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unused) {
sync(node);
scan(node.getDeconstructor(), null);
builder.open(plusFour);
token("(");
Expand All @@ -82,25 +80,6 @@ public Void visitDeconstructionPattern(DeconstructionPatternTree node, Void unus
return null;
}

@SuppressWarnings("preview")
@Override
public Void visitStringTemplate(StringTemplateTree node, Void unused) {
sync(node);
builder.open(plusFour);
scan(node.getProcessor(), null);
token(".");
token(builder.peekToken().get());
for (int i = 0; i < node.getFragments().size() - 1; i++) {
token("{");
builder.breakOp();
scan(node.getExpressions().get(i), null);
token("}");
token(builder.peekToken().get());
}
builder.close();
return null;
}

@Override
protected void variableName(Name name) {
if (name.isEmpty()) {
Expand All @@ -118,7 +97,7 @@ public Void scan(Tree tree, Void unused) {
visitJcAnyPattern((JCTree.JCAnyPattern) tree);
return null;
} else {
return super.scan(tree, unused);
return super.scan(tree, null);
}
}

Expand Down
4 changes: 1 addition & 3 deletions core/src/test/java/com/google/googlejavaformat/java/FormatterIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,8 @@ public class FormatterIntegrationTest {
"I880",
"Unnamed",
"I981",
"StringTemplate",
"I1020",
"I1037",
"TextBlockTemplates")
"I1037")
.build();

@Parameters(name = "{index}: {0}")
Expand Down
24 changes: 0 additions & 24 deletions core/src/test/java/com/google/googlejavaformat/java/FormatterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import static com.google.common.truth.Truth.assertWithMessage;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeTrue;

import com.google.common.base.Joiner;
import com.google.common.io.CharStreams;
Expand Down Expand Up @@ -511,27 +510,4 @@ public void removeTrailingTabsInComments() throws Exception {
+ " }\n"
+ "}\n");
}

@Test
public void stringTemplateTests() throws Exception {
assumeTrue(Runtime.version().feature() >= 21);
assertThat(
new Formatter()
.formatSource(
"public class Foo {\n"
+ " String test(){\n"
+ " var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n"
+ " var nested = STR.\"template \\{example. foo()+"
+ " STR.\"templateInner\\{ example}\"}xxx }\";\n"
+ " }\n"
+ "}\n"))
.isEqualTo(
"public class Foo {\n"
+ " String test() {\n"
+ " var simple = STR.\"mytemplate1XXXX \\{exampleXXXX.foo()}yyy\";\n"
+ " var nested = STR.\"template \\{example.foo() +"
+ " STR.\"templateInner\\{example}\"}xxx }\";\n"
+ " }\n"
+ "}\n");
}
}
12 changes: 0 additions & 12 deletions core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.input
View file

This file was deleted.

14 changes: 0 additions & 14 deletions core/src/test/resources/com/google/googlejavaformat/java/testdata/I981.output
View file

This file was deleted.

10 changes: 0 additions & 10 deletions core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.input
View file

This file was deleted.

9 changes: 0 additions & 9 deletions core/src/test/resources/com/google/googlejavaformat/java/testdata/StringTemplate.output
View file

This file was deleted.

11 changes: 0 additions & 11 deletions core/src/test/resources/com/google/googlejavaformat/java/testdata/TextBlockTemplates.input
View file

This file was deleted.

0 comments on commit fdf4b29

Please sign in to comment.