Skip to content

Make Token be a record. #1352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Mar 27, 2026
+32 −53
Merged

Make Token be a record. #1352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public static String formatJavadoc(String input, int blockIndent) {
private static String render(List<Token> input, int blockIndent) {
JavadocWriter output = new JavadocWriter(blockIndent);
for (Token token : input) {
switch (token.getType()) {
switch (token.type()) {
case BEGIN_JAVADOC -> output.writeBeginJavadoc();
case END_JAVADOC -> {
output.writeEndJavadoc();
Expand Down Expand Up @@ -105,7 +105,7 @@ private static Token standardizePToken(Token token) {
}

private static Token standardize(Token token, Token standardToken) {
return SIMPLE_TAG_PATTERN.matcher(token.getValue()).matches() ? standardToken : token;
return SIMPLE_TAG_PATTERN.matcher(token.value()).matches() ? standardToken : token;
}

private static final Token STANDARD_BR_TOKEN = new Token(BR_TAG, "<br>");
Expand Down
55 changes: 25 additions & 30 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,8 +295,8 @@ private static ImmutableList<Token> joinAdjacentLiteralsAndAdjacentWhitespace(Li
StringBuilder accumulated = new StringBuilder();

for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {
if (tokens.peek().getType() == LITERAL) {
accumulated.append(tokens.peek().getValue());
if (tokens.peek().type() == LITERAL) {
accumulated.append(tokens.peek().value());
tokens.next();
continue;
}
Expand All @@ -315,14 +315,14 @@ private static ImmutableList<Token> joinAdjacentLiteralsAndAdjacentWhitespace(Li
}

StringBuilder seenWhitespace = new StringBuilder();
while (tokens.peek().getType() == WHITESPACE) {
seenWhitespace.append(tokens.next().getValue());
while (tokens.peek().type() == WHITESPACE) {
seenWhitespace.append(tokens.next().value());
}

if (tokens.peek().getType() == LITERAL && tokens.peek().getValue().startsWith("@")) {
if (tokens.peek().type() == LITERAL && tokens.peek().value().startsWith("@")) {
// OK, we're in the case described above.
accumulated.append(" ");
accumulated.append(tokens.peek().getValue());
accumulated.append(tokens.peek().value());
tokens.next();
continue;
}
Expand Down Expand Up @@ -355,14 +355,13 @@ private static ImmutableList<Token> inferParagraphTags(List<Token> input) {
ImmutableList.Builder<Token> output = ImmutableList.builder();

for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {
if (tokens.peek().getType() == LITERAL) {
if (tokens.peek().type() == LITERAL) {
output.add(tokens.next());

if (tokens.peek().getType() == WHITESPACE
&& hasMultipleNewlines(tokens.peek().getValue())) {
if (tokens.peek().type() == WHITESPACE && hasMultipleNewlines(tokens.peek().value())) {
output.add(tokens.next());

if (tokens.peek().getType() == LITERAL) {
if (tokens.peek().type() == LITERAL) {
output.add(new Token(PARAGRAPH_OPEN_TAG, "<p>"));
}
}
Expand Down Expand Up @@ -393,11 +392,11 @@ private static ImmutableList<Token> optionalizeSpacesAfterLinks(List<Token> inpu
ImmutableList.Builder<Token> output = ImmutableList.builder();

for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {
if (tokens.peek().getType() == LITERAL && tokens.peek().getValue().matches("href=[^>]*>")) {
if (tokens.peek().type() == LITERAL && tokens.peek().value().matches("href=[^>]*>")) {
output.add(tokens.next());

if (tokens.peek().getType() == WHITESPACE) {
output.add(new Token(OPTIONAL_LINE_BREAK, tokens.next().getValue()));
if (tokens.peek().type() == WHITESPACE) {
output.add(new Token(OPTIONAL_LINE_BREAK, tokens.next().value()));
}
} else {
output.add(tokens.next());
Expand All @@ -422,18 +421,17 @@ private static ImmutableList<Token> deindentPreCodeBlocks(List<Token> input) {
// TODO: b/323389829 - De-indent {@snippet ...} blocks, too.
ImmutableList.Builder<Token> output = ImmutableList.builder();
for (PeekingIterator<Token> tokens = peekingIterator(input.iterator()); tokens.hasNext(); ) {
if (tokens.peek().getType() != PRE_OPEN_TAG) {
if (tokens.peek().type() != PRE_OPEN_TAG) {
output.add(tokens.next());
continue;
}

output.add(tokens.next());
List<Token> initialNewlines = new ArrayList<>();
while (tokens.hasNext() && tokens.peek().getType() == FORCED_NEWLINE) {
while (tokens.hasNext() && tokens.peek().type() == FORCED_NEWLINE) {
initialNewlines.add(tokens.next());
}
if (tokens.peek().getType() != LITERAL
|| !tokens.peek().getValue().matches("[ \t]*[{]@code")) {
if (tokens.peek().type() != LITERAL || !tokens.peek().value().matches("[ \t]*[{]@code")) {
output.addAll(initialNewlines);
output.add(tokens.next());
continue;
Expand All @@ -447,15 +445,15 @@ private static ImmutableList<Token> deindentPreCodeBlocks(List<Token> input) {
private static void deindentPreCodeBlock(
ImmutableList.Builder<Token> output, PeekingIterator<Token> tokens) {
Deque<Token> saved = new ArrayDeque<>();
output.add(new Token(LITERAL, tokens.next().getValue().trim()));
while (tokens.hasNext() && tokens.peek().getType() != PRE_CLOSE_TAG) {
output.add(new Token(LITERAL, tokens.next().value().trim()));
while (tokens.hasNext() && tokens.peek().type() != PRE_CLOSE_TAG) {
Token token = tokens.next();
saved.addLast(token);
}
while (!saved.isEmpty() && saved.peekFirst().getType() == FORCED_NEWLINE) {
while (!saved.isEmpty() && saved.peekFirst().type() == FORCED_NEWLINE) {
saved.removeFirst();
}
while (!saved.isEmpty() && saved.peekLast().getType() == FORCED_NEWLINE) {
while (!saved.isEmpty() && saved.peekLast().type() == FORCED_NEWLINE) {
saved.removeLast();
}
if (saved.isEmpty()) {
Expand All @@ -465,20 +463,19 @@ private static void deindentPreCodeBlock(
// move the trailing `}` to its own line
Token last = saved.peekLast();
boolean trailingBrace = false;
if (last.getType() == LITERAL && last.getValue().endsWith("}")) {
if (last.type() == LITERAL && last.value().endsWith("}")) {
saved.removeLast();
if (last.length() > 1) {
saved.addLast(
new Token(LITERAL, last.getValue().substring(0, last.getValue().length() - 1)));
saved.addLast(new Token(LITERAL, last.value().substring(0, last.value().length() - 1)));
saved.addLast(new Token(FORCED_NEWLINE, null));
}
trailingBrace = true;
}

int trim = -1;
for (Token token : saved) {
if (token.getType() == LITERAL) {
int idx = CharMatcher.isNot(' ').indexIn(token.getValue());
if (token.type() == LITERAL) {
int idx = CharMatcher.isNot(' ').indexIn(token.value());
if (idx != -1 && (trim == -1 || idx < trim)) {
trim = idx;
}
Expand All @@ -487,13 +484,11 @@ private static void deindentPreCodeBlock(

output.add(new Token(FORCED_NEWLINE, "\n"));
for (Token token : saved) {
if (token.getType() == LITERAL) {
if (token.type() == LITERAL) {
output.add(
new Token(
LITERAL,
trim > 0 && token.length() > trim
? token.getValue().substring(trim)
: token.getValue()));
trim > 0 && token.length() > trim ? token.value().substring(trim) : token.value()));
} else {
output.add(token);
}
Expand Down
6 changes: 3 additions & 3 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,17 +359,17 @@ private void writeToken(Token token) {
}

if (requestedMoeBeginStripComment != null) {
output.append(requestedMoeBeginStripComment.getValue());
output.append(requestedMoeBeginStripComment.value());
requestedMoeBeginStripComment = null;
indentForMoeEndStripComment = innerIndent();
requestNewline();
writeToken(token);
return;
}

output.append(token.getValue());
output.append(token.value());

if (!START_OF_LINE_TOKENS.contains(token.getType())) {
if (!START_OF_LINE_TOKENS.contains(token.type())) {
atStartOfLine = false;
}

Expand Down
20 changes: 2 additions & 18 deletions core/src/main/java/com/google/googlejavaformat/java/javadoc/Token.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
* naturally expect. The decision is usually pragmatic rather than theoretical. Most of the details
* are in {@link JavadocLexer}.
*/
final class Token {
record Token(Token.Type type, String value) {
/**
* Javadoc token type.
*
Expand Down Expand Up @@ -108,28 +108,12 @@ enum Type {
;
}

private final Type type;
private final String value;

Token(Type type, String value) {
this.type = type;
this.value = value;
}

Type getType() {
return type;
}

String getValue() {
return value;
}

int length() {
return value.length();
}

@Override
public String toString() {
return "\n" + getType() + ": " + getValue();
return "\n" + type() + ": " + value();
}
}
Loading