diff --git a/.gitattributes b/.gitattributes
index bcc0efa..bc4cd17 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,4 @@
* text eol=lf
src/test/resources/testcrlf* text eol=crlf
+# Preserve the exact line endings and EOF bytes used by round-trip regression fixtures.
+src/test/resources/roundtrip-*.properties -text
diff --git a/src/main/java/org/codejive/properties/PropertiesParser.java b/src/main/java/org/codejive/properties/PropertiesParser.java
index c9fc70c..3c610b9 100644
--- a/src/main/java/org/codejive/properties/PropertiesParser.java
+++ b/src/main/java/org/codejive/properties/PropertiesParser.java
@@ -4,10 +4,7 @@
import java.io.Reader;
import java.util.Objects;
import java.util.Spliterators;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
-import java.util.function.Function;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
@@ -153,10 +150,24 @@ public String toString() {
private final Reader rdr;
- private Type state;
+ private enum State {
+ START,
+ COMMENT,
+ WHITESPACE,
+ KEY,
+ SEPARATOR,
+ SEPARATOR_BACKSLASH,
+ VALUE_BACKSLASH,
+ VALUE
+ }
+
+ private State state;
private int pch;
+ private boolean separatorSeen;
private StringBuilder str;
private boolean hasEscapes;
+ private boolean hasLogicalContent;
+ private boolean continuationPadding;
/**
* Constructor that takes a Reader for reading the input to parse.
@@ -166,7 +177,7 @@ public String toString() {
*/
public PropertiesParser(Reader rdr) throws IOException {
this.rdr = rdr;
- state = null;
+ state = State.START;
str = new StringBuilder();
readChar();
}
@@ -208,55 +219,95 @@ public boolean tryAdvance(Consumer super Token> action) {
* @throws IOException Thrown when any IO error occurs during parsing
*/
public Token nextToken() throws IOException {
- int ch = peekChar();
- // A property at EOF may still need its empty separator and/or value token.
- if (isEof(ch) && state == null) {
- return null;
- }
- Function isValid = (c) -> false;
- Type nextState = null;
- if (state == null) {
- if (isCommentChar(ch)) {
- state = Type.COMMENT;
- isValid = (c) -> !isEol(c) && !isEof(c);
- } else if (isWhitespaceEolChar(ch)) {
- state = Type.WHITESPACE;
- final AtomicInteger oldc = new AtomicInteger(-1);
- isValid = (c) -> isWhitespaceEolChar(c) && !isEol(oldc.getAndSet(c));
- } else {
- state = Type.KEY;
- isValid =
- (c) ->
- !isSeparatorChar(c)
- && !isWhitespaceChar(c)
- && !isEol(c)
- && !isEof(c);
- nextState = Type.SEPARATOR;
- }
- } else if (state == Type.SEPARATOR) {
- final AtomicBoolean once = new AtomicBoolean(true);
- isValid = (c) -> isWhitespaceChar(c) || (isSeparatorChar(c) && once.getAndSet(false));
- nextState = Type.VALUE;
- } else if (state == Type.VALUE) {
- isValid = (c) -> !isEol(c) && !isEof(c);
- }
while (true) {
- if (isValid.apply(ch)) {
- addChar(readChar());
- ch = peekChar();
- } else {
- String text = string();
- Token token =
- hasEscapes
- ? new Token(state, text, unescape(text))
- : new Token(state, text);
- hasEscapes = false;
- state = nextState;
- return token;
+ int ch = peekChar();
+ switch (state) {
+ case START:
+ if (isEof(ch)) {
+ return null;
+ }
+ separatorSeen = false;
+ state =
+ isCommentChar(ch)
+ ? State.COMMENT
+ : isWhitespaceEolChar(ch) ? State.WHITESPACE : State.KEY;
+ break;
+ case COMMENT:
+ if (isEol(ch) || isEof(ch)) {
+ return finishToken(Type.COMMENT, State.START);
+ }
+ addChar(readChar());
+ break;
+ case WHITESPACE:
+ if (!isWhitespaceEolChar(ch)) {
+ return finishToken(Type.WHITESPACE, State.START);
+ }
+ addChar(readChar());
+ if (isEol(ch)) {
+ return finishToken(Type.WHITESPACE, State.START);
+ }
+ break;
+ case KEY:
+ // A continuation-only prefix is formatting before a comment or blank line.
+ if (!hasLogicalContent
+ && str.length() > 0
+ && (isCommentChar(ch)
+ || isEol(ch)
+ || (isEof(ch) && continuationPadding))) {
+ return finishToken(Type.WHITESPACE, State.START);
+ }
+ if (isSeparatorChar(ch) || isWhitespaceEolChar(ch) || isEof(ch)) {
+ return finishToken(Type.KEY, State.SEPARATOR);
+ }
+ addChar(readChar());
+ break;
+ case SEPARATOR:
+ if (ch == '\\') {
+ // Consume the backslash, but defer which token owns it.
+ readChar();
+ state = State.SEPARATOR_BACKSLASH;
+ } else if (isWhitespaceChar(ch)) {
+ addChar(readChar());
+ } else if (isSeparatorChar(ch) && !separatorSeen) {
+ separatorSeen = true;
+ addChar(readChar());
+ } else {
+ return finishToken(Type.SEPARATOR, State.VALUE);
+ }
+ break;
+ case SEPARATOR_BACKSLASH:
+ if (isEol(ch)) {
+ // The pending backslash belongs to a separator continuation.
+ addChar('\\');
+ state = State.SEPARATOR;
+ } else {
+ // Carry the pending backslash into the next token, including at EOF.
+ return finishToken(Type.SEPARATOR, State.VALUE_BACKSLASH);
+ }
+ break;
+ case VALUE_BACKSLASH:
+ addChar('\\');
+ state = State.VALUE;
+ break;
+ case VALUE:
+ if (isEol(ch) || isEof(ch)) {
+ return finishToken(Type.VALUE, State.START);
+ }
+ addChar(readChar());
+ break;
}
}
}
+ private Token finishToken(Type type, State nextState) {
+ String raw = string();
+ Token token = hasEscapes ? new Token(type, raw, unescape(raw)) : new Token(type, raw);
+ hasEscapes = false;
+ hasLogicalContent = false;
+ state = nextState;
+ return token;
+ }
+
private int peekChar() {
return pch;
}
@@ -270,32 +321,30 @@ private int readChar() throws IOException {
private void addChar(int ch) throws IOException {
str.append((char) ch);
// Comments have no escapes or line continuations
- if (ch == '\\' && state != Type.COMMENT) {
+ if (ch == '\\' && state != State.COMMENT) {
hasEscapes = true;
int ch2 = readChar();
if (isEof(ch2)) {
// A backslash at the end of the input escapes nothing
+ hasLogicalContent = true;
return;
}
str.append((char) ch2);
- if (ch2 == 'u') {
- for (int i = 0; i < 4; i++) {
- int chu = readChar();
- if (!isHexDigitChar(chu)) {
- throw new IOException("Invalid unicode escape character: " + chu);
- }
- str.append((char) chu);
+ // Match the JDK's empty-line handling at EOF: a bare continuation
+ // creates an empty property, but skipped LF/indentation leaves a blank line.
+ continuationPadding = ch2 == '\r' && peekChar() == '\n';
+ readEol(ch2);
+ if (isEol(ch2)) {
+ // Decode escapes only after the entire logical token has been read.
+ while (isWhitespaceChar(peekChar())) {
+ continuationPadding = true;
+ str.append((char) readChar());
}
} else {
- readEol(ch2);
- if (isEol(ch2)) {
- // A line continuation also skips the leading whitespace of the next line
- while (isWhitespaceChar(peekChar())) {
- str.append((char) readChar());
- }
- }
+ hasLogicalContent = true;
}
} else {
+ hasLogicalContent = true;
readEol(ch);
}
}
@@ -322,6 +371,7 @@ private String string() {
* @return Decoded string
*/
public static String unescape(String escape) {
+ escape = removeContinuations(escape);
StringBuilder txt = new StringBuilder();
for (int i = 0; i < escape.length(); i++) {
char ch = escape.charAt(i);
@@ -344,6 +394,14 @@ public static String unescape(String escape) {
txt.append('\r');
break;
case 'u':
+ if (i + 4 >= escape.length()) {
+ throw new IllegalArgumentException("Malformed Unicode escape");
+ }
+ for (int j = i + 1; j <= i + 4; j++) {
+ if (!isHexDigitChar(escape.charAt(j))) {
+ throw new IllegalArgumentException("Malformed Unicode escape");
+ }
+ }
String num = escape.substring(i + 1, i + 5);
txt.append((char) Integer.parseInt(num, 16));
i += 4;
@@ -372,6 +430,30 @@ && isWhitespaceChar(ch = escape.charAt(i + 1))) {
return txt.toString();
}
+ private static String removeContinuations(String raw) {
+ StringBuilder logical = new StringBuilder();
+ for (int i = 0; i < raw.length(); i++) {
+ char ch = raw.charAt(i);
+ if (ch == '\\' && i + 1 < raw.length()) {
+ char next = raw.charAt(++i);
+ if (isEol(next)) {
+ if (next == '\r' && i + 1 < raw.length() && raw.charAt(i + 1) == '\n') {
+ i++;
+ }
+ while (i + 1 < raw.length() && isWhitespaceChar(raw.charAt(i + 1))) {
+ i++;
+ }
+ } else {
+ // Consume escape pairs together to preserve backslash parity.
+ logical.append(ch).append(next);
+ }
+ } else {
+ logical.append(ch);
+ }
+ }
+ return logical.toString();
+ }
+
public static boolean isSeparatorChar(int ch) {
return ch == '=' || ch == ':';
}
@@ -389,8 +471,7 @@ public static boolean isCommentChar(int ch) {
}
private static boolean isHexDigitChar(int ch) {
- int uch = Character.toUpperCase(ch);
- return Character.isDigit(ch) || (uch >= 'A' && uch <= 'F');
+ return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f');
}
private static boolean isEol(int ch) {
diff --git a/src/test/java/org/codejive/properties/TestProperties.java b/src/test/java/org/codejive/properties/TestProperties.java
index 5570a79..653178f 100644
--- a/src/test/java/org/codejive/properties/TestProperties.java
+++ b/src/test/java/org/codejive/properties/TestProperties.java
@@ -1428,6 +1428,66 @@ void testPutLastLineWithFinalEol() throws IOException {
assertThat(p.getComment("put")).isEmpty();
}
+ @Test
+ void testRoundTripContinuedSeparators() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-separators.properties");
+ assertThat(p)
+ .containsEntry("equals", "value")
+ .containsEntry("colon", "value")
+ .containsEntry("alreadySeparated", ":value");
+ assertThat(p).hasSize(3);
+ }
+
+ @Test
+ void testRoundTripContinuedUnicodeKey() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-unicode-key.properties");
+ assertThat(p).containsEntry("\u1234", "value").hasSize(1);
+ }
+
+ @Test
+ void testRoundTripContinuedUnicodeValue() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-unicode-value.properties");
+ assertThat(p).containsEntry("key", "\u1234").hasSize(1);
+ }
+
+ @Test
+ void testRoundTripEmptyContinuationBeforeComment() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-comment.properties");
+ assertThat(p).containsEntry("key", "value").hasSize(1);
+ }
+
+ @Test
+ void testRoundTripEmptyContinuationBeforeBlankLine() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-blank.properties");
+ assertThat(p).containsEntry("key", "value").hasSize(1);
+ }
+
+ @Test
+ void testRoundTripPendingBackslashAtEof() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-backslash-eof.properties");
+ assertThat(p).containsEntry("key", "").hasSize(1);
+ }
+
+ @Test
+ void testRoundTripEmptyContinuationAtEof() throws IOException, URISyntaxException {
+ Properties p = assertByteExactRoundTrip("/roundtrip-continuation-eof.properties");
+ assertThat(p).isEmpty();
+ }
+
+ private Properties assertByteExactRoundTrip(String resource)
+ throws IOException, URISyntaxException {
+ Path file = getResource(resource);
+ byte[] original = Files.readAllBytes(file);
+ Properties p = new Properties();
+ try (InputStream input = Files.newInputStream(file)) {
+ p.load(input);
+ }
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ p.store(output);
+ assertThat(output.toByteArray()).as("Exact bytes of %s", resource).isEqualTo(original);
+ return p;
+ }
+
private void expectStoreText(Properties props, String expectedText) throws IOException {
StringWriter sw = new StringWriter();
props.store(sw);
diff --git a/src/test/java/org/codejive/properties/TestPropertiesParser.java b/src/test/java/org/codejive/properties/TestPropertiesParser.java
index e9ab7cc..3a1f1ce 100644
--- a/src/test/java/org/codejive/properties/TestPropertiesParser.java
+++ b/src/test/java/org/codejive/properties/TestPropertiesParser.java
@@ -171,4 +171,112 @@ void testBackslashAtEof() throws IOException {
new Token(Type.SEPARATOR, "="),
new Token(Type.VALUE, "bar\\", "bar"));
}
+
+ @Test
+ void testPendingBackslashBelongsToValue() throws IOException {
+ for (String separator : new String[] {" ", "=", " : ", " \\\r\n = "}) {
+ for (String rawValue :
+ new String[] {"\\", "\\t", "\\=value", "\\ value", "\\\\tail", "\\u1234"}) {
+ String input = "key" + separator + rawValue;
+ PropertiesParser parser = new PropertiesParser(new StringReader(input));
+ assertThat(parser.nextToken()).isEqualTo(new Token(Type.KEY, "key"));
+ Token separatorToken = parser.nextToken();
+ assertThat(separatorToken.getType()).isEqualTo(Type.SEPARATOR);
+ assertThat(separatorToken.getRaw()).isEqualTo(separator);
+ Token valueToken = parser.nextToken();
+ assertThat(valueToken.getType()).isEqualTo(Type.VALUE);
+ assertThat(valueToken.getRaw()).isEqualTo(rawValue);
+ assertThat(valueToken.getText()).isEqualTo(PropertiesParser.unescape(rawValue));
+ assertThat(parser.nextToken()).isNull();
+ assertThat(parser.nextToken()).isNull();
+ assertLoadsLikeJava(input);
+ }
+ }
+ }
+
+ @Test
+ void testContinuedSeparators() throws IOException {
+ for (String eol : new String[] {"\n", "\r", "\r\n"}) {
+ for (String separator : new String[] {"=", ":"}) {
+ assertLoadsLikeJava("key \\" + eol + " \t\f" + separator + " value");
+ assertLoadsLikeJava("key \\" + eol + " \\" + eol + separator + " value");
+ assertLoadsLikeJava("key=" + "\\" + eol + " :value");
+ assertLoadsLikeJava("key" + separator + "\\" + eol + "\\" + eol + " :value");
+ assertLoadsLikeJava("key" + separator + "\\" + eol);
+ }
+ }
+ }
+
+ @Test
+ void testContinuedUnicodeEscapes() throws IOException {
+ for (String eol : new String[] {"\n", "\r", "\r\n"}) {
+ for (int split = 2; split < 6; split++) {
+ String escape = "\\u1234";
+ String continued =
+ escape.substring(0, split) + "\\" + eol + " \t\f" + escape.substring(split);
+ assertLoadsLikeJava(continued + "=" + continued);
+ assertThat(PropertiesParser.unescape(continued)).isEqualTo("\u1234");
+ }
+ }
+ }
+
+ @Test
+ void testInvalidUnicodeEscapes() {
+ for (String escape :
+ new String[] {"\\u1234", "\\u١٢٣٤", "\\uu1234", "\\u123", "\\u12x4", "\\u+123"}) {
+ for (String input : new String[] {"key=" + escape, escape + "=value"}) {
+ assertThatThrownBy(() -> new Properties().load(new StringReader(input)))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+ assertThatThrownBy(() -> PropertiesParser.unescape(escape))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+ }
+
+ @Test
+ void testEmptyContinuationPrefixes() throws IOException {
+ for (String eol : new String[] {"\n", "\r", "\r\n"}) {
+ for (String tail :
+ new String[] {"#comment", "!comment", "", " \t\f", "key=value", "=value"}) {
+ assertLoadsLikeJava("\\" + eol + tail + eol + "k=v");
+ assertLoadsLikeJava(" \\" + eol + "\t\\" + eol + tail + eol + "k=v");
+ }
+ assertLoadsLikeJava("\\" + eol);
+ assertLoadsLikeJava("\\" + eol + " \t\f");
+ assertLoadsLikeJava("\\" + eol + "\\\n");
+ assertLoadsLikeJava("key=one\\" + eol + " #two");
+ }
+ assertLoadsLikeJava("\\");
+ }
+
+ @Test
+ void testBackslashParity() throws IOException {
+ for (String eol : new String[] {"\n", "\r", "\r\n"}) {
+ for (int count = 0; count < 8; count++) {
+ String slashes = new String(new char[count]).replace('\0', '\\');
+ for (String prefix : new String[] {"key", "key ", "key=", "key=x", "#comment"}) {
+ for (String tail :
+ new String[] {" value", " = value", "#comment", "", "\nnext=v"}) {
+ assertLoadsLikeJava(prefix + slashes + eol + tail);
+ }
+ }
+ }
+ }
+ }
+
+ private void assertLoadsLikeJava(String input) throws IOException {
+ java.util.Properties expected = new java.util.Properties();
+ expected.load(new StringReader(input));
+ Properties actual = new Properties();
+ actual.load(new StringReader(input));
+ assertThat(actual).as("Input: %s", input).isEqualTo(expected);
+ String raw =
+ PropertiesParser.tokens(new StringReader(input))
+ .map(Token::getRaw)
+ .collect(Collectors.joining());
+ assertThat(raw).isEqualTo(input);
+ java.io.StringWriter writer = new java.io.StringWriter();
+ actual.store(writer);
+ assertThat(writer.toString()).isEqualTo(input);
+ }
}
diff --git a/src/test/resources/roundtrip-backslash-eof.properties b/src/test/resources/roundtrip-backslash-eof.properties
new file mode 100644
index 0000000..7f2cfd4
--- /dev/null
+++ b/src/test/resources/roundtrip-backslash-eof.properties
@@ -0,0 +1 @@
+key \
\ No newline at end of file
diff --git a/src/test/resources/roundtrip-blank.properties b/src/test/resources/roundtrip-blank.properties
new file mode 100644
index 0000000..8737fae
--- /dev/null
+++ b/src/test/resources/roundtrip-blank.properties
@@ -0,0 +1,5 @@
+\
+
+\
+
+key=value
\ No newline at end of file
diff --git a/src/test/resources/roundtrip-comment.properties b/src/test/resources/roundtrip-comment.properties
new file mode 100644
index 0000000..bde2f20
--- /dev/null
+++ b/src/test/resources/roundtrip-comment.properties
@@ -0,0 +1,5 @@
+\
+ # comment with a trailing backslash \
+\
+ ! another comment
+key=value
\ No newline at end of file
diff --git a/src/test/resources/roundtrip-continuation-eof.properties b/src/test/resources/roundtrip-continuation-eof.properties
new file mode 100644
index 0000000..a30732a
--- /dev/null
+++ b/src/test/resources/roundtrip-continuation-eof.properties
@@ -0,0 +1,2 @@
+\
+
\ No newline at end of file
diff --git a/src/test/resources/roundtrip-separators.properties b/src/test/resources/roundtrip-separators.properties
new file mode 100644
index 0000000..c4fecc0
--- /dev/null
+++ b/src/test/resources/roundtrip-separators.properties
@@ -0,0 +1,6 @@
+equals \
+ = value
+colon \
:\
+ value
+alreadySeparated=\
+ :value
\ No newline at end of file
diff --git a/src/test/resources/roundtrip-unicode-key.properties b/src/test/resources/roundtrip-unicode-key.properties
new file mode 100644
index 0000000..f370858
--- /dev/null
+++ b/src/test/resources/roundtrip-unicode-key.properties
@@ -0,0 +1,2 @@
+\u12\
+ 34=value
diff --git a/src/test/resources/roundtrip-unicode-value.properties b/src/test/resources/roundtrip-unicode-value.properties
new file mode 100644
index 0000000..2626ec8
--- /dev/null
+++ b/src/test/resources/roundtrip-unicode-value.properties
@@ -0,0 +1,3 @@
+key=\u1\
+ 2\
+ 3\
4
\ No newline at end of file