diff --git a/.gitattributes b/.gitattributes index bcc0efa..bc4cd17 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,4 @@ * text eol=lf src/test/resources/testcrlf* text eol=crlf +# Preserve the exact line endings and EOF bytes used by round-trip regression fixtures. +src/test/resources/roundtrip-*.properties -text diff --git a/src/main/java/org/codejive/properties/PropertiesParser.java b/src/main/java/org/codejive/properties/PropertiesParser.java index c9fc70c..3c610b9 100644 --- a/src/main/java/org/codejive/properties/PropertiesParser.java +++ b/src/main/java/org/codejive/properties/PropertiesParser.java @@ -4,10 +4,7 @@ import java.io.Reader; import java.util.Objects; import java.util.Spliterators; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import java.util.function.Function; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -153,10 +150,24 @@ public String toString() { private final Reader rdr; - private Type state; + private enum State { + START, + COMMENT, + WHITESPACE, + KEY, + SEPARATOR, + SEPARATOR_BACKSLASH, + VALUE_BACKSLASH, + VALUE + } + + private State state; private int pch; + private boolean separatorSeen; private StringBuilder str; private boolean hasEscapes; + private boolean hasLogicalContent; + private boolean continuationPadding; /** * Constructor that takes a Reader for reading the input to parse. @@ -166,7 +177,7 @@ public String toString() { */ public PropertiesParser(Reader rdr) throws IOException { this.rdr = rdr; - state = null; + state = State.START; str = new StringBuilder(); readChar(); } @@ -208,55 +219,95 @@ public boolean tryAdvance(Consumer action) { * @throws IOException Thrown when any IO error occurs during parsing */ public Token nextToken() throws IOException { - int ch = peekChar(); - // A property at EOF may still need its empty separator and/or value token. - if (isEof(ch) && state == null) { - return null; - } - Function isValid = (c) -> false; - Type nextState = null; - if (state == null) { - if (isCommentChar(ch)) { - state = Type.COMMENT; - isValid = (c) -> !isEol(c) && !isEof(c); - } else if (isWhitespaceEolChar(ch)) { - state = Type.WHITESPACE; - final AtomicInteger oldc = new AtomicInteger(-1); - isValid = (c) -> isWhitespaceEolChar(c) && !isEol(oldc.getAndSet(c)); - } else { - state = Type.KEY; - isValid = - (c) -> - !isSeparatorChar(c) - && !isWhitespaceChar(c) - && !isEol(c) - && !isEof(c); - nextState = Type.SEPARATOR; - } - } else if (state == Type.SEPARATOR) { - final AtomicBoolean once = new AtomicBoolean(true); - isValid = (c) -> isWhitespaceChar(c) || (isSeparatorChar(c) && once.getAndSet(false)); - nextState = Type.VALUE; - } else if (state == Type.VALUE) { - isValid = (c) -> !isEol(c) && !isEof(c); - } while (true) { - if (isValid.apply(ch)) { - addChar(readChar()); - ch = peekChar(); - } else { - String text = string(); - Token token = - hasEscapes - ? new Token(state, text, unescape(text)) - : new Token(state, text); - hasEscapes = false; - state = nextState; - return token; + int ch = peekChar(); + switch (state) { + case START: + if (isEof(ch)) { + return null; + } + separatorSeen = false; + state = + isCommentChar(ch) + ? State.COMMENT + : isWhitespaceEolChar(ch) ? State.WHITESPACE : State.KEY; + break; + case COMMENT: + if (isEol(ch) || isEof(ch)) { + return finishToken(Type.COMMENT, State.START); + } + addChar(readChar()); + break; + case WHITESPACE: + if (!isWhitespaceEolChar(ch)) { + return finishToken(Type.WHITESPACE, State.START); + } + addChar(readChar()); + if (isEol(ch)) { + return finishToken(Type.WHITESPACE, State.START); + } + break; + case KEY: + // A continuation-only prefix is formatting before a comment or blank line. + if (!hasLogicalContent + && str.length() > 0 + && (isCommentChar(ch) + || isEol(ch) + || (isEof(ch) && continuationPadding))) { + return finishToken(Type.WHITESPACE, State.START); + } + if (isSeparatorChar(ch) || isWhitespaceEolChar(ch) || isEof(ch)) { + return finishToken(Type.KEY, State.SEPARATOR); + } + addChar(readChar()); + break; + case SEPARATOR: + if (ch == '\\') { + // Consume the backslash, but defer which token owns it. + readChar(); + state = State.SEPARATOR_BACKSLASH; + } else if (isWhitespaceChar(ch)) { + addChar(readChar()); + } else if (isSeparatorChar(ch) && !separatorSeen) { + separatorSeen = true; + addChar(readChar()); + } else { + return finishToken(Type.SEPARATOR, State.VALUE); + } + break; + case SEPARATOR_BACKSLASH: + if (isEol(ch)) { + // The pending backslash belongs to a separator continuation. + addChar('\\'); + state = State.SEPARATOR; + } else { + // Carry the pending backslash into the next token, including at EOF. + return finishToken(Type.SEPARATOR, State.VALUE_BACKSLASH); + } + break; + case VALUE_BACKSLASH: + addChar('\\'); + state = State.VALUE; + break; + case VALUE: + if (isEol(ch) || isEof(ch)) { + return finishToken(Type.VALUE, State.START); + } + addChar(readChar()); + break; } } } + private Token finishToken(Type type, State nextState) { + String raw = string(); + Token token = hasEscapes ? new Token(type, raw, unescape(raw)) : new Token(type, raw); + hasEscapes = false; + hasLogicalContent = false; + state = nextState; + return token; + } + private int peekChar() { return pch; } @@ -270,32 +321,30 @@ private int readChar() throws IOException { private void addChar(int ch) throws IOException { str.append((char) ch); // Comments have no escapes or line continuations - if (ch == '\\' && state != Type.COMMENT) { + if (ch == '\\' && state != State.COMMENT) { hasEscapes = true; int ch2 = readChar(); if (isEof(ch2)) { // A backslash at the end of the input escapes nothing + hasLogicalContent = true; return; } str.append((char) ch2); - if (ch2 == 'u') { - for (int i = 0; i < 4; i++) { - int chu = readChar(); - if (!isHexDigitChar(chu)) { - throw new IOException("Invalid unicode escape character: " + chu); - } - str.append((char) chu); + // Match the JDK's empty-line handling at EOF: a bare continuation + // creates an empty property, but skipped LF/indentation leaves a blank line. + continuationPadding = ch2 == '\r' && peekChar() == '\n'; + readEol(ch2); + if (isEol(ch2)) { + // Decode escapes only after the entire logical token has been read. + while (isWhitespaceChar(peekChar())) { + continuationPadding = true; + str.append((char) readChar()); } } else { - readEol(ch2); - if (isEol(ch2)) { - // A line continuation also skips the leading whitespace of the next line - while (isWhitespaceChar(peekChar())) { - str.append((char) readChar()); - } - } + hasLogicalContent = true; } } else { + hasLogicalContent = true; readEol(ch); } } @@ -322,6 +371,7 @@ private String string() { * @return Decoded string */ public static String unescape(String escape) { + escape = removeContinuations(escape); StringBuilder txt = new StringBuilder(); for (int i = 0; i < escape.length(); i++) { char ch = escape.charAt(i); @@ -344,6 +394,14 @@ public static String unescape(String escape) { txt.append('\r'); break; case 'u': + if (i + 4 >= escape.length()) { + throw new IllegalArgumentException("Malformed Unicode escape"); + } + for (int j = i + 1; j <= i + 4; j++) { + if (!isHexDigitChar(escape.charAt(j))) { + throw new IllegalArgumentException("Malformed Unicode escape"); + } + } String num = escape.substring(i + 1, i + 5); txt.append((char) Integer.parseInt(num, 16)); i += 4; @@ -372,6 +430,30 @@ && isWhitespaceChar(ch = escape.charAt(i + 1))) { return txt.toString(); } + private static String removeContinuations(String raw) { + StringBuilder logical = new StringBuilder(); + for (int i = 0; i < raw.length(); i++) { + char ch = raw.charAt(i); + if (ch == '\\' && i + 1 < raw.length()) { + char next = raw.charAt(++i); + if (isEol(next)) { + if (next == '\r' && i + 1 < raw.length() && raw.charAt(i + 1) == '\n') { + i++; + } + while (i + 1 < raw.length() && isWhitespaceChar(raw.charAt(i + 1))) { + i++; + } + } else { + // Consume escape pairs together to preserve backslash parity. + logical.append(ch).append(next); + } + } else { + logical.append(ch); + } + } + return logical.toString(); + } + public static boolean isSeparatorChar(int ch) { return ch == '=' || ch == ':'; } @@ -389,8 +471,7 @@ public static boolean isCommentChar(int ch) { } private static boolean isHexDigitChar(int ch) { - int uch = Character.toUpperCase(ch); - return Character.isDigit(ch) || (uch >= 'A' && uch <= 'F'); + return (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f'); } private static boolean isEol(int ch) { diff --git a/src/test/java/org/codejive/properties/TestProperties.java b/src/test/java/org/codejive/properties/TestProperties.java index 5570a79..653178f 100644 --- a/src/test/java/org/codejive/properties/TestProperties.java +++ b/src/test/java/org/codejive/properties/TestProperties.java @@ -1428,6 +1428,66 @@ void testPutLastLineWithFinalEol() throws IOException { assertThat(p.getComment("put")).isEmpty(); } + @Test + void testRoundTripContinuedSeparators() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-separators.properties"); + assertThat(p) + .containsEntry("equals", "value") + .containsEntry("colon", "value") + .containsEntry("alreadySeparated", ":value"); + assertThat(p).hasSize(3); + } + + @Test + void testRoundTripContinuedUnicodeKey() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-unicode-key.properties"); + assertThat(p).containsEntry("\u1234", "value").hasSize(1); + } + + @Test + void testRoundTripContinuedUnicodeValue() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-unicode-value.properties"); + assertThat(p).containsEntry("key", "\u1234").hasSize(1); + } + + @Test + void testRoundTripEmptyContinuationBeforeComment() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-comment.properties"); + assertThat(p).containsEntry("key", "value").hasSize(1); + } + + @Test + void testRoundTripEmptyContinuationBeforeBlankLine() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-blank.properties"); + assertThat(p).containsEntry("key", "value").hasSize(1); + } + + @Test + void testRoundTripPendingBackslashAtEof() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-backslash-eof.properties"); + assertThat(p).containsEntry("key", "").hasSize(1); + } + + @Test + void testRoundTripEmptyContinuationAtEof() throws IOException, URISyntaxException { + Properties p = assertByteExactRoundTrip("/roundtrip-continuation-eof.properties"); + assertThat(p).isEmpty(); + } + + private Properties assertByteExactRoundTrip(String resource) + throws IOException, URISyntaxException { + Path file = getResource(resource); + byte[] original = Files.readAllBytes(file); + Properties p = new Properties(); + try (InputStream input = Files.newInputStream(file)) { + p.load(input); + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + p.store(output); + assertThat(output.toByteArray()).as("Exact bytes of %s", resource).isEqualTo(original); + return p; + } + private void expectStoreText(Properties props, String expectedText) throws IOException { StringWriter sw = new StringWriter(); props.store(sw); diff --git a/src/test/java/org/codejive/properties/TestPropertiesParser.java b/src/test/java/org/codejive/properties/TestPropertiesParser.java index e9ab7cc..3a1f1ce 100644 --- a/src/test/java/org/codejive/properties/TestPropertiesParser.java +++ b/src/test/java/org/codejive/properties/TestPropertiesParser.java @@ -171,4 +171,112 @@ void testBackslashAtEof() throws IOException { new Token(Type.SEPARATOR, "="), new Token(Type.VALUE, "bar\\", "bar")); } + + @Test + void testPendingBackslashBelongsToValue() throws IOException { + for (String separator : new String[] {" ", "=", " : ", " \\\r\n = "}) { + for (String rawValue : + new String[] {"\\", "\\t", "\\=value", "\\ value", "\\\\tail", "\\u1234"}) { + String input = "key" + separator + rawValue; + PropertiesParser parser = new PropertiesParser(new StringReader(input)); + assertThat(parser.nextToken()).isEqualTo(new Token(Type.KEY, "key")); + Token separatorToken = parser.nextToken(); + assertThat(separatorToken.getType()).isEqualTo(Type.SEPARATOR); + assertThat(separatorToken.getRaw()).isEqualTo(separator); + Token valueToken = parser.nextToken(); + assertThat(valueToken.getType()).isEqualTo(Type.VALUE); + assertThat(valueToken.getRaw()).isEqualTo(rawValue); + assertThat(valueToken.getText()).isEqualTo(PropertiesParser.unescape(rawValue)); + assertThat(parser.nextToken()).isNull(); + assertThat(parser.nextToken()).isNull(); + assertLoadsLikeJava(input); + } + } + } + + @Test + void testContinuedSeparators() throws IOException { + for (String eol : new String[] {"\n", "\r", "\r\n"}) { + for (String separator : new String[] {"=", ":"}) { + assertLoadsLikeJava("key \\" + eol + " \t\f" + separator + " value"); + assertLoadsLikeJava("key \\" + eol + " \\" + eol + separator + " value"); + assertLoadsLikeJava("key=" + "\\" + eol + " :value"); + assertLoadsLikeJava("key" + separator + "\\" + eol + "\\" + eol + " :value"); + assertLoadsLikeJava("key" + separator + "\\" + eol); + } + } + } + + @Test + void testContinuedUnicodeEscapes() throws IOException { + for (String eol : new String[] {"\n", "\r", "\r\n"}) { + for (int split = 2; split < 6; split++) { + String escape = "\\u1234"; + String continued = + escape.substring(0, split) + "\\" + eol + " \t\f" + escape.substring(split); + assertLoadsLikeJava(continued + "=" + continued); + assertThat(PropertiesParser.unescape(continued)).isEqualTo("\u1234"); + } + } + } + + @Test + void testInvalidUnicodeEscapes() { + for (String escape : + new String[] {"\\u1234", "\\u١٢٣٤", "\\uu1234", "\\u123", "\\u12x4", "\\u+123"}) { + for (String input : new String[] {"key=" + escape, escape + "=value"}) { + assertThatThrownBy(() -> new Properties().load(new StringReader(input))) + .isInstanceOf(IllegalArgumentException.class); + } + assertThatThrownBy(() -> PropertiesParser.unescape(escape)) + .isInstanceOf(IllegalArgumentException.class); + } + } + + @Test + void testEmptyContinuationPrefixes() throws IOException { + for (String eol : new String[] {"\n", "\r", "\r\n"}) { + for (String tail : + new String[] {"#comment", "!comment", "", " \t\f", "key=value", "=value"}) { + assertLoadsLikeJava("\\" + eol + tail + eol + "k=v"); + assertLoadsLikeJava(" \\" + eol + "\t\\" + eol + tail + eol + "k=v"); + } + assertLoadsLikeJava("\\" + eol); + assertLoadsLikeJava("\\" + eol + " \t\f"); + assertLoadsLikeJava("\\" + eol + "\\\n"); + assertLoadsLikeJava("key=one\\" + eol + " #two"); + } + assertLoadsLikeJava("\\"); + } + + @Test + void testBackslashParity() throws IOException { + for (String eol : new String[] {"\n", "\r", "\r\n"}) { + for (int count = 0; count < 8; count++) { + String slashes = new String(new char[count]).replace('\0', '\\'); + for (String prefix : new String[] {"key", "key ", "key=", "key=x", "#comment"}) { + for (String tail : + new String[] {" value", " = value", "#comment", "", "\nnext=v"}) { + assertLoadsLikeJava(prefix + slashes + eol + tail); + } + } + } + } + } + + private void assertLoadsLikeJava(String input) throws IOException { + java.util.Properties expected = new java.util.Properties(); + expected.load(new StringReader(input)); + Properties actual = new Properties(); + actual.load(new StringReader(input)); + assertThat(actual).as("Input: %s", input).isEqualTo(expected); + String raw = + PropertiesParser.tokens(new StringReader(input)) + .map(Token::getRaw) + .collect(Collectors.joining()); + assertThat(raw).isEqualTo(input); + java.io.StringWriter writer = new java.io.StringWriter(); + actual.store(writer); + assertThat(writer.toString()).isEqualTo(input); + } } diff --git a/src/test/resources/roundtrip-backslash-eof.properties b/src/test/resources/roundtrip-backslash-eof.properties new file mode 100644 index 0000000..7f2cfd4 --- /dev/null +++ b/src/test/resources/roundtrip-backslash-eof.properties @@ -0,0 +1 @@ +key \ \ No newline at end of file diff --git a/src/test/resources/roundtrip-blank.properties b/src/test/resources/roundtrip-blank.properties new file mode 100644 index 0000000..8737fae --- /dev/null +++ b/src/test/resources/roundtrip-blank.properties @@ -0,0 +1,5 @@ +\ + +\ + +key=value \ No newline at end of file diff --git a/src/test/resources/roundtrip-comment.properties b/src/test/resources/roundtrip-comment.properties new file mode 100644 index 0000000..bde2f20 --- /dev/null +++ b/src/test/resources/roundtrip-comment.properties @@ -0,0 +1,5 @@ +\ + # comment with a trailing backslash \ +\ + ! another comment +key=value \ No newline at end of file diff --git a/src/test/resources/roundtrip-continuation-eof.properties b/src/test/resources/roundtrip-continuation-eof.properties new file mode 100644 index 0000000..a30732a --- /dev/null +++ b/src/test/resources/roundtrip-continuation-eof.properties @@ -0,0 +1,2 @@ +\ + \ No newline at end of file diff --git a/src/test/resources/roundtrip-separators.properties b/src/test/resources/roundtrip-separators.properties new file mode 100644 index 0000000..c4fecc0 --- /dev/null +++ b/src/test/resources/roundtrip-separators.properties @@ -0,0 +1,6 @@ +equals \ + = value +colon \ :\ + value +alreadySeparated=\ + :value \ No newline at end of file diff --git a/src/test/resources/roundtrip-unicode-key.properties b/src/test/resources/roundtrip-unicode-key.properties new file mode 100644 index 0000000..f370858 --- /dev/null +++ b/src/test/resources/roundtrip-unicode-key.properties @@ -0,0 +1,2 @@ +\u12\ + 34=value diff --git a/src/test/resources/roundtrip-unicode-value.properties b/src/test/resources/roundtrip-unicode-value.properties new file mode 100644 index 0000000..2626ec8 --- /dev/null +++ b/src/test/resources/roundtrip-unicode-value.properties @@ -0,0 +1,3 @@ +key=\u1\ + 2\ + 3\ 4 \ No newline at end of file