Fix handling of empty C++ comments in JSON parsing
When AOS parses a JSON file, it first performs some preprocessing
to remove comments before passing the document to the real parser.
The removal of C++ style comments using // had a corner case. If
you had a comment that had no content, just //\n, then the \n was
being swallowed and the next line was commented out too.
This change reorganizes the handling of C++ comments to allow this
to be correctly handled, including if the document ends with an
empty comment.
Change-Id: Id2874faafd21e68d8926c42c9cda3581c02069a7
Signed-off-by: James Kuszmaul <james.kuszmaul@bluerivertech.com>
diff --git a/aos/json_to_flatbuffer_test.cc b/aos/json_to_flatbuffer_test.cc
index b6b2c04..7c69acd 100644
--- a/aos/json_to_flatbuffer_test.cc
+++ b/aos/json_to_flatbuffer_test.cc
@@ -348,6 +348,36 @@
} // foo)",
"{ \"vector_foo_double\": [ 9.0, 7.0, 1.0 ] }",
TestReflection::kNo));
+
+ // Test empty comment on its own line doesn't remove the next line.
+ EXPECT_TRUE(JsonAndBack(R"({
+ //
+ "vector_foo_double": [ 9, 7, 1 ], // foo
+ "vector_foo_float": [ 3, 1, 4 ]
+} // foo)",
+ "{ \"vector_foo_float\": [ 3.0, 1.0, 4.0 ], "
+ "\"vector_foo_double\": [ 9.0, 7.0, 1.0 ] }",
+ TestReflection::kNo));
+
+ // Test empty comment at end of line doesn't remove the next line.
+ EXPECT_TRUE(JsonAndBack(R"({
+ // foo
+ "vector_foo_double": [ 2, 7, 1 ], //
+ "vector_foo_float": [ 3, 1, 4 ]
+} // foo)",
+ "{ \"vector_foo_float\": [ 3.0, 1.0, 4.0 ], "
+ "\"vector_foo_double\": [ 2.0, 7.0, 1.0 ] }",
+ TestReflection::kNo));
+
+ // Test empty comment at end of document doesn't cause error.
+ EXPECT_TRUE(JsonAndBack(R"({
+ // foo
+ "vector_foo_double": [ 5, 6, 7 ], // foo
+ "vector_foo_float": [ 7, 8, 9 ]
+} //)",
+ "{ \"vector_foo_float\": [ 7.0, 8.0, 9.0 ], "
+ "\"vector_foo_double\": [ 5.0, 6.0, 7.0 ] }",
+ TestReflection::kNo));
}
// Tests that mixed style comments get stripped.
diff --git a/aos/json_tokenizer.cc b/aos/json_tokenizer.cc
index d277c1e..32c9247 100644
--- a/aos/json_tokenizer.cc
+++ b/aos/json_tokenizer.cc
@@ -27,14 +27,17 @@
// C++ style comment. Keep consuming chars until newline, or until the
// end of the file if this is the last line (no newline at end of file).
while (true) {
- ConsumeChar();
+ // First check if we are at the end of the file.
if (AtEnd()) {
return;
}
+ // Then check if we are at the end of the line.
if (Char() == '\n') {
++linenumber_;
break;
}
+ // Advance to next character and repeat.
+ ConsumeChar();
}
} else {
// There is no fail. Once we are out of whitespace (including 0 of it),