fix: solve issue 2181

simdjson · May 12, 2024 · 2848701 · 2848701
1 parent 6ea7739
commit 2848701
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 5 deletions.
diff --git a/include/simdjson/dom/document_stream-inl.h b/include/simdjson/dom/document_stream-inl.h
@@ -224,10 +224,10 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
   } else {
     size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index];
     size_t svlen = next_doc_index - current_index();
-    if(svlen > 1) {
+    while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
       svlen--;
     }
-    return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+    return std::string_view(start, svlen);
   }
 }
 

diff --git a/include/simdjson/generic/ondemand/document_stream-inl.h b/include/simdjson/generic/ondemand/document_stream-inl.h
@@ -348,10 +348,11 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
           auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index];
           // normally the length would be next_index - current_index() - 1, except for the last document
           size_t svlen = next_index - current_index();
-          if(svlen > 1) {
+          const char *start = reinterpret_cast<const char*>(stream->buf) + current_index();
+          while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
             svlen--;
           }
-          return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+          return std::string_view(start, svlen);
         }
     }
     cur_struct_index++;

diff --git a/tests/dom/document_stream_tests.cpp b/tests/dom/document_stream_tests.cpp
@@ -249,6 +249,27 @@ namespace document_stream_tests {
       TEST_SUCCEED();
   }
 
+  bool issue2181() {
+      TEST_START();
+      auto json = R"(1 2 34)"_padded;
+      simdjson::dom::parser parser;
+      simdjson::dom::document_stream stream;
+      ASSERT_SUCCESS(parser.parse_many(json).get(stream));
+      auto i = stream.begin();
+      size_t count{0};
+      std::vector<size_t> indexes = { 0, 2, 4 };
+      std::vector<std::string_view> expected = { "1", "2", "34" };
+
+      for(; i != stream.end(); ++i) {
+          auto doc = *i;
+          ASSERT_SUCCESS(doc);
+          ASSERT_TRUE(count < 3);
+          ASSERT_EQUAL(i.current_index(), indexes[count]);
+          ASSERT_EQUAL(i.source(), expected[count]);
+          count++;
+      }
+      TEST_SUCCEED();
+  }
   bool issue1310() {
     std::cout << "Running " << __func__ << std::endl;
     // hex  : 20 20 5B 20 33 2C 31 5D 20 22 22 22 22 22 22 22 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
@@ -963,7 +984,8 @@ namespace document_stream_tests {
   }
 
   bool run() {
-    return issue2170() &&
+    return issue2181() &&
+           issue2170() &&
            skipbom() &&
            fuzzaccess() &&
            baby_fuzzer() &&

diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp
@@ -217,6 +217,27 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool issue2181() {
+        TEST_START();
+        auto json = R"(1 2 34)"_padded;
+        ondemand::parser parser;
+        ondemand::document_stream stream;
+        ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
+        auto i = stream.begin();
+        size_t count{0};
+        std::vector<size_t> indexes = { 0, 2, 4 };
+        std::vector<std::string_view> expected = { "1", "2", "34" };
+
+        for(; i != stream.end(); ++i) {
+            ASSERT_SUCCESS(i.error());
+            ASSERT_TRUE(count < 3);
+            ASSERT_EQUAL(i.current_index(), indexes[count]);
+            ASSERT_EQUAL(i.source(), expected[count]);
+            count++;
+        }
+        TEST_SUCCEED();
+    }
+
     bool issue1977() {
         TEST_START();
         std::string json = R"( 1111 })";
@@ -902,6 +923,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            issue2181() &&
             issue2170() &&
             issue2137() &&
             skipbom() &&