issue 2170 (#2172)

simdjson · Apr 24, 2024 · 8519e24 · 8519e24
1 parent fffb627
commit 8519e24
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 4 deletions.
diff --git a/include/simdjson/dom/document_stream-inl.h b/include/simdjson/dom/document_stream-inl.h
@@ -223,7 +223,11 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
     return std::string_view(start, next_doc_index - current_index() + 1);
   } else {
     size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index];
-    return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), next_doc_index - current_index() - 1);
+    size_t svlen = next_doc_index - current_index();
+    if(svlen > 1) {
+      svlen--;
+    }
+    return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
   }
 }
 

diff --git a/include/simdjson/generic/ondemand/document_stream-inl.h b/include/simdjson/generic/ondemand/document_stream-inl.h
@@ -342,9 +342,17 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
         depth--;
         break;
       default:    // Scalar value document
-        // TODO: Remove any trailing whitespaces
+        // TODO: We could remove trailing whitespaces
         // This returns a string spanning from start of value to the beginning of the next document (excluded)
-        return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1);
+        {
+          auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index];
+          // normally the length would be next_index - current_index() - 1, except for the last document
+          size_t svlen = next_index - current_index();
+          if(svlen > 1) {
+            svlen--;
+          }
+          return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), svlen);
+        }
     }
     cur_struct_index++;
   }

diff --git a/tests/dom/document_stream_tests.cpp b/tests/dom/document_stream_tests.cpp
@@ -227,6 +227,28 @@ namespace document_stream_tests {
     return true;
   }
 
+  bool issue2170() {
+      TEST_START();
+      auto json = R"(1 2 3)"_padded;
+      simdjson::dom::parser parser;
+      simdjson::dom::document_stream stream;
+      ASSERT_SUCCESS(parser.parse_many(json).get(stream));
+      auto i = stream.begin();
+      size_t count{0};
+      std::vector<size_t> indexes = { 0, 2, 4 };
+      std::vector<std::string_view> expected = { "1", "2", "3" };
+
+      for(; i != stream.end(); ++i) {
+          auto doc = *i;
+          ASSERT_SUCCESS(doc);
+          ASSERT_TRUE(count < 3);
+          ASSERT_EQUAL(i.current_index(), indexes[count]);
+          ASSERT_EQUAL(i.source(), expected[count]);
+          count++;
+      }
+      TEST_SUCCEED();
+  }
+
   bool issue1310() {
     std::cout << "Running " << __func__ << std::endl;
     // hex  : 20 20 5B 20 33 2C 31 5D 20 22 22 22 22 22 22 22 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
@@ -941,7 +963,8 @@ namespace document_stream_tests {
   }
 
   bool run() {
-    return skipbom() &&
+    return issue2170() &&
+           skipbom() &&
            fuzzaccess() &&
            baby_fuzzer() &&
            issue1649() &&

diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp
@@ -196,6 +196,27 @@ namespace document_stream_tests {
         TEST_SUCCEED();
     }
 
+    bool issue2170() {
+        TEST_START();
+        auto json = R"(1 2 3)"_padded;
+        ondemand::parser parser;
+        ondemand::document_stream stream;
+        ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
+        auto i = stream.begin();
+        size_t count{0};
+        std::vector<size_t> indexes = { 0, 2, 4 };
+        std::vector<std::string_view> expected = { "1", "2", "3" };
+
+        for(; i != stream.end(); ++i) {
+            ASSERT_SUCCESS(i.error());
+            ASSERT_TRUE(count < 3);
+            ASSERT_EQUAL(i.current_index(), indexes[count]);
+            ASSERT_EQUAL(i.source(), expected[count]);
+            count++;
+        }
+        TEST_SUCCEED();
+    }
+
     bool issue1977() {
         TEST_START();
         std::string json = R"( 1111 })";
@@ -881,6 +902,7 @@ namespace document_stream_tests {
 
     bool run() {
         return
+            issue2170() &&
             issue2137() &&
             skipbom() &&
             issue1977() &&