Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for parsing parial JSON strings in Python #66

Merged
merged 3 commits into from
Mar 25, 2024
Merged

Support for parsing parial JSON strings in Python #66

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ed71053
add "allow_partial" argument to python parsing
samuelcolvin Feb 6, 2024
c8a669e
Merge branch 'main' into partial-parsing
samuelcolvin Mar 25, 2024
c3f6b6b
separate _parse_array and _parse_object
samuelcolvin Mar 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion benches/python.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ fn python_parse_numeric(bench: &mut Bencher) {
black_box(br#" { "int": 1, "bigint": 123456789012345678901234567890, "float": 1.2} "#),
false,
true,
false,
))
.unwrap()
});
Expand All @@ -34,6 +35,7 @@ fn python_parse_other(bench: &mut Bencher) {
black_box(br#"["string", true, false, null]"#),
false,
true,
false,
))
.unwrap()
});
Expand All @@ -50,7 +52,7 @@ fn _python_parse_file(path: &str, bench: &mut Bencher, cache_strings: bool) {
bench.iter(|| {
// Clear PyO3 memory on each loop iteration to avoid long GC traversal overheads.
let _pool = unsafe { py.new_pool() };
black_box(python_parse(py, black_box(json_data), false, cache_strings)).unwrap()
black_box(python_parse(py, black_box(json_data), false, cache_strings, false)).unwrap()
});
})
}
Expand Down
59 changes: 46 additions & 13 deletions src/python.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use crate::errors::{json_err, json_error, JsonError, JsonResult, DEFAULT_RECURSI
use crate::number_decoder::{NumberAny, NumberInt};
use crate::parse::{Parser, Peek};
use crate::string_decoder::{StringDecoder, Tape};
use crate::JsonErrorType;

/// Parse a JSON value from a byte slice and return a Python object.
///
Expand All @@ -26,12 +27,19 @@ use crate::string_decoder::{StringDecoder, Tape};
/// # Returns
///
/// A [PyObject](https://docs.rs/pyo3/latest/pyo3/type.PyObject.html) representing the parsed JSON value.
pub fn python_parse(py: Python, json_data: &[u8], allow_inf_nan: bool, cache_strings: bool) -> JsonResult<PyObject> {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need to update the docstring above.

pub fn python_parse(
py: Python,
json_data: &[u8],
allow_inf_nan: bool,
cache_strings: bool,
allow_partial: bool,
) -> JsonResult<PyObject> {
let mut python_parser = PythonParser {
parser: Parser::new(json_data),
tape: Tape::default(),
recursion_limit: DEFAULT_RECURSION_LIMIT,
allow_inf_nan,
allow_partial,
};

let peek = python_parser.parser.peek()?;
Expand All @@ -40,7 +48,9 @@ pub fn python_parse(py: Python, json_data: &[u8], allow_inf_nan: bool, cache_str
} else {
python_parser.py_take_value::<StringNoCache>(py, peek)?
};
python_parser.parser.finish()?;
if !allow_partial {
python_parser.parser.finish()?;
}
Ok(v)
}

Expand All @@ -54,10 +64,34 @@ struct PythonParser<'j> {
tape: Tape,
recursion_limit: u8,
allow_inf_nan: bool,
allow_partial: bool,
}

impl<'j> PythonParser<'j> {
fn py_take_value<StringCache: StringMaybeCache>(&mut self, py: Python, peek: Peek) -> JsonResult<PyObject> {
macro_rules! tri {
($result:expr, $partial_value:expr) => {
match $result {
Ok(k) => k,
Err(e) => {
return if self.allow_partial {
match e.error_type {
JsonErrorType::EofWhileParsingList
| JsonErrorType::EofWhileParsingObject
| JsonErrorType::EofWhileParsingString
| JsonErrorType::EofWhileParsingValue
| JsonErrorType::ExpectedListCommaOrEnd
| JsonErrorType::ExpectedObjectCommaOrEnd => Ok($partial_value.to_object(py)),
samuelcolvin marked this conversation as resolved.
Show resolved Hide resolved
_ => Err(e),
}
} else {
Err(e)
}
}
}
};
}

match peek {
Peek::Null => {
self.parser.consume_null()?;
Expand All @@ -76,12 +110,12 @@ impl<'j> PythonParser<'j> {
Ok(StringCache::get(py, s.as_str()))
}
Peek::Array => {
let list = if let Some(peek_first) = self.parser.array_first()? {
let list = if let Some(peek_first) = tri!(self.parser.array_first(), PyList::empty(py)) {
let mut vec: SmallVec<[PyObject; 8]> = SmallVec::with_capacity(8);
let v = self._check_take_value::<StringCache>(py, peek_first)?;
let v = tri!(self._check_take_value::<StringCache>(py, peek_first), PyList::empty(py));
vec.push(v);
while let Some(peek) = self.parser.array_step()? {
let v = self._check_take_value::<StringCache>(py, peek)?;
while let Some(peek) = tri!(self.parser.array_step(), PyList::new(py, vec)) {
let v = tri!(self._check_take_value::<StringCache>(py, peek), PyList::new(py, vec));
vec.push(v);
}
PyList::new(py, vec)
samuelcolvin marked this conversation as resolved.
Show resolved Hide resolved
Expand All @@ -102,16 +136,15 @@ impl<'j> PythonParser<'j> {
panic!("PyDict_SetItem failed")
}
};

if let Some(first_key) = self.parser.object_first::<StringDecoder>(&mut self.tape)? {
if let Some(first_key) = tri!(self.parser.object_first::<StringDecoder>(&mut self.tape), dict) {
samuelcolvin marked this conversation as resolved.
Show resolved Hide resolved
let first_key = StringCache::get(py, first_key.as_str());
let peek = self.parser.peek()?;
let first_value = self._check_take_value::<StringCache>(py, peek)?;
let peek = tri!(self.parser.peek(), dict);
let first_value = tri!(self._check_take_value::<StringCache>(py, peek), dict);
set_item(first_key, first_value);
while let Some(key) = self.parser.object_step::<StringDecoder>(&mut self.tape)? {
while let Some(key) = tri!(self.parser.object_step::<StringDecoder>(&mut self.tape), dict) {
let key = StringCache::get(py, key.as_str());
let peek = self.parser.peek()?;
let value = self._check_take_value::<StringCache>(py, peek)?;
let peek = tri!(self.parser.peek(), dict);
let value = tri!(self._check_take_value::<StringCache>(py, peek), dict);
set_item(key, value);
}
}
Expand Down
67 changes: 59 additions & 8 deletions tests/python.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList};
use pyo3::ToPyObject;

use jiter::{map_json_error, python_parse, JsonValue};
Expand Down Expand Up @@ -42,6 +43,7 @@ fn test_python_parse_numeric() {
br#" { "int": 1, "bigint": 123456789012345678901234567890, "float": 1.2} "#,
false,
true,
false,
)
.unwrap();
assert_eq!(
Expand All @@ -59,6 +61,7 @@ fn test_python_parse_other_cached() {
br#"["string", true, false, null, NaN, Infinity, -Infinity]"#,
true,
true,
false,
)
.unwrap();
assert_eq!(
Expand All @@ -71,15 +74,15 @@ fn test_python_parse_other_cached() {
#[test]
fn test_python_parse_other_no_cache() {
Python::with_gil(|py| {
let obj = python_parse(py, br#"["string", true, false, null]"#, false, false).unwrap();
let obj = python_parse(py, br#"["string", true, false, null]"#, false, false, false).unwrap();
assert_eq!(obj.as_ref(py).to_string(), "['string', True, False, None]");
})
}

#[test]
fn test_python_disallow_nan() {
Python::with_gil(|py| {
let r = python_parse(py, br#"[NaN]"#, false, true);
let r = python_parse(py, br#"[NaN]"#, false, true, false);
let e = r.map_err(|e| map_json_error(br#"[NaN]"#, &e)).unwrap_err();
assert_eq!(e.to_string(), "ValueError: expected value at line 1 column 2");
})
Expand All @@ -89,7 +92,7 @@ fn test_python_disallow_nan() {
fn test_error() {
Python::with_gil(|py| {
let bytes = br#"["string""#;
let r = python_parse(py, bytes, false, true);
let r = python_parse(py, bytes, false, true, false);
let e = r.map_err(|e| map_json_error(bytes, &e)).unwrap_err();
assert_eq!(e.to_string(), "ValueError: EOF while parsing a list at line 1 column 9");
})
Expand All @@ -101,7 +104,7 @@ fn test_recursion_limit() {
let bytes = json.as_bytes();

Python::with_gil(|py| {
let r = python_parse(py, bytes, false, true);
let r = python_parse(py, bytes, false, true, false);
let e = r.map_err(|e| map_json_error(bytes, &e)).unwrap_err();
assert_eq!(
e.to_string(),
Expand All @@ -117,24 +120,72 @@ fn test_recursion_limit_incr() {
let bytes = json.as_bytes();

Python::with_gil(|py| {
let v = python_parse(py, bytes, false, true).unwrap();
let v = python_parse(py, bytes, false, true, false).unwrap();
assert_eq!(v.as_ref(py).len().unwrap(), 2000);
});

Python::with_gil(|py| {
let v = python_parse(py, bytes, false, true).unwrap();
let v = python_parse(py, bytes, false, true, false).unwrap();
assert_eq!(v.as_ref(py).len().unwrap(), 2000);
});
}

#[test]
fn test_exected_value_error() {
fn test_extracted_value_error() {
let json = "xx";
let bytes = json.as_bytes();

Python::with_gil(|py| {
let r = python_parse(py, bytes, false, true);
let r = python_parse(py, bytes, false, true, false);
let e = r.map_err(|e| map_json_error(bytes, &e)).unwrap_err();
assert_eq!(e.to_string(), "ValueError: expected value at line 1 column 1");
})
}

#[test]
fn test_partial_array() {
Python::with_gil(|py| {
let bytes = br#"["string", true, null, 1, "foo"#;
let py_value = python_parse(py, bytes, false, true, true).unwrap();
let string = py_value.as_ref(py).to_string();
assert_eq!(string, "['string', True, None, 1]");

// test that stopping at every points is ok
for i in 1..bytes.len() {
let py_value = python_parse(py, &bytes[..i], false, true, true).unwrap();
assert!(py_value.as_ref(py).is_instance_of::<PyList>());
}
})
}

#[test]
fn test_partial_object() {
Python::with_gil(|py| {
let bytes = br#"{"a": 1, "b": 2, "c"#;
let py_value = python_parse(py, bytes, false, true, true).unwrap();
let string = py_value.as_ref(py).to_string();
assert_eq!(string, "{'a': 1, 'b': 2}");

// test that stopping at every points is ok
for i in 1..bytes.len() {
let py_value = python_parse(py, &bytes[..i], false, true, true).unwrap();
assert!(py_value.as_ref(py).is_instance_of::<PyDict>());
}
})
}

#[test]
fn test_partial_nested() {
Python::with_gil(|py| {
let bytes = br#"{"a": 1, "b": 2, "c": [1, 2, {"d": 1, "#;
let py_value = python_parse(py, bytes, false, true, true).unwrap();
let string = py_value.as_ref(py).to_string();
assert_eq!(string, "{'a': 1, 'b': 2, 'c': [1, 2, {'d': 1}]}");

// test that stopping at every points is ok
for i in 1..bytes.len() {
let py_value = python_parse(py, &bytes[..i], false, true, true).unwrap();
assert!(py_value.as_ref(py).is_instance_of::<PyDict>());
}
})
}