diff --git a/src/fallback.rs b/src/fallback.rs index 4c476e5d..fe4f248d 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -182,7 +182,13 @@ impl FromStr for TokenStream { fn from_str(src: &str) -> Result { // Create a dummy file & add it to the source map - let cursor = get_cursor(src); + let mut cursor = get_cursor(src); + + // Strip a byte order mark if present + const BYTE_ORDER_MARK: &str = "\u{feff}"; + if cursor.starts_with(BYTE_ORDER_MARK) { + cursor = cursor.advance(BYTE_ORDER_MARK.len()); + } parse::token_stream(cursor) } diff --git a/src/parse.rs b/src/parse.rs index d2b86a41..04c48336 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -14,7 +14,7 @@ pub(crate) struct Cursor<'a> { } impl<'a> Cursor<'a> { - fn advance(&self, bytes: usize) -> Cursor<'a> { + pub fn advance(&self, bytes: usize) -> Cursor<'a> { let (_front, rest) = self.rest.split_at(bytes); Cursor { rest, @@ -23,7 +23,7 @@ impl<'a> Cursor<'a> { } } - fn starts_with(&self, s: &str) -> bool { + pub fn starts_with(&self, s: &str) -> bool { self.rest.starts_with(s) } diff --git a/tests/test.rs b/tests/test.rs index 16f775ed..8f5624db 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -630,3 +630,16 @@ fn check_spans_internal(ts: TokenStream, lines: &mut &[(usize, usize, usize, usi } } } + +#[test] +fn byte_order_mark() { + let string = "\u{feff}foo"; + let tokens = string.parse::().unwrap(); + match tokens.into_iter().next().unwrap() { + TokenTree::Ident(ident) => assert_eq!(ident, "foo"), + _ => unreachable!(), + } + + let string = "foo\u{feff}"; + string.parse::().unwrap_err(); +}