Skip to content

Commit

Permalink
feat: SourcemapVisualizer (#2773)
Browse files Browse the repository at this point in the history
Export `SourcemapVisualizer` from codegen, it will be used oxc and
rolldown sourcemap test, so it support multiply source print, it will
using sourcemap `sourcesContent` as original source.
  • Loading branch information
underfin committed Mar 21, 2024
1 parent 3c9e77d commit a2cfc86
Show file tree
Hide file tree
Showing 5 changed files with 171 additions and 99 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/oxc_codegen/Cargo.toml
Expand Up @@ -25,6 +25,7 @@ oxc_allocator = { workspace = true }
oxc_syntax = { workspace = true }
sourcemap = { version = "7.1.1" }
bitflags = { workspace = true }
rustc-hash = { workspace = true }

[dev-dependencies]
oxc_parser = { workspace = true }
Expand Down
3 changes: 2 additions & 1 deletion crates/oxc_codegen/src/lib.rs
Expand Up @@ -13,7 +13,7 @@ mod gen;
mod gen_ts;
mod operator;
mod sourcemap_builder;

mod sourcemap_visualizer;
use std::str::from_utf8_unchecked;

#[allow(clippy::wildcard_imports)]
Expand All @@ -31,6 +31,7 @@ pub use crate::{
context::Context,
gen::{Gen, GenExpr},
operator::Operator,
sourcemap_visualizer::SourcemapVisualizer,
};
// use crate::mangler::Mangler;

Expand Down
159 changes: 159 additions & 0 deletions crates/oxc_codegen/src/sourcemap_visualizer.rs
@@ -0,0 +1,159 @@
use rustc_hash::FxHashMap;
use sourcemap::SourceMap;

pub struct SourcemapVisualizer<'a> {
output: &'a str,
sourcemap: &'a SourceMap,
}

impl<'a> SourcemapVisualizer<'a> {
pub fn new(output: &'a str, sourcemap: &'a SourceMap) -> Self {
Self { output, sourcemap }
}

#[allow(clippy::cast_possible_truncation)]
pub fn into_visualizer_text(self) -> String {
let mut source_log_map = FxHashMap::default();
let source_contents_lines_map: FxHashMap<String, Option<Vec<Vec<u16>>>> = self
.sourcemap
.sources()
.enumerate()
.map(|(source_id, source)| {
(
source.to_string(),
self.sourcemap
.get_source_view(source_id as u32)
.map(|source_view| Self::generate_line_utf16_tables(source_view.source())),
)
})
.collect();
let output_lines = Self::generate_line_utf16_tables(self.output);
let mut s = String::new();

self.sourcemap.tokens().reduce(|pre_token, token| {
if let Some(source) = pre_token.get_source() {
if let Some(Some(source_contents_lines)) = source_contents_lines_map.get(source) {
// Print source
source_log_map.entry(source).or_insert_with(|| {
s.push('-');
s.push(' ');
s.push_str(source);
s.push('\n');
true
});

// Print token
s.push_str(&format!(
"({}:{}-{}:{}) {:?}",
pre_token.get_src_line(),
pre_token.get_src_col(),
token.get_src_line(),
token.get_src_col(),
Self::str_slice_by_token(
source_contents_lines,
(pre_token.get_src_line(), pre_token.get_src_col()),
(token.get_src_line(), token.get_src_col())
)
));

s.push_str(" --> ");

s.push_str(&format!(
"({}:{}-{}:{}) {:?}",
pre_token.get_dst_line(),
pre_token.get_dst_col(),
token.get_dst_line(),
token.get_dst_col(),
Self::str_slice_by_token(
&output_lines,
(pre_token.get_dst_line(), pre_token.get_dst_col(),),
(token.get_dst_line(), token.get_dst_col(),)
)
));
s.push('\n');
}
}

token
});

s
}

fn generate_line_utf16_tables(content: &str) -> Vec<Vec<u16>> {
let mut tables = vec![];
let mut line_byte_offset = 0;
for (i, ch) in content.char_indices() {
match ch {
'\r' | '\n' | '\u{2028}' | '\u{2029}' => {
// Handle Windows-specific "\r\n" newlines
if ch == '\r' && content.chars().nth(i + 1) == Some('\n') {
continue;
}
tables.push(content[line_byte_offset..i].encode_utf16().collect::<Vec<_>>());
line_byte_offset = i;
}
_ => {}
}
}
tables.push(content[line_byte_offset..].encode_utf16().collect::<Vec<_>>());
tables
}

fn str_slice_by_token(buff: &[Vec<u16>], start: (u32, u32), end: (u32, u32)) -> String {
if start.0 == end.0 {
return String::from_utf16(&buff[start.0 as usize][start.1 as usize..end.1 as usize])
.unwrap();
}

let mut s = String::new();

for i in start.0..end.0 {
let slice = &buff[i as usize];
if i == start.0 {
s.push_str(&String::from_utf16(&slice[start.1 as usize..]).unwrap());
} else if i == end.0 {
s.push_str(&String::from_utf16(&slice[..end.1 as usize]).unwrap());
} else {
s.push_str(&String::from_utf16(slice).unwrap());
}
}

s
}
}

#[cfg(test)]
mod test {
use super::*;

#[test]
fn should_work() {
let sourcemap = SourceMap::from_slice(r#"{
"version":3,
"sources":["shared.js","index.js"],
"sourcesContent":["const a = 'shared.js'\n\nexport { a }","import { a as a2 } from './shared'\nconst a = 'index.js'\nconsole.log(a, a2)\n"],
"names":["a","a$1"],
"mappings":";;AAAA,MAAMA,IAAI;;;ACCV,MAAMC,MAAI;AACV,QAAQ,IAAIA,KAAGD,EAAG"
}"#.as_bytes()).unwrap();
let output = "\n// shared.js\nconst a = 'shared.js';\n\n// index.js\nconst a$1 = 'index.js';\nconsole.log(a$1, a);\n";
let visualizer = SourcemapVisualizer::new(output, &sourcemap);
let visualizer_text = visualizer.into_visualizer_text();
assert_eq!(
visualizer_text,
r#"- shared.js
(0:0-0:6) "const " --> (2:0-2:6) "\nconst"
(0:6-0:10) "a = " --> (2:6-2:10) " a ="
(0:10-1:0) "'shared.js'" --> (2:10-5:0) " 'shared.js';\n\n// index.js"
- index.js
(1:0-1:6) "\nconst" --> (5:0-5:6) "\nconst"
(1:6-1:10) " a =" --> (5:6-5:12) " a$1 ="
(1:10-2:0) " 'index.js'" --> (5:12-6:0) " 'index.js';"
(2:0-2:8) "\nconsole" --> (6:0-6:8) "\nconsole"
(2:8-2:12) ".log" --> (6:8-6:12) ".log"
(2:12-2:15) "(a," --> (6:12-6:17) "(a$1,"
(2:15-2:18) " a2" --> (6:17-6:19) " a"
"#
);
}
}
106 changes: 8 additions & 98 deletions tasks/coverage/src/sourcemap.rs
Expand Up @@ -5,7 +5,7 @@ use std::{
};

use oxc_allocator::Allocator;
use oxc_codegen::{Codegen, CodegenOptions};
use oxc_codegen::{Codegen, CodegenOptions, SourcemapVisualizer};
use oxc_parser::Parser;
use oxc_span::SourceType;
use oxc_tasks_common::{project_root, TestFiles};
Expand Down Expand Up @@ -67,13 +67,12 @@ impl<T: Case> Suite<T> for SourcemapSuite<T> {
let result = case.test_result();
let path = case.path().to_string_lossy();
let result = match result {
TestResult::Snapshot(snapshot) => snapshot,
TestResult::ParseError(error, _) => error,
TestResult::Snapshot(snapshot) => snapshot.to_string(),
TestResult::ParseError(error, _) => format!("- {path}\n{error}"),
_ => {
unreachable!()
}
};
writeln!(file, "- {path}").unwrap();
writeln!(file, "{result}\n\n").unwrap();
}
}
Expand All @@ -90,82 +89,6 @@ impl SourcemapCase {
pub fn source_type(&self) -> SourceType {
self.source_type
}

fn generate_line_utf16_tables(content: &str) -> Vec<Vec<u16>> {
let mut tables = vec![];
let mut line_byte_offset = 0;
for (i, ch) in content.char_indices() {
match ch {
'\r' | '\n' | '\u{2028}' | '\u{2029}' => {
// Handle Windows-specific "\r\n" newlines
if ch == '\r' && content.chars().nth(i + 1) == Some('\n') {
continue;
}
tables.push(content[line_byte_offset..i].encode_utf16().collect::<Vec<_>>());
line_byte_offset = i;
}
_ => {}
}
}
tables.push(content[line_byte_offset..].encode_utf16().collect::<Vec<_>>());
tables
}

fn create_visualizer_text(
source: &str,
output: &str,
tokens: &[(u32, u32, u32, u32)],
) -> String {
let source_lines = Self::generate_line_utf16_tables(source);
let output_lines = Self::generate_line_utf16_tables(output);
let mut s = String::new();

tokens.iter().reduce(|pre, cur| {
s.push_str(&format!(
"({}:{}-{}:{}) {:?}",
pre.0,
pre.1,
cur.0,
cur.1,
Self::str_slice_by_token(&source_lines, (pre.0, pre.1), (cur.0, cur.1))
));
s.push_str(" --> ");
s.push_str(&format!(
"({}:{}-{}:{}) {:?}",
pre.2,
pre.3,
cur.2,
cur.3,
Self::str_slice_by_token(&output_lines, (pre.2, pre.3), (cur.2, cur.3))
));
s.push('\n');
cur
});

s
}

fn str_slice_by_token(buff: &[Vec<u16>], start: (u32, u32), end: (u32, u32)) -> String {
if start.0 == end.0 {
return String::from_utf16(&buff[start.0 as usize][start.1 as usize..end.1 as usize])
.unwrap();
}

let mut s = String::new();

for i in start.0..end.0 {
let slice = &buff[i as usize];
if i == start.0 {
s.push_str(&String::from_utf16(&slice[start.1 as usize..]).unwrap());
} else if i == end.0 {
s.push_str(&String::from_utf16(&slice[..end.1 as usize]).unwrap());
} else {
s.push_str(&String::from_utf16(slice).unwrap());
}
}

s
}
}

impl Case for SourcemapCase {
Expand Down Expand Up @@ -208,23 +131,10 @@ impl Case for SourcemapCase {
..CodegenOptions::default()
};
let codegen_ret = Codegen::<false>::new(source_text, codegen_options).build(&ret.program);
let content = codegen_ret.source_text;
let map = codegen_ret.source_map.unwrap();
let tokens = map
.tokens()
.map(|token| {
(
token.get_src_line(),
token.get_src_col(),
token.get_dst_line(),
token.get_dst_col(),
)
})
.collect::<Vec<_>>();

let mut result = String::new();
result.push_str(Self::create_visualizer_text(source_text, &content, &tokens).as_str());

TestResult::Snapshot(result)

TestResult::Snapshot(
SourcemapVisualizer::new(&codegen_ret.source_text, &codegen_ret.source_map.unwrap())
.into_visualizer_text(),
)
}
}

0 comments on commit a2cfc86

Please sign in to comment.