wehub/1jehuang--jcode
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
FreeBSD Smoke / FreeBSD Smoke (x86_64) (push) Has been cancelled
Details
CI / Quality Guardrails (push) Has been cancelled
Details
CI / Build & Test (macos-latest) (push) Has been cancelled
Details
CI / Build & Test (ubuntu-latest) (push) Has been cancelled
Details
CI / Build & Test (windows-latest) (push) Has been cancelled
Details
CI / Format (push) Has been cancelled
Details
CI / PowerShell Syntax (push) Has been cancelled
Details
CI / Windows Cross-Target Check (Linux) (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:10:34 +08:00
commit a789495a98
1551 changed files with 718128 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/jcode-render-core/Cargo.toml
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "jcode-render-core"
version = "0.1.0"
edition = "2024"
publish = false
# Backend-neutral document/render model shared by the TUI and desktop
# front-ends. This crate owns markdown parsing, the semantic block/span
# model, and width-parameterized wrapping. It has NO dependency on ratatui
# or any GPU/glyph layer; front-ends provide thin adapters that turn the
# neutral model into their own draw primitives.
[dependencies]
pulldown-cmark = "0.12"
serde = { version = "1", features = ["derive"] }
unicode-width = "0.2"
[dev-dependencies]
crates/jcode-render-core/src/lib.rs
+42
View File
@@ -0,0 +1,42 @@
//! # jcode-render-core
//!
//! Backend-neutral document/render model shared by jcode's front-ends (the
//! ratatui TUI and the desktop GPU UI).
//!
//! The pipeline is split at the seam where backends actually differ:
//!
//! ```text
//! text ─▶ parse_markdown ─▶ Document (neutral blocks/spans) ─▶ wrap ─▶ adapter ─▶ backend draw
//! ```
//!
//! Everything up to and including wrapping is shared here. Each front-end owns
//! only a thin adapter: it resolves [`model::StyleRole`] to concrete colors and
//! turns [`model::StyledLine`]s into its own draw primitives (`ratatui::Line`
//! for the TUI, glyph runs for the desktop), and supplies a
//! [`wrap::WidthMeasure`] for its width units.
//!
//! This model is extracted from the *working* TUI markdown renderer
//! (`jcode-tui-markdown`); that renderer remains authoritative until this core
//! reaches parity, after which front-ends migrate onto it.
pub mod markdown;
pub mod math;
pub mod model;
pub mod preprocess;
pub mod reasoning;
pub mod wrap;
pub use markdown::parse_markdown;
pub use math::{render_display_latex, render_inline_latex};
pub use model::{
Alignment, Block, BlockKind, Document, FillRole, StyleRole, StyledLine, StyledSpan, TextAttrs,
};
pub use preprocess::{escape_currency_dollars, normalize_latex_math};
pub use reasoning::{
REASONING_SENTINEL, reasoning_line_markup, reasoning_partial_markup,
reasoning_summary_line_markup,
};
pub use wrap::{ColumnWidth, WidthMeasure, wrap_line, wrap_lines};
#[cfg(test)]
mod tests;
crates/jcode-render-core/src/markdown.rs
+684
View File
@@ -0,0 +1,684 @@
//! Markdown -> backend-neutral [`Document`].
//!
//! This mirrors the *semantics* of the TUI markdown renderer
//! (`jcode-tui-markdown`) but emits the neutral [`crate::model`] types instead
//! of `ratatui` spans. Front-ends adapt the document to their backend and may
//! wrap it with [`crate::wrap`].
//!
//! Scope note: this is the shared foundation. It currently covers headings,
//! paragraphs, inline emphasis/strong/strike/code, fenced & indented code
//! blocks, blockquotes, ordered/unordered (incl. nested) lists, thematic
//! breaks, links, raw HTML passthrough, tables, and terminal-friendly math.
use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag, TagEnd};
use crate::model::{
Alignment, Block, BlockKind, Document, FillRole, StyleRole, StyledLine, StyledSpan, TextAttrs,
};
#[derive(Clone, Copy, Default)]
struct InlineStyle {
bold: bool,
italic: bool,
strike: bool,
}
impl InlineStyle {
fn attrs(self) -> TextAttrs {
TextAttrs {
bold: self.bold,
italic: self.italic,
strikethrough: self.strike,
underline: false,
}
}
fn role(self) -> StyleRole {
if self.bold {
StyleRole::Strong
} else {
StyleRole::Text
}
}
}
struct ListFrame {
ordered: bool,
next_number: u64,
/// Block index in `doc.blocks` where this list's content begins, used to
/// right-align ordered markers once the list's width is known.
start_block: usize,
/// Nesting depth of this list (0 = outermost).
depth: usize,
}
/// Right-align ordered-list markers within a single list level, mirroring the
/// legacy renderer: when the list has multi-digit item numbers, shorter markers
/// are padded with leading spaces so the `.` separators line up and wrapped
/// continuation lines indent consistently.
///
/// Only markers at exactly `depth` indentation are touched, so nested lists
/// (which carry deeper indentation) are aligned by their own `End(List)`.
fn align_ordered_markers(doc: &mut Document, start_block: usize, depth: usize) {
let indent = " ".repeat(depth);
// First pass: find the widest digit run among this level's markers.
let mut max_digits = 0usize;
for block in doc.blocks.iter().skip(start_block) {
if let Some(d) = ordered_marker_digits(block, &indent) {
max_digits = max_digits.max(d);
}
}
if max_digits <= 1 {
return;
}
// Second pass: pad shorter markers.
for block in doc.blocks.iter_mut().skip(start_block) {
let Some(d) = ordered_marker_digits(block, &indent) else {
continue;
};
let extra = max_digits - d;
if extra == 0 {
continue;
}
if let Some(first_span) = block
.lines
.first_mut()
.and_then(|line| line.spans.first_mut())
{
// Insert padding right after the indent, before the digits.
let rest = &first_span.text[indent.len()..];
first_span.text = format!("{indent}{}{rest}", " ".repeat(extra));
}
}
}
/// If `block`'s first span is an ordered-list marker at exactly `indent`
/// (i.e. `{indent}{digits}. `), return the digit count.
fn ordered_marker_digits(block: &Block, indent: &str) -> Option<usize> {
let text = &block.lines.first()?.spans.first()?.text;
let rest = text.strip_prefix(indent)?;
// Reject deeper-nested markers (extra leading spaces) and non-markers.
if rest.starts_with(' ') {
return None;
}
let digits = rest.chars().take_while(|c| c.is_ascii_digit()).count();
if digits == 0 {
return None;
}
if !rest[digits..].starts_with(". ") {
return None;
}
Some(digits)
}
/// The block kind that inline content flushed in the current context belongs
/// to, based on enclosing blockquote/list nesting.
fn current_kind(blockquote_depth: usize, list_stack: &[ListFrame]) -> BlockKind {
if blockquote_depth > 0 {
BlockKind::BlockQuote
} else if let Some(frame) = list_stack.last() {
BlockKind::ListItem {
ordered: frame.ordered,
depth: list_stack.len().saturating_sub(1),
}
} else {
BlockKind::Paragraph
}
}
/// Parse markdown into a backend-neutral [`Document`].
pub fn parse_markdown(text: &str) -> Document {
let mut options = Options::empty();
options.insert(Options::ENABLE_TABLES);
options.insert(Options::ENABLE_MATH);
options.insert(Options::ENABLE_STRIKETHROUGH);
options.insert(Options::ENABLE_TASKLISTS);
options.insert(Options::ENABLE_FOOTNOTES);
options.insert(Options::ENABLE_GFM);
options.insert(Options::ENABLE_SMART_PUNCTUATION);
options.insert(Options::ENABLE_DEFINITION_LIST);
let normalized = crate::preprocess::normalize_latex_math(text);
let escaped = crate::preprocess::escape_currency_dollars(&normalized);
let parser = Parser::new_ext(&escaped, options);
let mut doc = Document::default();
// Inline accumulation for the block currently being built.
let mut spans: Vec<StyledSpan> = Vec::new();
let mut style = InlineStyle::default();
// Block context.
let mut heading_level: Option<u8> = None;
let mut blockquote_depth = 0usize;
let mut list_stack: Vec<ListFrame> = Vec::new();
// Code block accumulation.
let mut in_code_block = false;
let mut code_lang: Option<String> = None;
let mut code_buf = String::new();
// Pending list-item marker prefix to emit when the item's first inline
// text arrives.
let mut pending_item_marker: Option<String> = None;
// Table accumulation. While `in_table`, inline text is collected into the
// current cell (as raw text) rather than styled spans, mirroring the legacy
// renderer which lays tables out by width.
let mut in_table = false;
let mut table_rows: Vec<Vec<String>> = Vec::new();
let mut table_row: Vec<String> = Vec::new();
let mut current_cell = String::new();
// Blockquote line accumulation. Legacy emits one rendered line per source
// line inside a quote (soft breaks split lines), so we buffer the lines and
// emit a single BlockQuote block when the outermost quote closes.
let mut bq_lines: Vec<StyledLine> = Vec::new();
// Link destinations (stack for nesting); appended as a dim ` (url)` suffix
// after the link text, mirroring the legacy renderer.
let mut link_targets: Vec<String> = Vec::new();
// Image alt-text accumulation.
let mut in_image = false;
let mut image_url: Option<String> = None;
let mut image_alt = String::new();
let push_block = |doc: &mut Document, kind: BlockKind, lines: Vec<StyledLine>| {
if !lines.is_empty() {
doc.blocks.push(Block::new(kind, lines));
}
};
let flush_paragraph = |doc: &mut Document,
spans: &mut Vec<StyledSpan>,
kind: BlockKind,
alignment: Alignment,
blockquote_depth: usize,
bq_lines: &mut Vec<StyledLine>| {
if spans.is_empty() {
return;
}
if blockquote_depth > 0 {
// Inside a quote: accumulate as a gutter-prefixed line. The block is
// emitted when the outermost quote closes.
let mut line = std::mem::take(spans);
line.insert(
0,
StyledSpan::new("│ ".repeat(blockquote_depth), StyleRole::Dim),
);
bq_lines.push(StyledLine::from_spans(line));
return;
}
let line = StyledLine::aligned(std::mem::take(spans), alignment);
push_block(doc, kind, vec![line]);
};
for event in parser {
match event {
// ---- block starts ----
Event::Start(Tag::Heading { level, .. }) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
heading_level = Some(level as u8);
}
Event::Start(Tag::Paragraph) => {
// marker (if any) is emitted lazily on first text
}
Event::Start(Tag::BlockQuote(_)) => {
flush_paragraph(
&mut doc,
&mut spans,
current_kind(blockquote_depth, &list_stack),
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
blockquote_depth += 1;
}
Event::Start(Tag::List(first)) => {
flush_paragraph(
&mut doc,
&mut spans,
current_kind(blockquote_depth, &list_stack),
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
let depth = list_stack.len();
list_stack.push(ListFrame {
ordered: first.is_some(),
next_number: first.unwrap_or(1),
start_block: doc.blocks.len(),
depth,
});
}
Event::Start(Tag::Item) => {
let depth = list_stack.len().saturating_sub(1);
let indent = " ".repeat(depth);
let marker = if let Some(frame) = list_stack.last_mut() {
if frame.ordered {
let n = frame.next_number;
frame.next_number += 1;
format!("{indent}{n}. ")
} else {
format!("{indent}• ")
}
} else {
String::new()
};
pending_item_marker = Some(marker);
}
Event::Start(Tag::CodeBlock(kind)) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
in_code_block = true;
code_buf.clear();
code_lang = match kind {
CodeBlockKind::Fenced(lang) if !lang.is_empty() => Some(lang.to_string()),
_ => None,
};
}
Event::Start(Tag::Emphasis) => style.italic = true,
Event::Start(Tag::Strong) => style.bold = true,
Event::Start(Tag::Strikethrough) => style.strike = true,
Event::Start(Tag::Link { dest_url, .. }) => {
link_targets.push(dest_url.to_string());
}
Event::Start(Tag::Image { dest_url, .. }) => {
in_image = true;
image_url = Some(dest_url.to_string());
image_alt.clear();
}
// ---- footnote definitions ----
Event::Start(Tag::FootnoteDefinition(label)) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
spans.push(StyledSpan::new(format!("[^{label}]: "), StyleRole::Dim));
}
Event::End(TagEnd::FootnoteDefinition) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
}
// ---- definition lists ----
Event::Start(Tag::DefinitionListTitle) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
spans.push(StyledSpan::new("• ".to_string(), StyleRole::Dim));
}
Event::End(TagEnd::DefinitionListTitle) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
}
Event::Start(Tag::DefinitionListDefinition) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
spans.push(StyledSpan::new(" -> ".to_string(), StyleRole::Dim));
}
Event::End(TagEnd::DefinitionListDefinition) => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
}
// ---- tables ----
Event::Start(Tag::Table(_)) => {
flush_paragraph(
&mut doc,
&mut spans,
current_kind(blockquote_depth, &list_stack),
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
in_table = true;
table_rows.clear();
}
Event::Start(Tag::TableHead) | Event::Start(Tag::TableRow) => {
table_row.clear();
}
Event::Start(Tag::TableCell) => {
current_cell.clear();
}
Event::End(TagEnd::TableCell) => {
table_row.push(current_cell.trim().to_string());
current_cell.clear();
}
Event::End(TagEnd::TableHead) | Event::End(TagEnd::TableRow) => {
if !table_row.is_empty() {
table_rows.push(std::mem::take(&mut table_row));
}
}
Event::End(TagEnd::Table) => {
in_table = false;
if !table_rows.is_empty() {
doc.blocks
.push(Block::table(std::mem::take(&mut table_rows)));
}
}
// ---- inline content ----
Event::Text(t) => {
if in_image {
image_alt.push_str(&t);
} else if in_table {
current_cell.push_str(&t);
} else if in_code_block {
code_buf.push_str(&t);
} else {
if let Some(marker) = pending_item_marker.take() {
spans.push(StyledSpan::new(marker, StyleRole::Dim));
}
spans.push(StyledSpan {
text: t.to_string(),
role: style.role(),
fill: FillRole::None,
attrs: style.attrs(),
});
}
}
Event::Code(t) => {
if in_table {
current_cell.push_str(&t);
} else {
if let Some(marker) = pending_item_marker.take() {
spans.push(StyledSpan::new(marker, StyleRole::Dim));
}
spans.push(StyledSpan {
text: t.to_string(),
role: StyleRole::Code,
fill: FillRole::Code,
attrs: TextAttrs::none(),
});
}
}
Event::InlineMath(math) => {
if in_table {
current_cell.push_str(&crate::math::render_inline_latex(&math));
} else {
if let Some(marker) = pending_item_marker.take() {
spans.push(StyledSpan::new(marker, StyleRole::Dim));
}
spans.push(StyledSpan {
text: crate::math::render_inline_latex(&math),
role: StyleRole::Math,
fill: FillRole::None,
attrs: TextAttrs::none(),
});
}
}
Event::DisplayMath(math) => {
if in_table {
current_cell.push_str(&crate::math::render_inline_latex(&math));
} else {
flush_paragraph(
&mut doc,
&mut spans,
current_kind(blockquote_depth, &list_stack),
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
let mut lines: Vec<StyledLine> = crate::math::render_display_latex(&math)
.into_iter()
.map(|l| StyledLine::from_spans(vec![StyledSpan::new(l, StyleRole::Math)]))
.collect();
if lines.is_empty() {
lines.push(StyledLine::from_spans(vec![StyledSpan::new(
String::new(),
StyleRole::Math,
)]));
}
push_block(&mut doc, BlockKind::MathDisplay, lines);
}
}
Event::FootnoteReference(label) => {
let text = format!("[^{label}]");
if in_image {
image_alt.push_str(&text);
} else if in_table {
current_cell.push_str(&text);
} else {
spans.push(StyledSpan::new(text, StyleRole::Dim));
}
}
Event::TaskListMarker(checked) => {
let marker = if checked { "[x] " } else { "[ ] " };
if in_table {
current_cell.push_str(marker);
} else {
if let Some(m) = pending_item_marker.take() {
spans.push(StyledSpan::new(m, StyleRole::Dim));
}
spans.push(StyledSpan::new(marker.to_string(), StyleRole::Dim));
}
}
Event::SoftBreak | Event::HardBreak => {
if in_table {
current_cell.push(' ');
} else if blockquote_depth > 0 {
// Inside a quote, a soft/hard break starts a new quoted line.
flush_paragraph(
&mut doc,
&mut spans,
current_kind(blockquote_depth, &list_stack),
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
} else {
spans.push(StyledSpan::plain(" "));
}
}
Event::Html(raw) | Event::InlineHtml(raw) => {
if in_table {
current_cell.push_str(&raw);
} else {
spans.push(StyledSpan {
text: raw.to_string(),
role: StyleRole::Html,
fill: FillRole::None,
attrs: TextAttrs {
italic: true,
..TextAttrs::none()
},
});
}
}
Event::Rule => {
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
push_block(
&mut doc,
BlockKind::ThematicBreak,
vec![StyledLine::from_spans(vec![StyledSpan::new(
"─".repeat(3),
StyleRole::Dim,
)])],
);
}
// ---- block ends ----
Event::End(TagEnd::Heading(_)) => {
let level = heading_level.take().unwrap_or(1);
// Headings render with strong role + bold across the line.
for s in spans.iter_mut() {
s.role = StyleRole::Strong;
s.attrs.bold = true;
}
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Heading { level },
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
}
Event::End(TagEnd::Paragraph) => {
let kind = current_kind(blockquote_depth, &list_stack);
flush_paragraph(
&mut doc,
&mut spans,
kind,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
}
Event::End(TagEnd::Item) => {
// Item with no paragraph child (tight list): flush inline buffer.
if !spans.is_empty() {
let kind = current_kind(blockquote_depth, &list_stack);
flush_paragraph(
&mut doc,
&mut spans,
kind,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
}
pending_item_marker = None;
}
Event::End(TagEnd::List(_)) => {
if let Some(frame) = list_stack.pop()
&& frame.ordered
{
align_ordered_markers(&mut doc, frame.start_block, frame.depth);
}
}
Event::End(TagEnd::BlockQuote(_)) => {
blockquote_depth = blockquote_depth.saturating_sub(1);
if blockquote_depth == 0 && !bq_lines.is_empty() {
push_block(
&mut doc,
BlockKind::BlockQuote,
std::mem::take(&mut bq_lines),
);
}
}
Event::End(TagEnd::CodeBlock) => {
let lines: Vec<StyledLine> = code_buf
.trim_end_matches('\n')
.split('\n')
.map(|l| {
StyledLine::from_spans(vec![StyledSpan {
text: l.to_string(),
role: StyleRole::Code,
fill: FillRole::Code,
attrs: TextAttrs::none(),
}])
})
.collect();
push_block(
&mut doc,
BlockKind::CodeBlock {
language: code_lang.take(),
},
lines,
);
in_code_block = false;
code_buf.clear();
}
Event::End(TagEnd::Emphasis) => style.italic = false,
Event::End(TagEnd::Strong) => style.bold = false,
Event::End(TagEnd::Strikethrough) => style.strike = false,
Event::End(TagEnd::Link) => {
if let Some(url) = link_targets.pop()
&& !url.is_empty()
{
spans.push(StyledSpan::new(format!(" ({url})"), StyleRole::Dim));
}
}
Event::End(TagEnd::Image) => {
let alt = if image_alt.trim().is_empty() {
"image".to_string()
} else {
image_alt.trim().to_string()
};
let label = match image_url.take() {
Some(url) => format!("[image: {alt}] ({url})"),
None => format!("[image: {alt}]"),
};
in_image = false;
image_alt.clear();
if in_table {
current_cell.push_str(&label);
} else {
spans.push(StyledSpan::new(label, StyleRole::Dim));
}
}
_ => {}
}
}
// Trailing inline buffer.
flush_paragraph(
&mut doc,
&mut spans,
BlockKind::Paragraph,
Alignment::Left,
blockquote_depth,
&mut bq_lines,
);
doc
}
crates/jcode-render-core/src/math.rs
+1250
View File
File diff suppressed because it is too large Load Diff
crates/jcode-render-core/src/model.rs
+227
View File
@@ -0,0 +1,227 @@
//! Backend-neutral styled-span and block model.
//!
//! The TUI markdown renderer emits `ratatui::Span`s whose colors are chosen
//! from *semantic* helpers (`md_dim_color()`, `bold_color()`, `code_fg()`,
//! ...), not raw colors. We preserve that: the neutral model carries a
//! semantic [`StyleRole`] plus boolean text attributes, and each front-end
//! adapter resolves the role to a concrete color for its backend (terminal
//! palette vs. desktop theme).
use serde::{Deserialize, Serialize};
/// Semantic color role. Front-ends map these to concrete colors so the same
/// document renders consistently across backends while still honoring each
/// backend's theme.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
pub enum StyleRole {
/// Default body text.
#[default]
Text,
/// De-emphasized text (markers, rules, quote bars, metadata).
Dim,
/// Headings / strong emphasis color.
Strong,
/// Inline code and code-block foreground.
Code,
/// Hyperlink text.
Link,
/// Inline/raw HTML passthrough.
Html,
/// Model "reasoning" text (dim + italic latch in the TUI).
Reasoning,
/// Math (inline `$..$` or display `$$..$$`) content.
Math,
}
/// Background fill role for a span. Most spans have no background; code uses a
/// distinct fill.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
pub enum FillRole {
#[default]
None,
Code,
}
/// Text attributes that are backend-independent (bold/italic/etc map cleanly
/// onto both terminal modifiers and font variants).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
pub struct TextAttrs {
pub bold: bool,
pub italic: bool,
pub strikethrough: bool,
pub underline: bool,
}
impl TextAttrs {
pub const fn none() -> Self {
Self {
bold: false,
italic: false,
strikethrough: false,
underline: false,
}
}
}
/// A run of text sharing one style. The atomic unit of the neutral model.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct StyledSpan {
pub text: String,
pub role: StyleRole,
pub fill: FillRole,
pub attrs: TextAttrs,
}
impl StyledSpan {
pub fn new(text: impl Into<String>, role: StyleRole) -> Self {
Self {
text: text.into(),
role,
fill: FillRole::None,
attrs: TextAttrs::none(),
}
}
pub fn plain(text: impl Into<String>) -> Self {
Self::new(text, StyleRole::Text)
}
pub fn with_fill(mut self, fill: FillRole) -> Self {
self.fill = fill;
self
}
pub fn with_attrs(mut self, attrs: TextAttrs) -> Self {
self.attrs = attrs;
self
}
pub fn bold(mut self) -> Self {
self.attrs.bold = true;
self
}
pub fn italic(mut self) -> Self {
self.attrs.italic = true;
self
}
}
/// Horizontal alignment for a rendered line.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
pub enum Alignment {
#[default]
Left,
Center,
Right,
}
/// One visual line: a sequence of styled spans plus alignment. This is what
/// front-ends consume; the TUI adapter turns each into a `ratatui::Line`, the
/// desktop adapter into a glyph run.
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct StyledLine {
pub spans: Vec<StyledSpan>,
pub alignment: Alignment,
}
impl StyledLine {
pub fn empty() -> Self {
Self::default()
}
pub fn from_spans(spans: Vec<StyledSpan>) -> Self {
Self {
spans,
alignment: Alignment::Left,
}
}
pub fn aligned(spans: Vec<StyledSpan>, alignment: Alignment) -> Self {
Self { spans, alignment }
}
/// Plain-text content of the line (spans concatenated), for measuring and
/// copy/extraction.
pub fn plain_text(&self) -> String {
self.spans.iter().map(|s| s.text.as_str()).collect()
}
pub fn is_blank(&self) -> bool {
self.spans.iter().all(|s| s.text.trim().is_empty())
}
}
/// Kind of a semantic block, retained so front-ends and copy/extraction logic
/// can reason about structure (e.g. "this block is a fenced code block in
/// language X") without re-parsing.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum BlockKind {
Paragraph,
Heading {
level: u8,
},
CodeBlock {
language: Option<String>,
},
BlockQuote,
ListItem {
ordered: bool,
depth: usize,
},
/// GFM table. The raw cell text per row is carried in [`Block::table`]
/// (the first row is the header); column layout/wrapping is width-dependent
/// and therefore performed by each front-end adapter.
Table,
/// Display math (`$$..$$`). Terminal-friendly rendered math lines are in
/// [`Block::lines`] with [`StyleRole::Math`]; the adapter frames them.
MathDisplay,
ThematicBreak,
Html,
}
/// A semantic block: its kind plus the already-laid-out lines it produced.
/// Front-ends mostly render `lines`; `kind` enables copy targets, spacing
/// policy, and structural styling decisions.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Block {
pub kind: BlockKind,
pub lines: Vec<StyledLine>,
/// Raw table cells (row-major, first row is the header). Only populated for
/// [`BlockKind::Table`]; layout is deferred to the front-end adapter.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub table: Vec<Vec<String>>,
}
impl Block {
pub fn new(kind: BlockKind, lines: Vec<StyledLine>) -> Self {
Self {
kind,
lines,
table: Vec::new(),
}
}
/// Construct a table block carrying raw cells (row-major, header first).
pub fn table(rows: Vec<Vec<String>>) -> Self {
Self {
kind: BlockKind::Table,
lines: Vec::new(),
table: rows,
}
}
}
/// A fully parsed, backend-neutral document.
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct Document {
pub blocks: Vec<Block>,
}
impl Document {
/// All lines across all blocks, in order. Convenient for line-oriented
/// front-ends that don't need block structure.
pub fn lines(&self) -> impl Iterator<Item = &StyledLine> {
self.blocks.iter().flat_map(|b| b.lines.iter())
}
}
crates/jcode-render-core/src/preprocess.rs
+969
View File
@@ -0,0 +1,969 @@
//! Input preprocessing applied before markdown parsing.
//!
//! pulldown-cmark's math extension is aggressive: any `$`-delimited run can be
//! treated as inline math, so plain currency like `$5` or `$5x$` gets parsed as
//! math. The TUI renderer guards against this by escaping `$`-then-digit into
//! `\$` before parsing. We mirror that behavior here so the shared core matches
//! the authoritative renderer.
const DISPLAY_MATH_ENVIRONMENTS: &[&str] = &[
"equation",
"equation*",
"displaymath",
"align",
"align*",
"aligned",
"aligned*",
"gather",
"gather*",
"gathered",
"multline",
"multline*",
"split",
"eqnarray",
"eqnarray*",
"matrix",
"smallmatrix",
"array",
"pmatrix",
"bmatrix",
"Bmatrix",
"vmatrix",
"Vmatrix",
"cases",
"cases*",
];
#[derive(Clone, Copy, PartialEq, Eq)]
enum MathDelimiter {
DollarInline,
DollarDisplay,
LatexInline,
LatexDisplay,
}
/// Normalize common LaTeX math containers into pulldown-cmark's `$` / `$$`
/// syntax before markdown parsing.
///
/// This recognizes `\(...\)`, `\[...\]`, standalone display-math
/// environments, and fenced `math`/`latex`/`tex`/`katex` blocks. Generic code
/// spans and code fences are intentionally left byte-for-byte unchanged.
pub fn normalize_latex_math(text: &str) -> String {
let fenced = normalize_math_fences(text);
let normalized = normalize_latex_delimiters_and_environments(&fenced);
let promoted = promote_standalone_inline_math(&normalized);
stabilize_display_math(&promoted)
}
/// A single-dollar or `\(`/`\)` pair placed on lines by itself is visibly a
/// block construct even though its delimiter spelling is inline. pulldown-cmark
/// does not tokenize inline math across physical newlines, so promote this
/// unambiguous line-oriented form to display math. Embedded multiline inline
/// delimiters keep Markdown's ordinary literal fallback behavior.
fn promote_standalone_inline_math(text: &str) -> String {
let lines: Vec<&str> = text.split_inclusive('\n').collect();
let mut out = String::with_capacity(text.len());
let mut index = 0usize;
let mut fence: Option<(char, usize)> = None;
while index < lines.len() {
let line = lines[index];
let logical = line
.strip_suffix('\n')
.unwrap_or(line)
.trim_end_matches('\r');
if let Some((marker, len)) = fence {
out.push_str(line);
if closing_fence(logical, marker, len) {
fence = None;
}
index += 1;
continue;
}
let Some((prefix, marker_start)) = math_line_prefix(logical, "$") else {
if let Some((_, marker, len, _)) = opening_fence(logical) {
fence = Some((marker, len));
}
out.push_str(line);
index += 1;
continue;
};
let Some(close_offset) = lines[index + 1..].iter().position(|candidate| {
let candidate = candidate
.strip_suffix('\n')
.unwrap_or(candidate)
.trim_end_matches('\r');
prefix
.strip_continuation(candidate)
.and_then(|rest| math_line_prefix(rest, "$"))
.is_some()
}) else {
out.push_str(line);
index += 1;
continue;
};
let close_index = index + 1 + close_offset;
out.push_str(&line[..marker_start]);
out.push_str("$$");
out.push_str(&line[marker_start + 1..]);
for body_line in &lines[index + 1..close_index] {
out.push_str(body_line);
}
let close = lines[close_index];
let close_logical = close
.strip_suffix('\n')
.unwrap_or(close)
.trim_end_matches('\r');
let close_start = close_logical.rfind('$').expect("matched closing marker");
out.push_str(&close[..close_start]);
out.push_str("$$");
out.push_str(&close[close_start + 1..]);
index = close_index + 1;
}
out
}
/// pulldown-cmark recognizes block constructs before it finishes collecting a
/// multiline `$$` span. A perfectly valid TeX line such as `=` can therefore
/// become a Setext heading underline and split the math block. Prefix those
/// interrupting body lines with an empty TeX group (`{}`), which is invisible to
/// both the terminal renderer and a real LaTeX toolchain while making the line
/// unambiguously part of the math span. Unlike flattening the display to one
/// line, this preserves TeX comments and other newline-sensitive source.
fn stabilize_display_math(text: &str) -> String {
let lines: Vec<&str> = text.split_inclusive('\n').collect();
let mut out = String::with_capacity(text.len());
let mut index = 0usize;
let mut fence: Option<(char, usize)> = None;
while index < lines.len() {
let line = lines[index];
let logical = line
.strip_suffix('\n')
.unwrap_or(line)
.trim_end_matches('\r');
if let Some((marker, len)) = fence {
out.push_str(line);
if closing_fence(logical, marker, len) {
fence = None;
}
index += 1;
continue;
}
let Some((prefix, _)) = math_line_prefix(logical, "$$") else {
if let Some((_, marker, len, _)) = opening_fence(logical) {
fence = Some((marker, len));
}
out.push_str(line);
index += 1;
continue;
};
let Some(close_offset) = lines[index + 1..].iter().position(|candidate| {
let candidate = candidate
.strip_suffix('\n')
.unwrap_or(candidate)
.trim_end_matches('\r');
prefix
.strip_continuation(candidate)
.is_some_and(|rest| rest.trim() == "$$")
}) else {
out.push_str(line);
index += 1;
continue;
};
let close_index = index + 1 + close_offset;
out.push_str(line);
for body_line in &lines[index + 1..close_index] {
let without_newline = body_line.strip_suffix('\n').unwrap_or(body_line);
let logical_body = without_newline.trim_end_matches('\r');
let ending = &body_line[logical_body.len()..];
let (container, math) = prefix.split_body(logical_body);
out.push_str(container);
if markdown_interrupts_math(math) {
out.push_str("{}");
}
out.push_str(math);
out.push_str(ending);
}
out.push_str(lines[close_index]);
index = close_index + 1;
}
out
}
#[derive(Debug)]
struct DisplayMathLinePrefix {
continuation: String,
containerized: bool,
}
impl DisplayMathLinePrefix {
fn strip_continuation<'a>(&self, line: &'a str) -> Option<&'a str> {
if self.containerized {
line.strip_prefix(&self.continuation)
} else {
Some(line.strip_prefix(&self.continuation).unwrap_or(line))
}
}
fn split_body<'a>(&self, line: &'a str) -> (&'a str, &'a str) {
match line.strip_prefix(&self.continuation) {
Some(math) => line.split_at(line.len() - math.len()),
None => ("", line),
}
}
}
fn math_line_prefix(line: &str, marker: &str) -> Option<(DisplayMathLinePrefix, usize)> {
let marker_start = line.rfind(marker)?;
if line[marker_start..].trim() != marker
|| (marker == "$" && line[..marker_start].ends_with('$'))
{
return None;
}
let before = &line[..marker_start];
let leading_len = before.len() - before.trim_start_matches([' ', '\t']).len();
let leading = &before[..leading_len];
let mut position = leading_len;
let mut has_quote = false;
while line[position..marker_start].starts_with('>') {
has_quote = true;
position += 1;
if line[position..marker_start].starts_with([' ', '\t']) {
position += 1;
}
}
let list_marker = &line[position..marker_start];
if list_marker.is_empty() {
let leading_columns = leading
.chars()
.map(|ch| if ch == '\t' { 4 } else { 1 })
.sum::<usize>();
if !has_quote && leading_columns > 3 {
return None;
}
return Some((
DisplayMathLinePrefix {
continuation: before.to_string(),
containerized: has_quote,
},
marker_start,
));
}
let marker_width = markdown_list_marker_width(list_marker)?;
Some((
DisplayMathLinePrefix {
continuation: format!("{}{}", &line[..position], " ".repeat(marker_width)),
containerized: true,
},
marker_start,
))
}
fn markdown_list_marker_width(marker: &str) -> Option<usize> {
if matches!(marker, "- " | "* " | "+ ") {
return Some(marker.len());
}
let (number, suffix) = marker.split_once(['.', ')'])?;
(!number.is_empty()
&& number.chars().all(|ch| ch.is_ascii_digit())
&& matches!(suffix, " " | "\t"))
.then_some(marker.len())
}
fn markdown_interrupts_math(line: &str) -> bool {
let trimmed = line.trim();
if trimmed.is_empty() {
return true;
}
let is_setext = trimmed.chars().all(|ch| ch == '=')
|| (trimmed.len() >= 3 && trimmed.chars().all(|ch| ch == '-'));
let is_fence = trimmed.starts_with("```") || trimmed.starts_with("~~~");
let is_atx_heading = trimmed.starts_with('#');
let is_quote = trimmed.starts_with('>');
let is_list = trimmed.starts_with("- ")
|| trimmed.starts_with("* ")
|| trimmed.starts_with("+ ")
|| trimmed
.split_once(['.', ')'])
.is_some_and(|(number, rest)| {
!number.is_empty()
&& number.chars().all(|ch| ch.is_ascii_digit())
&& rest.starts_with([' ', '\t'])
});
is_setext || is_fence || is_atx_heading || is_quote || is_list
}
fn normalize_math_fences(text: &str) -> String {
let lines: Vec<&str> = text.split('\n').collect();
let mut out = String::with_capacity(text.len());
let mut index = 0usize;
while index < lines.len() {
let Some((indent, marker, marker_len, info)) = opening_fence(lines[index]) else {
out.push_str(lines[index]);
if index + 1 < lines.len() {
out.push('\n');
}
index += 1;
continue;
};
let Some(close_offset) = lines[index + 1..]
.iter()
.position(|line| closing_fence(line, marker, marker_len))
else {
out.push_str(lines[index]);
if index + 1 < lines.len() {
out.push('\n');
}
index += 1;
continue;
};
let close_index = index + 1 + close_offset;
if is_math_fence_info(info) {
let body = lines[index + 1..close_index].join("\n");
let body = strip_outer_display_delimiters(&body);
out.push_str(indent);
out.push_str("$$\n");
out.push_str(body.trim_matches('\n'));
out.push('\n');
out.push_str(indent);
out.push_str("$$");
if close_index + 1 < lines.len() {
out.push('\n');
}
} else {
for (offset, line) in lines[index..=close_index].iter().enumerate() {
out.push_str(line);
let is_closing_line = index + offset == close_index;
if close_index + 1 < lines.len() || !is_closing_line {
out.push('\n');
}
}
}
index = close_index + 1;
}
out
}
fn strip_outer_display_delimiters(body: &str) -> &str {
let trimmed = body.trim();
if let Some(inner) = trimmed
.strip_prefix("\\[")
.and_then(|value| value.strip_suffix("\\]"))
{
return inner;
}
if let Some(inner) = trimmed
.strip_prefix("$$")
.and_then(|value| value.strip_suffix("$$"))
{
return inner;
}
body
}
fn opening_fence(line: &str) -> Option<(&str, char, usize, &str)> {
let indent_len = line.len() - line.trim_start_matches([' ', '\t']).len();
let indent = &line[..indent_len];
if indent
.chars()
.map(|ch| if ch == '\t' { 4 } else { 1 })
.sum::<usize>()
> 3
{
return None;
}
let rest = &line[indent_len..];
let marker = rest.chars().next()?;
if !matches!(marker, '`' | '~') {
return None;
}
let marker_len = rest.chars().take_while(|ch| *ch == marker).count();
if marker_len < 3 {
return None;
}
let marker_bytes = marker.len_utf8() * marker_len;
Some((indent, marker, marker_len, rest[marker_bytes..].trim()))
}
fn closing_fence(line: &str, marker: char, minimum_len: usize) -> bool {
let trimmed = line.trim_start_matches([' ', '\t']);
let run = trimmed.chars().take_while(|ch| *ch == marker).count();
run >= minimum_len && trimmed[run * marker.len_utf8()..].trim().is_empty()
}
fn is_math_fence_info(info: &str) -> bool {
let language = info
.split_whitespace()
.next()
.unwrap_or_default()
.trim_matches(['{', '}', '.'])
.to_ascii_lowercase();
matches!(language.as_str(), "math" | "latex" | "tex" | "katex")
}
fn normalize_latex_delimiters_and_environments(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut index = 0usize;
let mut inline_ticks = 0usize;
let mut fence: Option<(char, usize)> = None;
let mut line_start = true;
let mut leading_columns = 0usize;
let mut math_delimiter = None;
let mut wrapped_environment: Option<(String, usize)> = None;
let mut literal_environment: Option<(String, usize)> = None;
while index < text.len() {
let rest = &text[index..];
let ch = rest.chars().next().expect("index stays on a char boundary");
if ch == '\n' {
out.push(ch);
index += 1;
line_start = true;
leading_columns = 0;
continue;
}
if line_start && matches!(ch, ' ' | '\t') {
leading_columns += if ch == '\t' { 4 } else { 1 };
out.push(ch);
index += ch.len_utf8();
continue;
}
if inline_ticks == 0
&& math_delimiter.is_none()
&& line_start
&& leading_columns <= 3
&& matches!(ch, '`' | '~')
{
let run = rest
.chars()
.take_while(|candidate| *candidate == ch)
.count();
if run >= 3 {
let current_line = rest.split_once('\n').map_or(rest, |(line, _)| line);
let trailing = &current_line[run * ch.len_utf8()..];
let is_close = fence.is_some_and(|(marker, len)| {
marker == ch && run >= len && trailing.trim().is_empty()
});
if fence.is_none() || is_close {
fence = if is_close { None } else { Some((ch, run)) };
}
out.push_str(&rest[..run * ch.len_utf8()]);
index += run * ch.len_utf8();
line_start = false;
continue;
}
}
line_start = false;
// Four-space and tab-indented Markdown code is literal just like fenced
// and inline code. Keep the whole logical line byte-for-byte unchanged.
if leading_columns >= 4 {
out.push(ch);
index += ch.len_utf8();
continue;
}
if fence.is_some() {
out.push(ch);
index += ch.len_utf8();
continue;
}
if ch == '`' && math_delimiter.is_none() {
let run = rest
.chars()
.take_while(|candidate| *candidate == '`')
.count();
if inline_ticks == 0 {
inline_ticks = run;
} else if inline_ticks == run {
inline_ticks = 0;
}
out.push_str(&rest[..run]);
index += run;
continue;
}
if inline_ticks > 0 {
out.push(ch);
index += ch.len_utf8();
continue;
}
if let Some((name, depth)) = literal_environment.as_mut() {
if let Some((begin_name, marker_len)) = environment_marker(rest, "begin")
&& begin_name == name
{
*depth += 1;
out.push_str(&rest[..marker_len]);
index += marker_len;
continue;
}
if let Some((end_name, marker_len)) = environment_marker(rest, "end")
&& end_name == name
{
*depth -= 1;
out.push_str(&rest[..marker_len]);
index += marker_len;
if *depth == 0 {
literal_environment = None;
}
continue;
}
out.push(ch);
index += ch.len_utf8();
continue;
}
if ch == '$' && !is_escaped_at(text, index) {
let run = rest
.chars()
.take_while(|candidate| *candidate == '$')
.count()
.min(2);
let looks_like_currency = run == 1
&& rest[1..]
.chars()
.next()
.is_some_and(|next| next.is_ascii_digit());
if !looks_like_currency {
math_delimiter = match (math_delimiter, run) {
(Some(MathDelimiter::DollarInline), 1)
| (Some(MathDelimiter::DollarDisplay), 2) => None,
(None, 1) => Some(MathDelimiter::DollarInline),
(None, 2) => Some(MathDelimiter::DollarDisplay),
(current, _) => current,
};
}
out.push_str(&rest[..run]);
index += run;
continue;
}
if math_delimiter.is_none()
&& rest.starts_with("\\(")
&& !is_escaped_at(text, index)
&& has_unescaped_delimiter(&rest[2..], "\\)")
{
out.push('$');
math_delimiter = Some(MathDelimiter::LatexInline);
index += 2;
continue;
}
if math_delimiter == Some(MathDelimiter::LatexInline)
&& rest.starts_with("\\)")
&& !is_escaped_at(text, index)
{
out.push('$');
math_delimiter = None;
index += 2;
continue;
}
if math_delimiter.is_none()
&& rest.starts_with("\\[")
&& !is_escaped_at(text, index)
&& has_unescaped_delimiter(&rest[2..], "\\]")
&& !looks_like_escaped_markdown_link(rest)
{
out.push_str("$$");
math_delimiter = Some(MathDelimiter::LatexDisplay);
index += 2;
continue;
}
if math_delimiter == Some(MathDelimiter::LatexDisplay)
&& rest.starts_with("\\]")
&& !is_escaped_at(text, index)
{
out.push_str("$$");
math_delimiter = None;
index += 2;
continue;
}
if ch == '\\'
&& math_delimiter.is_none()
&& wrapped_environment.is_none()
&& let Some((name, marker_len)) = environment_marker(rest, "begin")
&& DISPLAY_MATH_ENVIRONMENTS.contains(&name)
{
if has_matching_environment_end(rest, name) {
out.push_str("$$\n");
out.push_str(&rest[..marker_len]);
wrapped_environment = Some((name.to_string(), 1));
} else {
out.push_str(&rest[..marker_len]);
literal_environment = Some((name.to_string(), 1));
}
index += marker_len;
continue;
}
if let Some((name, depth)) = wrapped_environment.as_mut() {
if let Some((begin_name, marker_len)) = environment_marker(rest, "begin")
&& begin_name == name
{
*depth += 1;
out.push_str(&rest[..marker_len]);
index += marker_len;
continue;
}
if let Some((end_name, marker_len)) = environment_marker(rest, "end")
&& end_name == name
{
*depth -= 1;
out.push_str(&rest[..marker_len]);
index += marker_len;
if *depth == 0 {
out.push_str("\n$$");
wrapped_environment = None;
}
continue;
}
}
out.push(ch);
index += ch.len_utf8();
}
out
}
fn environment_marker<'a>(rest: &'a str, command: &str) -> Option<(&'a str, usize)> {
let prefix = format!("\\{command}{{");
let after_prefix = rest.strip_prefix(&prefix)?;
let close = after_prefix.find('}')?;
let name = &after_prefix[..close];
Some((name, prefix.len() + close + 1))
}
fn has_matching_environment_end(source: &str, name: &str) -> bool {
let begin_marker = format!("\\begin{{{name}}}");
let end_marker = format!("\\end{{{name}}}");
let mut search = begin_marker.len();
let mut depth = 1usize;
while search < source.len() {
let next_begin = source[search..]
.find(&begin_marker)
.map(|offset| search + offset);
let next_end = source[search..]
.find(&end_marker)
.map(|offset| search + offset);
match (next_begin, next_end) {
(_, None) => return false,
(Some(begin), Some(end)) if begin < end => {
depth += 1;
search = begin + begin_marker.len();
}
(_, Some(end)) => {
depth -= 1;
if depth == 0 {
return true;
}
search = end + end_marker.len();
}
}
}
false
}
fn is_escaped_at(text: &str, index: usize) -> bool {
let preceding = text[..index]
.chars()
.rev()
.take_while(|ch| *ch == '\\')
.count();
preceding % 2 == 1
}
fn has_unescaped_delimiter(text: &str, delimiter: &str) -> bool {
find_unescaped_delimiter(text, delimiter).is_some()
}
fn find_unescaped_delimiter(text: &str, delimiter: &str) -> Option<usize> {
let mut search_from = 0usize;
while let Some(offset) = text[search_from..].find(delimiter) {
let index = search_from + offset;
if !is_escaped_at(text, index) {
return Some(index);
}
search_from = index + delimiter.len();
}
None
}
fn looks_like_escaped_markdown_link(rest: &str) -> bool {
let Some(close) = find_unescaped_delimiter(&rest[2..], "\\]") else {
return false;
};
rest[2 + close + 2..].starts_with('(')
}
/// Escape dollar signs that look like currency amounts (`$` immediately
/// followed by an ASCII digit) into `\$`, so the math extension does not treat
/// them as inline math. Dollars inside inline code spans and fenced code blocks
/// are left untouched, and already-escaped `\$` is preserved. Display-math
/// `$$` runs are passed through unchanged.
pub fn escape_currency_dollars(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let len = chars.len();
let mut out = String::with_capacity(text.len());
let mut i = 0;
let mut code_fence: Option<(char, usize)> = None;
let mut inline_code_len: usize = 0;
let mut at_line_start = true;
let mut leading_spaces = 0;
let count_marker = |chars: &[char], start: usize, marker: char| {
let mut j = start;
while j < chars.len() && chars[j] == marker {
j += 1;
}
j - start
};
let is_escaped = |chars: &[char], pos: usize| {
let mut backslashes = 0usize;
let mut j = pos;
while j > 0 {
if chars[j - 1] != '\\' {
break;
}
backslashes += 1;
j -= 1;
}
backslashes % 2 == 1
};
while i < len {
let c = chars[i];
if c == '\n' {
at_line_start = true;
leading_spaces = 0;
out.push('\n');
i += 1;
continue;
}
if at_line_start && (c == ' ' || c == '\t') {
leading_spaces += if c == '\t' { 4 } else { 1 };
out.push(c);
i += 1;
continue;
}
let marker_run = if matches!(c, '`' | '~') {
count_marker(&chars, i, c)
} else {
0
};
let maybe_fence = inline_code_len == 0 && marker_run >= 3;
if maybe_fence && at_line_start && leading_spaces <= 3 {
let run = marker_run;
let line_end = chars[i + run..]
.iter()
.position(|ch| *ch == '\n')
.map(|offset| i + run + offset)
.unwrap_or(len);
let trailing_is_blank = chars[i + run..line_end].iter().all(|ch| ch.is_whitespace());
let is_close = code_fence.is_some_and(|(marker, minimum)| {
marker == c && run >= minimum && trailing_is_blank
});
if code_fence.is_none() || is_close {
code_fence = if is_close { None } else { Some((c, run)) };
}
for _ in 0..run {
out.push(c);
}
i += run;
at_line_start = false;
leading_spaces = 0;
continue;
}
if code_fence.is_some() {
out.push(c);
i += 1;
at_line_start = false;
leading_spaces = 0;
continue;
}
if c == '`' {
let run = count_marker(&chars, i, '`');
if inline_code_len > 0 {
if run == inline_code_len {
inline_code_len = 0;
}
for _ in 0..run {
out.push('`');
}
i += run;
at_line_start = false;
leading_spaces = 0;
continue;
}
inline_code_len = run;
for _ in 0..run {
out.push('`');
}
i += run;
at_line_start = false;
leading_spaces = 0;
continue;
}
if at_line_start {
at_line_start = false;
}
if c == ' ' || c == '\t' {
out.push(c);
i += 1;
continue;
}
if inline_code_len > 0 {
out.push(c);
i += 1;
continue;
}
if c == '$' && i + 1 < len && chars[i + 1] == '$' {
out.push_str("$$");
i += 2;
continue;
}
if c == '$' && i + 1 < len && chars[i + 1].is_ascii_digit() {
if is_escaped(&chars, i) {
out.push('$');
} else {
out.push_str("\\$");
}
i += 1;
continue;
}
out.push(c);
i += 1;
}
out
}
#[cfg(test)]
mod tests {
use super::{escape_currency_dollars, normalize_latex_math};
#[test]
fn escapes_currency() {
assert_eq!(escape_currency_dollars("$5"), "\\$5");
assert_eq!(escape_currency_dollars("$5x$"), "\\$5x$");
}
#[test]
fn leaves_real_math() {
assert_eq!(escape_currency_dollars("$a+b$"), "$a+b$");
}
#[test]
fn passes_through_display_math() {
assert_eq!(escape_currency_dollars("$$x=5$$"), "$$x=5$$");
}
#[test]
fn skips_inline_code() {
assert_eq!(escape_currency_dollars("`$5`"), "`$5`");
}
#[test]
fn skips_fenced_code() {
let input = "```\n$5\n```";
assert_eq!(escape_currency_dollars(input), input);
let tilde = "~~~text\n$5\n~~~";
assert_eq!(escape_currency_dollars(tilde), tilde);
let longer = "````text\n``` is content and $5\n````";
assert_eq!(escape_currency_dollars(longer), longer);
}
#[test]
fn preserves_already_escaped() {
assert_eq!(escape_currency_dollars("\\$5"), "\\$5");
}
#[test]
fn normalizes_parenthesized_and_bracketed_math() {
assert_eq!(
normalize_latex_math(r"Inline \(x^2 + \alpha\) done."),
r"Inline $x^2 + \alpha$ done."
);
assert_eq!(
normalize_latex_math("\\[\n\\frac{x+1}{y}\n\\]"),
"$$\n\\frac{x+1}{y}\n$$"
);
}
#[test]
fn normalizes_standalone_display_environments() {
let input = "before\n\\begin{align*}\nx &= 1 \\\\\ny &= 2\n\\end{align*}\nafter";
let normalized = normalize_latex_math(input);
assert!(normalized.contains("$$\n\\begin{align*}"), "{normalized}");
assert!(normalized.contains("\\end{align*}\n$$"), "{normalized}");
let nested = r"\begin{matrix}\begin{matrix}a\end{matrix}\end{matrix}";
assert_eq!(normalize_latex_math(nested), format!("$$\n{nested}\n$$"));
let unmatched = r"\begin{matrix}\begin{matrix}a\end{matrix}";
assert_eq!(normalize_latex_math(unmatched), unmatched);
}
#[test]
fn normalizes_explicit_math_fences() {
assert_eq!(
normalize_latex_math("```math\n\\frac{a}{b}\n```"),
"$$\n\\frac{a}{b}\n$$"
);
assert_eq!(
normalize_latex_math("~~~latex\n\\[\nx^2\n\\]\n~~~"),
"$$\nx^2\n$$"
);
}
#[test]
fn preserves_literal_code_and_malformed_delimiters() {
let inline = r"Use `\(x^2\)` literally";
let fenced = "```rust\nlet latex = r\"\\[x\\]\";\n```";
let plain_fenced = "```\nlet x = 1;\n```";
let escaped_link = r"a \[link\](https://example.com) reference";
let malformed = r"unfinished \[x^2";
assert_eq!(normalize_latex_math(inline), inline);
assert_eq!(normalize_latex_math(fenced), fenced);
assert_eq!(normalize_latex_math(plain_fenced), plain_fenced);
assert_eq!(normalize_latex_math(escaped_link), escaped_link);
assert_eq!(normalize_latex_math(malformed), malformed);
}
#[test]
fn currency_does_not_hide_later_latex_math() {
assert_eq!(
normalize_latex_math(r"Costs $35. Then \[x^2\]."),
r"Costs $35. Then $$x^2$$."
);
}
}
crates/jcode-render-core/src/reasoning.rs
+94
View File
@@ -0,0 +1,94 @@
//! Reasoning-line markdown formatting.
//!
//! Pure string helpers shared by the server/streaming path and the TUI renderer
//! so the wrapping/escaping rules stay in lockstep with the renderer that
//! consumes them. These live in `jcode-render-core` (a backend-neutral, pure
//! crate) rather than in `jcode-tui-markdown` so the foundation/streaming layer
//! can format reasoning lines without depending on any `jcode-tui-*` crate.
/// Invisible separator placed just inside both ends of an emphasis run so the
/// flanking `*` are always adjacent to non-whitespace (see
/// [`reasoning_line_markup`]).
pub const REASONING_SENTINEL: &str = "\u{2063}";
/// Escape the characters that would otherwise be interpreted as inline markdown
/// inside a reasoning line, so the body renders literally inside the dim/italic
/// emphasis run.
fn escape_reasoning_inline_markdown(line: &str) -> String {
let mut out = String::with_capacity(line.len() + 8);
for ch in line.chars() {
match ch {
'\\' | '*' | '_' | '`' | '[' | ']' | '<' | '>' | '&' | '~' | '|' | '$' => {
out.push('\\');
out.push(ch);
}
_ => out.push(ch),
}
}
out
}
/// Wrap a completed reasoning line as dim+italic markdown.
///
/// Empty lines become a bare newline (no empty emphasis run). The result always
/// ends in a CommonMark hard break (`" \n"`).
///
/// The trailing two spaces are a CommonMark *hard break*: without them,
/// consecutive reasoning lines (each terminated by a single `\n`) collapse into
/// one paragraph where the line breaks render as spaces, so multi-line thinking
/// shows up as a single run-on line. The hard break keeps each reasoning line on
/// its own visual row, matching the model's line structure.
///
/// The sentinel must wrap both ends because CommonMark's emphasis flanking rules
/// require the opening `*` to not be followed by whitespace and the closing `*`
/// to not be preceded by whitespace. A reasoning line that starts or ends with
/// whitespace (or is whitespace-only) would otherwise leave the asterisks as
/// literal text and break the dim/italic styling. The zero-width sentinels
/// guarantee both asterisks are flanked by non-whitespace regardless of the body.
pub fn reasoning_line_markup(line: &str) -> String {
if line.is_empty() {
"\n".to_string()
} else {
format!(
"*{0}{1}{0}* \n",
REASONING_SENTINEL,
escape_reasoning_inline_markdown(line)
)
}
}
/// Wrap the in-progress (not yet newline-terminated) reasoning line as dim+italic
/// markdown, identical to [`reasoning_line_markup`] but *without* the trailing
/// newline so it renders as the live tail of the streaming buffer. Callers
/// truncate and re-emit this tail on each streamed delta so reasoning trickles in
/// token-by-token instead of one whole line at a time. An empty line yields an
/// empty string (nothing to render yet).
pub fn reasoning_partial_markup(line: &str) -> String {
if line.is_empty() {
String::new()
} else {
format!(
"*{0}{1}{0}*",
REASONING_SENTINEL,
escape_reasoning_inline_markdown(line)
)
}
}
/// One-line collapsed reasoning summary markup (e.g. `▸ thought (3 lines)`),
/// styled dim+italic like the live reasoning lines. Used to fold a persisted
/// reasoning block down to a single trace line when the transcript is
/// re-rendered from history in `current` reasoning-display mode (so reloaded /
/// resumed sessions match the live collapse instead of replaying every line).
///
/// Lives here (a backend-neutral, pure crate) rather than in `jcode-tui-markdown`
/// so the foundation/streaming layer can format the summary without depending on
/// any `jcode-tui-*` crate. Re-exported from `jcode-tui-markdown` for the
/// existing `jcode_tui_markdown::reasoning_summary_line_markup` path.
pub fn reasoning_summary_line_markup(line_count: usize) -> String {
let label = match line_count {
0 | 1 => "▸ thought".to_string(),
n => format!("▸ thought ({} lines)", n),
};
reasoning_line_markup(&label)
}
crates/jcode-render-core/src/tests.rs
+235
View File
@@ -0,0 +1,235 @@
use crate::{
Alignment, BlockKind, ColumnWidth, FillRole, StyleRole, StyledLine, StyledSpan, WidthMeasure,
parse_markdown, wrap_line,
};
fn block_kinds(doc: &crate::Document) -> Vec<&BlockKind> {
doc.blocks.iter().map(|b| &b.kind).collect()
}
#[test]
fn parses_heading() {
let doc = parse_markdown("# Hello world");
assert_eq!(doc.blocks.len(), 1);
assert_eq!(doc.blocks[0].kind, BlockKind::Heading { level: 1 });
let line = &doc.blocks[0].lines[0];
assert_eq!(line.plain_text(), "Hello world");
assert!(line.spans.iter().all(|s| s.attrs.bold));
assert!(line.spans.iter().all(|s| s.role == StyleRole::Strong));
}
#[test]
fn parses_paragraph_with_emphasis() {
let doc = parse_markdown("plain *italic* **bold**");
assert_eq!(doc.blocks.len(), 1);
assert_eq!(doc.blocks[0].kind, BlockKind::Paragraph);
let spans = &doc.blocks[0].lines[0].spans;
let italic = spans.iter().find(|s| s.text == "italic").unwrap();
assert!(italic.attrs.italic && !italic.attrs.bold);
let bold = spans.iter().find(|s| s.text == "bold").unwrap();
assert!(bold.attrs.bold);
assert_eq!(bold.role, StyleRole::Strong);
}
#[test]
fn parses_inline_code() {
let doc = parse_markdown("use `cargo build` now");
let spans = &doc.blocks[0].lines[0].spans;
let code = spans.iter().find(|s| s.text == "cargo build").unwrap();
assert_eq!(code.role, StyleRole::Code);
assert_eq!(code.fill, FillRole::Code);
}
#[test]
fn parses_inline_math_into_unicode_math_span() {
let doc = parse_markdown(r"Euler: $e^{i\pi} + 1 = 0$");
let math = doc.blocks[0].lines[0]
.spans
.iter()
.find(|span| span.role == StyleRole::Math)
.expect("math span");
assert_eq!(math.text, "e^(iπ) + 1 = 0");
}
#[test]
fn parses_display_math_into_laid_out_block() {
let doc = parse_markdown(r"$$\frac{x+1}{y}$$");
assert_eq!(doc.blocks[0].kind, BlockKind::MathDisplay);
let lines: Vec<String> = doc.blocks[0]
.lines
.iter()
.map(StyledLine::plain_text)
.collect();
assert_eq!(lines, vec![" x+1 ", "─────", " y "]);
assert!(
doc.blocks[0]
.lines
.iter()
.flat_map(|line| &line.spans)
.all(|span| span.role == StyleRole::Math)
);
}
#[test]
fn parses_all_common_latex_math_containers() {
let inline = parse_markdown(r"Euler: \(e^{i\pi} + 1 = 0\)");
assert!(
inline.blocks[0].lines[0]
.spans
.iter()
.any(|span| span.role == StyleRole::Math && span.text == "e^(iπ) + 1 = 0")
);
for markdown in [
r"\[\frac{x+1}{y}\]",
"```math\n\\frac{x+1}{y}\n```",
r"\begin{equation}\frac{x+1}{y}\end{equation}",
] {
let doc = parse_markdown(markdown);
assert_eq!(doc.blocks[0].kind, BlockKind::MathDisplay, "{markdown}");
assert!(
doc.blocks[0]
.lines
.iter()
.any(|line| line.plain_text().contains("─────")),
"{markdown}: {:?}",
doc.blocks[0].lines
);
}
}
#[test]
fn parses_fenced_code_block() {
let md = "```rust\nfn main() {}\nlet x = 1;\n```";
let doc = parse_markdown(md);
assert_eq!(
block_kinds(&doc),
vec![&BlockKind::CodeBlock {
language: Some("rust".to_string())
}]
);
let lines: Vec<String> = doc.blocks[0].lines.iter().map(|l| l.plain_text()).collect();
assert_eq!(lines, vec!["fn main() {}", "let x = 1;"]);
assert!(
doc.blocks[0].lines[0]
.spans
.iter()
.all(|s| s.fill == FillRole::Code)
);
}
#[test]
fn parses_unordered_list_with_markers() {
let doc = parse_markdown("- one\n- two");
let items: Vec<_> = doc
.blocks
.iter()
.filter(|b| matches!(b.kind, BlockKind::ListItem { .. }))
.collect();
assert_eq!(items.len(), 2);
assert!(items[0].lines[0].plain_text().starts_with("• "));
assert!(items[0].lines[0].plain_text().contains("one"));
}
#[test]
fn parses_ordered_list_numbers() {
let doc = parse_markdown("1. first\n2. second");
let items: Vec<_> = doc
.blocks
.iter()
.filter(|b| matches!(b.kind, BlockKind::ListItem { ordered: true, .. }))
.collect();
assert_eq!(items.len(), 2);
assert!(items[0].lines[0].plain_text().starts_with("1. "));
assert!(items[1].lines[0].plain_text().starts_with("2. "));
}
#[test]
fn parses_nested_list_depth() {
let doc = parse_markdown("- top\n - nested");
let depths: Vec<usize> = doc
.blocks
.iter()
.filter_map(|b| match b.kind {
BlockKind::ListItem { depth, .. } => Some(depth),
_ => None,
})
.collect();
assert!(depths.contains(&0));
assert!(depths.contains(&1));
}
#[test]
fn parses_blockquote() {
let doc = parse_markdown("> quoted text");
assert!(
doc.blocks
.iter()
.any(|b| b.kind == BlockKind::BlockQuote && b.lines[0].plain_text().contains("quoted"))
);
}
#[test]
fn parses_thematic_break() {
let doc = parse_markdown("a\n\n---\n\nb");
assert!(
doc.blocks
.iter()
.any(|b| b.kind == BlockKind::ThematicBreak)
);
}
#[test]
fn wrap_preserves_styling_and_width() {
let line = StyledLine::from_spans(vec![
StyledSpan::new("hello ", StyleRole::Text),
StyledSpan::new("beautiful", StyleRole::Strong).bold(),
StyledSpan::new(" world", StyleRole::Text),
]);
let wrapped = wrap_line(&line, 8, &ColumnWidth);
assert!(wrapped.len() > 1);
// Every produced line is within the width budget.
for l in &wrapped {
assert!(
ColumnWidth.measure(&l.plain_text()) <= 8,
"line too wide: {:?}",
l.plain_text()
);
}
// The bold word retains its styling across the wrap (it may be split).
let has_bold = wrapped
.iter()
.flat_map(|l| l.spans.iter())
.any(|s| s.attrs.bold && s.role == StyleRole::Strong && !s.text.trim().is_empty());
assert!(has_bold);
}
#[test]
fn wrap_hard_splits_overlong_word() {
let line = StyledLine::from_spans(vec![StyledSpan::plain("abcdefghij")]);
let wrapped = wrap_line(&line, 4, &ColumnWidth);
assert!(wrapped.len() >= 3);
for l in &wrapped {
assert!(ColumnWidth.measure(&l.plain_text()) <= 4);
}
let joined: String = wrapped.iter().map(|l| l.plain_text()).collect();
assert_eq!(joined, "abcdefghij");
}
#[test]
fn wrap_keeps_alignment() {
let line = StyledLine::aligned(
vec![StyledSpan::plain("one two three four five")],
Alignment::Center,
);
let wrapped = wrap_line(&line, 7, &ColumnWidth);
assert!(wrapped.iter().all(|l| l.alignment == Alignment::Center));
}
#[test]
fn wrap_noop_when_fits() {
let line = StyledLine::from_spans(vec![StyledSpan::plain("short")]);
let wrapped = wrap_line(&line, 80, &ColumnWidth);
assert_eq!(wrapped.len(), 1);
assert_eq!(wrapped[0].plain_text(), "short");
}
crates/jcode-render-core/src/wrap.rs
+187
View File
@@ -0,0 +1,187 @@
//! Backend-neutral width measurement and line wrapping.
//!
//! Wrapping needs to know how "wide" text is, but width is backend-specific:
//! the terminal measures in monospace columns, the desktop measures in pixels
//! via font metrics. We abstract that behind [`WidthMeasure`] so the wrapping
//! algorithm itself is shared.
use crate::model::{StyledLine, StyledSpan};
/// Measures the display width of text in a backend's units.
pub trait WidthMeasure {
/// Width of a whole string in this backend's units.
fn measure(&self, text: &str) -> usize;
/// Width of a single `char`. Defaults to measuring a one-char string, but
/// backends can override for speed.
fn measure_char(&self, ch: char) -> usize {
let mut buf = [0u8; 4];
self.measure(ch.encode_utf8(&mut buf))
}
}
/// Terminal-style measurer: Unicode display columns (what ratatui uses).
#[derive(Debug, Clone, Copy, Default)]
pub struct ColumnWidth;
impl WidthMeasure for ColumnWidth {
fn measure(&self, text: &str) -> usize {
use unicode_width::UnicodeWidthStr;
text.width()
}
fn measure_char(&self, ch: char) -> usize {
use unicode_width::UnicodeWidthChar;
ch.width().unwrap_or(0)
}
}
/// Wrap a single styled line to `max_width` (in the measurer's units),
/// preserving span styling across the wrap boundary. Wrapping prefers to break
/// at whitespace; words longer than `max_width` are hard-split.
///
/// Alignment is carried onto every produced line.
pub fn wrap_line<M: WidthMeasure>(
line: &StyledLine,
max_width: usize,
measure: &M,
) -> Vec<StyledLine> {
if max_width == 0 {
return vec![line.clone()];
}
if measure.measure(&line.plain_text()) <= max_width {
return vec![line.clone()];
}
let mut out: Vec<StyledLine> = Vec::new();
let mut cur: Vec<StyledSpan> = Vec::new();
let mut cur_width = 0usize;
let push_line = |out: &mut Vec<StyledLine>, spans: Vec<StyledSpan>| {
out.push(StyledLine {
spans,
alignment: line.alignment,
});
};
for span in &line.spans {
// Split the span text into word / whitespace runs, keeping whitespace
// so we can drop a trailing space at a wrap point but keep interior
// spacing.
for token in tokenize(&span.text) {
let tok_width = measure.measure(token);
let is_ws = token.chars().all(char::is_whitespace);
if cur_width + tok_width <= max_width {
push_token(&mut cur, span, token);
cur_width += tok_width;
continue;
}
// Token doesn't fit on the current line.
if is_ws {
// Whitespace at a wrap boundary is dropped; start a fresh line.
if !cur.is_empty() {
push_line(&mut out, std::mem::take(&mut cur));
cur_width = 0;
}
continue;
}
// Flush what we have, then place the word (hard-splitting if the
// word itself is wider than max_width).
if !cur.is_empty() {
push_line(&mut out, std::mem::take(&mut cur));
cur_width = 0;
}
if tok_width <= max_width {
push_token(&mut cur, span, token);
cur_width += tok_width;
} else {
// Hard-split an over-long word by char width.
let mut chunk = String::new();
let mut chunk_w = 0usize;
for ch in token.chars() {
let cw = measure.measure_char(ch);
if chunk_w + cw > max_width && !chunk.is_empty() {
push_token(&mut cur, span, &chunk);
push_line(&mut out, std::mem::take(&mut cur));
chunk.clear();
chunk_w = 0;
}
chunk.push(ch);
chunk_w += cw;
}
if !chunk.is_empty() {
push_token(&mut cur, span, &chunk);
cur_width = chunk_w;
}
}
}
}
if !cur.is_empty() {
push_line(&mut out, cur);
}
if out.is_empty() {
out.push(StyledLine {
spans: vec![],
alignment: line.alignment,
});
}
out
}
/// Wrap many lines.
pub fn wrap_lines<M: WidthMeasure>(
lines: &[StyledLine],
max_width: usize,
measure: &M,
) -> Vec<StyledLine> {
lines
.iter()
.flat_map(|l| wrap_line(l, max_width, measure))
.collect()
}
fn push_token(cur: &mut Vec<StyledSpan>, src: &StyledSpan, text: &str) {
// Merge into the previous span when it shares styling, to avoid span
// fragmentation across token boundaries.
if let Some(last) = cur.last_mut()
&& last.role == src.role
&& last.fill == src.fill
&& last.attrs == src.attrs
{
last.text.push_str(text);
return;
}
cur.push(StyledSpan {
text: text.to_string(),
role: src.role,
fill: src.fill,
attrs: src.attrs,
});
}
/// Split text into alternating non-whitespace / whitespace runs.
fn tokenize(text: &str) -> Vec<&str> {
let mut out = Vec::new();
let mut start = 0usize;
let mut prev_ws: Option<bool> = None;
for (idx, ch) in text.char_indices() {
let is_ws = ch.is_whitespace();
match prev_ws {
Some(p) if p != is_ws => {
out.push(&text[start..idx]);
start = idx;
}
_ => {}
}
prev_ws = Some(is_ws);
}
if start < text.len() {
out.push(&text[start..]);
}
out
}
crates/jcode-render-core/tests/latex_robustness.rs
+515
View File
@@ -0,0 +1,515 @@
use jcode_render_core::{
BlockKind, StyleRole, normalize_latex_math, parse_markdown, render_display_latex,
render_inline_latex,
};
use unicode_width::UnicodeWidthStr;
fn assert_visible_and_deterministic(source: &str) {
let inline = render_inline_latex(source);
let display = render_display_latex(source);
assert_eq!(inline, render_inline_latex(source), "inline: {source:?}");
assert_eq!(display, render_display_latex(source), "display: {source:?}");
if !source.trim().is_empty() {
assert!(
!inline.trim().is_empty(),
"inline content vanished: {source:?}"
);
assert!(
display.iter().any(|line| !line.trim().is_empty()),
"display content vanished: {source:?}"
);
}
}
#[test]
fn renders_the_supported_symbol_vocabulary() {
let cases = [
(r"\alpha \beta \gamma \delta \epsilon \varepsilon", "αβγδεε"),
(
r"\theta \vartheta \lambda \mu \pi \varpi \phi \varphi \omega",
"θϑλμπϖφϕω",
),
(
r"\Gamma \Delta \Theta \Lambda \Xi \Pi \Sigma \Phi \Psi \Omega",
"ΓΔΘΛΞΠΣΦΨΩ",
),
(
r"\sum \prod \coprod \int \iint \iiint \oint \partial \nabla \infty",
"∑∏∐∫∬∭∮∂∇∞",
),
(r"\times \div \cdot \circ \pm \mp \ast \star", "×÷·∘±∓∗⋆"),
(
r"\le \leq \ge \geq \ne \neq \approx \sim \simeq \equiv \propto",
"≤≤≥≥≠≠≈∼≃≡∝",
),
(
r"\in \notin \ni \subset \supset \subseteq \supseteq \cup \cap \setminus",
"∈∉∋⊂⊃⊆⊇∪∩∖",
),
(
r"\forall \exists \nexists \neg \land \lor \oplus \otimes \vdash \models",
"∀∃∄¬∧∨⊕⊗⊢⊨",
),
(
r"\to \leftarrow \leftrightarrow \Rightarrow \Leftarrow \Leftrightarrow \mapsto \uparrow \downarrow",
"→←↔⇒⇐⇔↦↑↓",
),
(
r"\ldots \cdots \vdots \ddots \angle \degree \prime \perp \parallel \mid",
"…⋯⋮⋱∠°′⊥∥∣",
),
(
r"\langle x \rangle \lceil x \rceil \lfloor x \rfloor",
"⟨x ⟩⌈x ⌉⌊x ⌋",
),
(r"\emptyset \varnothing \ell \hbar", "∅∅ℓℏ"),
];
for (source, expected) in cases {
assert_eq!(render_inline_latex(source), expected, "{source}");
}
}
#[test]
fn renders_all_supported_script_characters_and_falls_back_for_others() {
assert_eq!(
render_inline_latex("x^{0123456789+-=()ni}"),
"x⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ"
);
assert_eq!(
render_inline_latex("x_{0123456789+-=()aehijklmnoprstuvx}"),
"x₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₕᵢⱼₖₗₘₙₒₚᵣₛₜᵤᵥₓ"
);
assert_eq!(render_inline_latex("x^{ab}_{qy}"), "x_(qy)^(ab)");
assert_eq!(render_inline_latex("x_1^2_3^4"), "x₁₃²⁴");
}
#[test]
fn formatting_commands_preserve_content_and_spacing() {
for command in [
"mathbf",
"mathrm",
"mathit",
"mathsf",
"mathtt",
"mathcal",
"mathbb",
"boldsymbol",
"displaystyle",
"scriptstyle",
] {
assert_eq!(
render_inline_latex(&format!(r"\{command}{{x_2+y}}")),
"x₂+y",
"{command}"
);
}
assert_eq!(
render_inline_latex(r"a\,b\;c\:d\quad e\qquad f\!g"),
"a b c d e fg"
);
assert_eq!(
render_inline_latex(r"\text{rate = 5 percent}"),
"rate = 5 percent"
);
assert_eq!(
render_inline_latex(r"\operatorname{arg max}_x f(x)"),
"arg maxₓ f(x)"
);
}
#[test]
fn accents_lines_and_delimiters_are_stable_unicode() {
assert_eq!(
render_inline_latex(r"\hat{x}\bar{y}\vec{v}\dot{x}\ddot{y}\tilde{z}"),
"x̂ȳv⃗ẋÿz̃"
);
assert_eq!(render_inline_latex(r"\overline{ab}"), "a̅b̅");
assert_eq!(render_inline_latex(r"\underline{ab}"), "a̲b̲");
assert_eq!(render_inline_latex(r"\left( x \right)"), "( x )");
assert_eq!(render_inline_latex(r"\left\{ x \right\}"), "{ x }");
assert_eq!(render_inline_latex(r"\bigl\langle x \bigr\rangle"), "⟨ x ⟩");
assert_eq!(render_inline_latex(r"\left. x \right|"), " x |");
}
#[test]
fn fraction_root_and_script_layouts_have_consistent_geometry() {
let cases = [
r"\frac{a}{b}",
r"\frac{x+1}{y-z}",
r"\frac{\frac{a}{b}}{\sqrt{x}}",
r"x^{a+b}_{i-j}",
r"\sqrt[3]{\frac{x}{y}}",
r"A+\frac{界}{\alpha}+B",
];
for source in cases {
let lines = render_display_latex(source);
assert!(!lines.is_empty(), "{source}");
let width = lines.iter().map(|line| line.width()).max().unwrap();
assert!(width > 0, "{source}: {lines:?}");
assert!(
lines.iter().all(|line| line.width() <= width),
"{source}: {lines:?}"
);
assert!(
lines.iter().all(|line| !line.contains('\t')),
"{source}: {lines:?}"
);
}
}
#[test]
fn every_matrix_environment_has_the_expected_delimiter_family() {
let cases = [
("matrix", "", ""),
("smallmatrix", "", ""),
("array", "", ""),
("pmatrix", "⎛", "⎞"),
("bmatrix", "⎡", "⎤"),
("Bmatrix", "⎧", "⎫"),
("vmatrix", "│", "│"),
("Vmatrix", "‖", "‖"),
("cases", "⎧", ""),
];
for (environment, left, right) in cases {
let source = if environment == "array" {
r"\begin{array}{cc}a & bb \\ ccc & d\end{array}".to_string()
} else {
format!(r"\begin{{{environment}}}a & bb \\ ccc & d\end{{{environment}}}")
};
let lines = render_display_latex(&source);
assert_eq!(lines.len(), 2, "{environment}: {lines:?}");
if !left.is_empty() {
assert!(lines[0].starts_with(left), "{environment}: {lines:?}");
}
if !right.is_empty() {
assert!(lines[0].ends_with(right), "{environment}: {lines:?}");
}
assert_eq!(
lines[0].width(),
lines[1].width(),
"{environment}: {lines:?}"
);
}
}
#[test]
fn matrices_handle_ragged_rows_nested_groups_and_row_spacing() {
let ragged = render_display_latex(r"\begin{bmatrix}a & bb & ccc \\ d \\ e & ff\end{bmatrix}");
assert_eq!(ragged.len(), 3);
assert!(
ragged.iter().all(|line| line.width() == ragged[0].width()),
"{ragged:?}"
);
assert_eq!(
render_inline_latex(r"\begin{matrix}{a & b} & \frac{c&d}{e} \\ x & y\end{matrix}"),
"a & b, (c&d)⁄e; x, y"
);
assert_eq!(
render_inline_latex(r"\begin{matrix}a\\[12pt]b\\[-2pt]c\\\end{matrix}"),
"a; b; c"
);
}
#[test]
fn ordinary_display_environments_render_their_body() {
for environment in ["equation", "equation*", "displaymath"] {
let source = format!(r"\begin{{{environment}}}\frac{{x}}{{y}}\end{{{environment}}}");
let lines = render_display_latex(&source);
assert!(
lines.iter().any(|line| line.contains('─')),
"{source}: {lines:?}"
);
assert!(
!lines.iter().any(|line| line.contains("begin")),
"{source}: {lines:?}"
);
}
}
#[test]
fn unknown_commands_and_environments_remain_debuggable() {
let cases = [
(r"\unknown", r"\unknown"),
(r"\unknown value", r"\unknown value"),
(r"\unknown{value}", r"\unknownvalue"),
(
r"\begin{mystery}x+y\end{mystery}",
r"\begin{mystery}x+y\end{mystery}",
),
(r"\end{orphan}", r"\endorphan"),
];
for (source, expected) in cases {
assert_eq!(render_inline_latex(source), expected, "{source}");
}
}
#[test]
fn malformed_constructs_never_panic_and_keep_diagnostic_content() {
let cases = [
"{",
"}",
"[",
"]",
"^",
"_",
"\\",
r"\frac",
r"\frac{}",
r"\frac{}{}",
r"\sqrt",
r"\sqrt[",
r"\sqrt[]{}",
r"\text",
r"\left",
r"\right",
r"\begin",
r"\begin{}",
r"\begin{matrix}",
r"\begin{matrix}a&b\end{pmatrix}",
r"x_{",
r"x^{}}",
r"x^^__",
"😀_{界",
"\0\u{1b}[31m",
];
for source in cases {
assert_visible_and_deterministic(source);
}
}
#[test]
fn generated_latex_grammar_corpus_is_total_and_deterministic() {
let atoms = ["x", "7", "α", "界", "😀", r"\beta", r"\unknown"];
let wrappers = [
(r"{", "}"),
(r"\frac{", "}{2}"),
(r"\sqrt{", "}"),
(r"\sqrt[3]{", "}"),
(r"\mathbf{", "}"),
(r"\left(", r"\right)"),
(r"\begin{bmatrix}", r" & y \\ z & w\end{bmatrix}"),
];
let scripts = ["", "_1", "^2", "_{i+j}", "^{n-1}", "_i^2"];
for atom in atoms {
for (open, close) in wrappers {
for script in scripts {
let source = format!("{open}{atom}{close}{script}");
assert_visible_and_deterministic(&source);
}
}
}
}
#[test]
fn long_flat_inputs_and_deep_commands_remain_bounded() {
let flat = format!("{}{}", r"\alpha+".repeat(20_000), "x");
let inline = render_inline_latex(&flat);
assert!(inline.ends_with('x'));
assert!(inline.len() < flat.len());
let mut nested = "x".to_string();
for _ in 0..2_000 {
nested = format!(r"\sqrt{{{nested}}}");
}
std::thread::Builder::new()
.stack_size(256 * 1024)
.spawn(move || assert_visible_and_deterministic(&nested))
.expect("spawn bounded stack test")
.join()
.expect("deep command rendering must not panic");
}
#[test]
fn normalizes_every_supported_math_fence_spelling() {
for info in [
"math",
"MATH",
"latex",
"LaTeX",
"tex",
"TEX",
"katex",
"KaTeX",
"{.math}",
"{latex}",
"math title=demo",
] {
let source = format!("```{info}\n\\alpha_2\n```");
assert_eq!(normalize_latex_math(&source), "$$\n\\alpha_2\n$$", "{info}");
}
assert_eq!(
normalize_latex_math(" ~~~~latex\n\\[x^2\\]\n ~~~~~"),
" $$\nx^2\n $$"
);
}
#[test]
fn normalization_protects_all_literal_code_forms() {
let cases = [
r"`\(x\)`",
r"``code ` and \[x\]``",
"```rust\n\\(x\\)\n```",
"~~~text\n\\begin{matrix}x\\end{matrix}\n~~~",
" \\[indented code\\]",
"\t\\(tab-indented code\\)",
];
for source in cases {
assert_eq!(normalize_latex_math(source), source, "{source:?}");
}
}
#[test]
fn fence_like_content_does_not_end_a_generic_code_fence() {
for source in [
"```text\n```not a closing fence\n\\(literal\\)\n```",
"~~~~text\n~~~~still content\n\\[literal\\]\n~~~~",
"````rust\n``` shorter fence\n\\begin{matrix}x\\end{matrix}\n````",
] {
assert_eq!(normalize_latex_math(source), source, "{source:?}");
}
assert_eq!(
normalize_latex_math("```text\n\\(literal\\)\n```\n\n\\(real math\\)"),
"```text\n\\(literal\\)\n```\n\n$real math$"
);
}
#[test]
fn normalizes_delimiters_only_when_balanced_and_unescaped() {
let cases = [
(r"before \(x^2\) after", r"before $x^2$ after"),
(r"before \[x^2\] after", r"before $$x^2$$ after"),
(r"\\(literal\\)", r"\\(literal\\)"),
(r"\(missing", r"\(missing"),
(r"missing\)", r"missing\)"),
(r"\[label\](url)", r"\[label\](url)"),
(r"$x + \(y\)$", r"$x + \(y\)$"),
(r"$$x + \[y\]$$", r"$$x + \[y\]$$"),
];
for (source, expected) in cases {
assert_eq!(normalize_latex_math(source), expected, "{source}");
}
}
#[test]
fn normalizes_all_standalone_display_environments() {
let environments = [
"equation",
"equation*",
"displaymath",
"align",
"align*",
"aligned",
"aligned*",
"gather",
"gather*",
"gathered",
"multline",
"multline*",
"split",
"eqnarray",
"eqnarray*",
"matrix",
"smallmatrix",
"array",
"pmatrix",
"bmatrix",
"Bmatrix",
"vmatrix",
"Vmatrix",
"cases",
"cases*",
];
for environment in environments {
let source = format!(r"prefix \begin{{{environment}}}x\end{{{environment}}} suffix");
let normalized = normalize_latex_math(&source);
assert!(normalized.contains("$$\n"), "{environment}: {normalized:?}");
assert!(normalized.contains("\n$$"), "{environment}: {normalized:?}");
assert!(normalized.contains(&format!(r"\begin{{{environment}}}x\end{{{environment}}}")));
}
}
#[test]
fn environment_normalization_handles_nesting_and_mismatches() {
let nested = r"\begin{align}a\begin{align}b\end{align}c\end{align}";
let normalized = normalize_latex_math(nested);
assert_eq!(normalized.matches("$$").count(), 2, "{normalized}");
assert!(normalized.contains(nested));
for source in [
r"\begin{align}x",
r"\begin{align}x\end{equation}",
r"\begin{unknown}x\end{unknown}",
] {
assert_eq!(normalize_latex_math(source), source, "{source}");
}
}
#[test]
fn markdown_pipeline_preserves_structure_and_math_roles() {
let markdown = concat!(
"Price: $35.00 and inline \\(x_2 + \\alpha\\).\n\n",
"\\[\\frac{x+1}{y}\\]\n\n",
"```rust\nlet literal = r\"\\(x\\)\";\n```\n"
);
let doc = parse_markdown(markdown);
assert!(
doc.blocks
.iter()
.any(|block| block.kind == BlockKind::Paragraph)
);
let display = doc
.blocks
.iter()
.find(|block| block.kind == BlockKind::MathDisplay)
.expect("display math block");
assert!(
display
.lines
.iter()
.all(|line| line.spans.iter().all(|span| span.role == StyleRole::Math))
);
assert!(
display
.lines
.iter()
.any(|line| line.plain_text().contains('─'))
);
let all_text = doc
.blocks
.iter()
.flat_map(|block| &block.lines)
.map(|line| line.plain_text())
.collect::<Vec<_>>()
.join("\n");
assert!(all_text.contains("$35.00"), "{all_text}");
assert!(all_text.contains("x₂ + α"), "{all_text}");
assert!(all_text.contains(r"\(x\)"), "{all_text}");
}
#[test]
fn markdown_container_matrix_has_backend_neutral_parity() {
let containers = [
r"Inline \(\alpha_2 + x^2\).",
r"\[\frac{x+1}{y}\]",
"```math\n\\frac{x+1}{y}\n```",
"~~~latex\n\\begin{bmatrix}a & b \\\\ c & d\\end{bmatrix}\n~~~",
r"\begin{align*}x &= 1 \\ y &= 2\end{align*}",
];
for markdown in containers {
let first = parse_markdown(markdown);
let second = parse_markdown(markdown);
assert_eq!(first, second, "{markdown}");
assert!(
first
.blocks
.iter()
.flat_map(|block| &block.lines)
.flat_map(|line| &line.spans)
.any(|span| span.role == StyleRole::Math),
"{markdown}: {first:?}"
);
}
}
crates/jcode-render-core/tests/latex_streaming_regressions.rs
+186
View File
@@ -0,0 +1,186 @@
use jcode_render_core::{BlockKind, normalize_latex_math, parse_markdown};
fn math_display_count(markdown: &str) -> usize {
parse_markdown(markdown)
.blocks
.iter()
.filter(|block| block.kind == BlockKind::MathDisplay)
.count()
}
#[test]
fn parses_the_exact_multiline_equation_response_from_the_tui() {
let response = concat!(
"\\[\n\\boxed{\ne^{i\\pi}+1=0\n}\n\\]\n\n",
"\\[\n\\int_{-\\infty}^{\\infty} e^{-x^2}\\,dx=\\sqrt{\\pi}\n\\]\n\n",
"\\[\nx=\\frac{-b\\pm\\sqrt{b^2-4ac}}{2a}\n\\]\n\n",
"\\[\n\\nabla\\cdot\\mathbf{E}=\\frac{\\rho}{\\varepsilon_0}\n\\]\n\n",
"\\[\n\\frac{\\partial \\psi}{\\partial t}\n=\n",
"\\alpha\\frac{\\partial^2\\psi}{\\partial x^2}\n\\]",
);
let normalized = normalize_latex_math(response);
let parsed = parse_markdown(response);
assert_eq!(
math_display_count(response),
5,
"normalized={normalized:?} parsed={parsed:#?}"
);
assert_eq!(normalized.matches("$$").count(), 10);
}
#[test]
fn every_streaming_prefix_is_deterministic_and_the_complete_response_is_math() {
let equation = concat!(
"Result:\n\n\\[\n",
"\\frac{\\partial \\psi}{\\partial t}\n=\n",
"\\alpha\\frac{\\partial^2\\psi}{\\partial x^2}\n\\]",
);
for end in equation
.char_indices()
.map(|(index, _)| index)
.chain(std::iter::once(equation.len()))
{
let prefix = &equation[..end];
let first = parse_markdown(prefix);
let second = parse_markdown(prefix);
assert_eq!(
first, second,
"nondeterministic prefix ending at byte {end}"
);
}
assert_eq!(math_display_count(equation), 1);
}
#[test]
fn display_math_survives_common_markdown_containers() {
let cases = [
("> \\[\n> x^2\n> =\n> y^2\n> \\]", "blockquote"),
("- \\[\n x^2\n =\n y^2\n \\]", "bullet list"),
("1. \\[\n x^2\n =\n y^2\n \\]", "ordered list"),
];
for (source, label) in cases {
assert!(
math_display_count(source) >= 1,
"{label} lost display math after normalization: {:?}",
normalize_latex_math(source)
);
}
}
#[test]
fn standalone_display_math_resists_markdown_block_interruptors() {
let bodies = [
"x\n=\ny",
"x\n---\ny",
"x\n\ny",
"x\n# not a heading\ny",
"x\n> not a quote\ny",
"x\n- not a list\ny",
"x\n1. not a list\ny",
"x\n``` not a fence\ny",
];
for body in bodies {
for source in [format!("$$\n{body}\n$$"), format!("\\[\n{body}\n\\]")] {
assert_eq!(
math_display_count(&source),
1,
"block syntax escaped display math: {source:?}; normalized={:?}",
normalize_latex_math(&source)
);
}
}
}
#[test]
fn crlf_and_adjacent_display_blocks_remain_balanced_and_idempotent() {
let source = "\\[\r\nx\r\n=\r\ny\r\n\\]\r\n\r\n$$\r\na+b\r\n$$";
let normalized = normalize_latex_math(source);
assert_eq!(math_display_count(source), 2, "{normalized:?}");
assert_eq!(normalize_latex_math(&normalized), normalized);
assert_eq!(normalized.matches("$$").count(), 4);
}
#[test]
fn stabilization_preserves_newlines_comments_and_is_idempotent() {
let source = "\\[\na % this comment must still end at the newline\nb\n=\nc\n\\]";
let normalized = normalize_latex_math(source);
assert_eq!(
normalized,
"$$\na % this comment must still end at the newline\nb\n{}=\nc\n$$"
);
assert_eq!(normalize_latex_math(&normalized), normalized);
assert_eq!(math_display_count(source), 1);
}
#[test]
fn nested_quote_list_display_math_keeps_its_container() {
let source = "> - \\[\n> x\n> =\n> y\n> \\]";
let normalized = normalize_latex_math(source);
assert!(normalized.contains("> {}="), "{normalized:?}");
assert_eq!(math_display_count(source), 1, "{normalized:?}");
}
#[test]
fn escaped_and_literal_delimiters_never_become_math() {
let cases = [
r"\\[escaped\\]",
r"`\\[inline code\\]`",
"```text\n\\[fenced code\\]\n```",
" \\[indented code\\]",
r"\\[missing close",
];
for source in cases {
assert_eq!(
math_display_count(source),
0,
"literal case parsed as math: {source:?}"
);
}
let unclosed_display = "$$\nx\n=\ny";
assert_eq!(normalize_latex_math(unclosed_display), unclosed_display);
}
#[test]
fn standalone_multiline_inline_delimiters_promote_without_losing_content() {
for source in [
"\\(\nx\n=\ny\n\\)",
"$\nx\n=\ny\n$",
"> $\n> x\n> =\n> y\n> $",
"- $\n x\n =\n y\n $",
"$\r\nx\r\n=\r\ny\r\n$",
] {
let normalized = normalize_latex_math(source);
let visible = parse_markdown(source)
.blocks
.iter()
.flat_map(|block| &block.lines)
.map(|line| line.plain_text())
.collect::<Vec<_>>()
.join("\n");
for expected in ["x", "=", "y"] {
assert!(visible.contains(expected), "{source:?} => {visible:?}");
}
assert_eq!(
math_display_count(source),
1,
"{source:?} => {normalized:?}"
);
assert_eq!(normalize_latex_math(&normalized), normalized);
}
let unclosed = "$\nx\n=\ny";
assert_eq!(normalize_latex_math(unclosed), unclosed);
let fenced = "```text\n$\nx\n$\n```";
assert_eq!(normalize_latex_math(fenced), fenced);
assert_eq!(math_display_count(fenced), 0);
}