Implement markdown incremental code highlighting

examples/markdown/src/main.rs

@@ -19,7 +19,7 @@ struct Markdown {
 }
 
 enum Mode {
-    Oneshot(Vec<markdown::Item>),
+    Preview(Vec<markdown::Item>),
     Stream {
         pending: String,
         parsed: markdown::Content,
@@ -43,14 +43,14 @@ impl Markdown {
         (
             Self {
                 content: text_editor::Content::with_text(INITIAL_CONTENT),
-                mode: Mode::Oneshot(markdown::parse(INITIAL_CONTENT).collect()),
+                mode: Mode::Preview(markdown::parse(INITIAL_CONTENT).collect()),
                 theme,
             },
             widget::focus_next(),
         )
     }
 
-    fn update(&mut self, message: Message) {
+    fn update(&mut self, message: Message) -> Task<Message> {
         match message {
             Message::Edit(action) => {
                 let is_edit = action.is_edit();
@@ -58,48 +58,57 @@ impl Markdown {
                 self.content.perform(action);
 
                 if is_edit {
-                    self.mode = match self.mode {
+                    self.mode = Mode::Preview(
-                        Mode::Oneshot(_) => Mode::Oneshot(
+                        markdown::parse(&self.content.text()).collect(),
-                            markdown::parse(&self.content.text()).collect(),
+                    );
-                        ),
-                        Mode::Stream { .. } => Mode::Stream {
-                            pending: self.content.text(),
-                            parsed: markdown::Content::parse(""),
-                        },
-                    }
                 }
+
+                Task::none()
             }
             Message::LinkClicked(link) => {
                 let _ = open::that_in_background(link.to_string());
+
+                Task::none()
             }
             Message::ToggleStream(enable_stream) => {
-                self.mode = if enable_stream {
+                if enable_stream {
-                    Mode::Stream {
+                    self.mode = Mode::Stream {
                         pending: self.content.text(),
                         parsed: markdown::Content::parse(""),
-                    }
+                    };
-                } else {
+
-                    Mode::Oneshot(
+                    scrollable::snap_to(
-                        markdown::parse(&self.content.text()).collect(),
+                        "preview",
+                        scrollable::RelativeOffset::END,
                     )
-                };
+                } else {
+                    self.mode = Mode::Preview(
+                        markdown::parse(&self.content.text()).collect(),
+                    );
+
+                    Task::none()
+                }
             }
-            Message::NextToken => match &mut self.mode {
+            Message::NextToken => {
-                Mode::Oneshot(_) => {}
+                match &mut self.mode {
-                Mode::Stream { pending, parsed } => {
+                    Mode::Preview(_) => {}
-                    if pending.is_empty() {
+                    Mode::Stream { pending, parsed } => {
-                        self.mode = Mode::Oneshot(parsed.items().to_vec());
+                        if pending.is_empty() {
-                    } else {
+                            self.mode = Mode::Preview(parsed.items().to_vec());
-                        let mut tokens = pending.split(' ');
+                        } else {
+                            let mut tokens = pending.split(' ');
 
-                        if let Some(token) = tokens.next() {
+                            if let Some(token) = tokens.next() {
-                            parsed.push_str(&format!("{token} "));
+                                parsed.push_str(&format!("{token} "));
+                            }
+
+                            *pending = tokens.collect::<Vec<_>>().join(" ");
                         }
-
-                        *pending = tokens.collect::<Vec<_>>().join(" ");
                     }
                 }
-            },
+
+                Task::none()
+            }
         }
     }
 
@@ -113,7 +122,7 @@ impl Markdown {
             .highlight("markdown", highlighter::Theme::Base16Ocean);
 
         let items = match &self.mode {
-            Mode::Oneshot(items) => items.as_slice(),
+            Mode::Preview(items) => items.as_slice(),
             Mode::Stream { parsed, .. } => parsed.items(),
         };
 
@@ -127,7 +136,11 @@ impl Markdown {
         row![
             editor,
             hover(
-                scrollable(preview).spacing(10).width(Fill).height(Fill),
+                scrollable(preview)
+                    .spacing(10)
+                    .width(Fill)
+                    .height(Fill)
+                    .id("preview"),
                 right(
                     toggler(matches!(self.mode, Mode::Stream { .. }))
                         .label("Stream")
@@ -147,7 +160,7 @@ impl Markdown {
 
     fn subscription(&self) -> Subscription<Message> {
         match self.mode {
-            Mode::Oneshot(_) => Subscription::none(),
+            Mode::Preview(_) => Subscription::none(),
             Mode::Stream { .. } => {
                 time::every(milliseconds(20)).map(|_| Message::NextToken)
             }

highlighter/src/lib.rs

@@ -7,6 +7,7 @@ use crate::core::Color;
 
 use std::ops::Range;
 use std::sync::LazyLock;
+
 use syntect::highlighting;
 use syntect::parsing;
 
@@ -104,30 +105,7 @@ impl highlighter::Highlighter for Highlighter {
 
         let ops = parser.parse_line(line, &SYNTAXES).unwrap_or_default();
 
-        let highlighter = &self.highlighter;
+        Box::new(scope_iterator(ops, line, stack, &self.highlighter))
-
-        Box::new(
-            ScopeRangeIterator {
-                ops,
-                line_length: line.len(),
-                index: 0,
-                last_str_index: 0,
-            }
-            .filter_map(move |(range, scope)| {
-                let _ = stack.apply(&scope);
-
-                if range.is_empty() {
-                    None
-                } else {
-                    Some((
-                        range,
-                        Highlight(
-                            highlighter.style_mod_for_stack(&stack.scopes),
-                        ),
-                    ))
-                }
-            }),
-        )
     }
 
     fn current_line(&self) -> usize {
@@ -135,6 +113,92 @@ impl highlighter::Highlighter for Highlighter {
     }
 }
 
+fn scope_iterator<'a>(
+    ops: Vec<(usize, parsing::ScopeStackOp)>,
+    line: &str,
+    stack: &'a mut parsing::ScopeStack,
+    highlighter: &'a highlighting::Highlighter<'static>,
+) -> impl Iterator<Item = (Range<usize>, Highlight)> + 'a {
+    ScopeRangeIterator {
+        ops,
+        line_length: line.len(),
+        index: 0,
+        last_str_index: 0,
+    }
+    .filter_map(move |(range, scope)| {
+        let _ = stack.apply(&scope);
+
+        if range.is_empty() {
+            None
+        } else {
+            Some((
+                range,
+                Highlight(highlighter.style_mod_for_stack(&stack.scopes)),
+            ))
+        }
+    })
+}
+
+/// A streaming syntax highlighter.
+///
+/// It can efficiently highlight an immutable stream of tokens.
+#[derive(Debug)]
+pub struct Stream {
+    syntax: &'static parsing::SyntaxReference,
+    highlighter: highlighting::Highlighter<'static>,
+    commit: (parsing::ParseState, parsing::ScopeStack),
+    state: parsing::ParseState,
+    stack: parsing::ScopeStack,
+}
+
+impl Stream {
+    /// Creates a new [`Stream`] highlighter.
+    pub fn new(settings: &Settings) -> Self {
+        let syntax = SYNTAXES
+            .find_syntax_by_token(&settings.token)
+            .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text());
+
+        let highlighter = highlighting::Highlighter::new(
+            &THEMES.themes[settings.theme.key()],
+        );
+
+        let state = parsing::ParseState::new(syntax);
+        let stack = parsing::ScopeStack::new();
+
+        Self {
+            syntax,
+            highlighter,
+            commit: (state.clone(), stack.clone()),
+            state,
+            stack,
+        }
+    }
+
+    /// Highlights the given line from the last commit.
+    pub fn highlight_line(
+        &mut self,
+        line: &str,
+    ) -> impl Iterator<Item = (Range<usize>, Highlight)> + '_ {
+        self.state = self.commit.0.clone();
+        self.stack = self.commit.1.clone();
+
+        let ops = self.state.parse_line(line, &SYNTAXES).unwrap_or_default();
+        scope_iterator(ops, line, &mut self.stack, &self.highlighter)
+    }
+
+    /// Commits the last highlighted line.
+    pub fn commit(&mut self) {
+        self.commit = (self.state.clone(), self.stack.clone());
+    }
+
+    /// Resets the [`Stream`] highlighter.
+    pub fn reset(&mut self) {
+        self.state = parsing::ParseState::new(self.syntax);
+        self.stack = parsing::ScopeStack::new();
+        self.commit = (self.state.clone(), self.stack.clone());
+    }
+}
+
 /// The settings of a [`Highlighter`].
 #[derive(Debug, Clone, PartialEq)]
 pub struct Settings {

widget/src/markdown.rs

@@ -57,6 +57,7 @@ use crate::core::{
 };
 use crate::{column, container, rich_text, row, scrollable, span, text};
 
+use std::borrow::BorrowMut;
 use std::cell::{Cell, RefCell};
 use std::ops::Range;
 use std::sync::Arc;
@@ -65,7 +66,7 @@ pub use core::text::Highlight;
 pub use pulldown_cmark::HeadingLevel;
 pub use url::Url;
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct Content {
     items: Vec<Item>,
     state: State,
@@ -80,6 +81,10 @@ impl Content {
     }
 
     pub fn push_str(&mut self, markdown: &str) {
+        if markdown.is_empty() {
+            return;
+        }
+
         // Append to last leftover text
         let mut leftover = std::mem::take(&mut self.state.leftover);
         leftover.push_str(markdown);
@@ -90,8 +95,6 @@ impl Content {
         // Re-parse last item and new text
         let new_items = parse_with(&mut self.state, &leftover);
         self.items.extend(new_items);
-
-        dbg!(&self.state);
     }
 
     pub fn items(&self) -> &[Item] {
@@ -271,19 +274,91 @@ pub fn parse(markdown: &str) -> impl Iterator<Item = Item> + '_ {
     parse_with(State::default(), markdown)
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Default)]
-pub struct State {
+struct State {
     leftover: String,
+    #[cfg(feature = "highlighter")]
+    highlighter: Option<Highlighter>,
 }
 
-impl AsMut<Self> for State {
+#[cfg(feature = "highlighter")]
-    fn as_mut(&mut self) -> &mut Self {
+#[derive(Debug)]
-        self
+struct Highlighter {
+    lines: Vec<(String, Vec<Span>)>,
+    parser: iced_highlighter::Stream,
+    current: usize,
+}
+
+#[cfg(feature = "highlighter")]
+impl Highlighter {
+    pub fn new(language: &str) -> Self {
+        Self {
+            lines: Vec::new(),
+            parser: iced_highlighter::Stream::new(
+                &iced_highlighter::Settings {
+                    theme: iced_highlighter::Theme::Base16Ocean,
+                    token: language.to_string(),
+                },
+            ),
+            current: 0,
+        }
+    }
+
+    pub fn prepare(&mut self) {
+        self.current = 0;
+    }
+
+    pub fn highlight_line(&mut self, text: &str) -> &[Span] {
+        match self.lines.get(self.current) {
+            Some(line) if line.0 == text => {}
+            _ => {
+                if self.current + 1 < self.lines.len() {
+                    println!("Resetting...");
+                    self.parser.reset();
+                    self.lines.truncate(self.current);
+
+                    for line in &self.lines {
+                        println!("Refeeding {n} lines", n = self.lines.len());
+
+                        let _ = self.parser.highlight_line(&line.0);
+                    }
+                }
+
+                println!("Parsing: {text}", text = text.trim_end());
+                if self.current + 1 < self.lines.len() {
+                    self.parser.commit();
+                }
+
+                let mut spans = Vec::new();
+
+                for (range, highlight) in self.parser.highlight_line(text) {
+                    spans.push(Span::Highlight {
+                        text: text[range].to_owned(),
+                        color: highlight.color(),
+                        font: highlight.font(),
+                    });
+                }
+
+                if self.current + 1 == self.lines.len() {
+                    let _ = self.lines.pop();
+                }
+
+                self.lines.push((text.to_owned(), spans));
+            }
+        }
+
+        self.current += 1;
+
+        &self
+            .lines
+            .get(self.current - 1)
+            .expect("Line must be parsed")
+            .1
     }
 }
 
 fn parse_with<'a>(
-    mut state: impl AsMut<State> + 'a,
+    mut state: impl BorrowMut<State> + 'a,
     markdown: &'a str,
 ) -> impl Iterator<Item = Item> + 'a {
     struct List {
@@ -312,24 +387,26 @@ fn parse_with<'a>(
     )
     .into_offset_iter();
 
-    let mut produce =
+    let produce = move |state: &mut State,
-        move |lists: &mut Vec<List>, item, source: Range<usize>| {
+                        lists: &mut Vec<List>,
-            if lists.is_empty() {
+                        item,
-                state.as_mut().leftover = markdown[source.start..].to_owned();
+                        source: Range<usize>| {
+        if lists.is_empty() {
+            state.leftover = markdown[source.start..].to_owned();
 
-                Some(item)
+            Some(item)
-            } else {
+        } else {
-                lists
+            lists
-                    .last_mut()
+                .last_mut()
-                    .expect("list context")
+                .expect("list context")
-                    .items
+                .items
-                    .last_mut()
+                .last_mut()
-                    .expect("item context")
+                .expect("item context")
-                    .push(item);
+                .push(item);
 
-                None
+            None
-            }
+        }
-        };
+    };
 
     // We want to keep the `spans` capacity
     #[allow(clippy::drain_collect)]
@@ -367,6 +444,7 @@ fn parse_with<'a>(
                     None
                 } else {
                     produce(
+                        state.borrow_mut(),
                         &mut lists,
                         Item::Paragraph(Text::new(spans.drain(..).collect())),
                         source,
@@ -393,20 +471,24 @@ fn parse_with<'a>(
             ) if !metadata && !table => {
                 #[cfg(feature = "highlighter")]
                 {
-                    use iced_highlighter::Highlighter;
+                    highlighter = Some({
-                    use text::Highlighter as _;
+                        let mut highlighter = state
+                            .borrow_mut()
+                            .highlighter
+                            .take()
+                            .unwrap_or_else(|| Highlighter::new(&_language));
 
-                    highlighter =
+                        highlighter.prepare();
-                        Some(Highlighter::new(&iced_highlighter::Settings {
+
-                            theme: iced_highlighter::Theme::Base16Ocean,
+                        highlighter
-                            token: _language.to_string(),
+                    });
-                        }));
                 }
 
                 let prev = if spans.is_empty() {
                     None
                 } else {
                     produce(
+                        state.borrow_mut(),
                         &mut lists,
                         Item::Paragraph(Text::new(spans.drain(..).collect())),
                         source,
@@ -428,6 +510,7 @@ fn parse_with<'a>(
         pulldown_cmark::Event::End(tag) => match tag {
             pulldown_cmark::TagEnd::Heading(level) if !metadata && !table => {
                 produce(
+                    state.borrow_mut(),
                     &mut lists,
                     Item::Heading(level, Text::new(spans.drain(..).collect())),
                     source,
@@ -451,6 +534,7 @@ fn parse_with<'a>(
             }
             pulldown_cmark::TagEnd::Paragraph if !metadata && !table => {
                 produce(
+                    state.borrow_mut(),
                     &mut lists,
                     Item::Paragraph(Text::new(spans.drain(..).collect())),
                     source,
@@ -461,6 +545,7 @@ fn parse_with<'a>(
                     None
                 } else {
                     produce(
+                        state.borrow_mut(),
                         &mut lists,
                         Item::Paragraph(Text::new(spans.drain(..).collect())),
                         source,
@@ -471,6 +556,7 @@ fn parse_with<'a>(
                 let list = lists.pop().expect("list context");
 
                 produce(
+                    state.borrow_mut(),
                     &mut lists,
                     Item::List {
                         start: list.start,
@@ -482,10 +568,11 @@ fn parse_with<'a>(
             pulldown_cmark::TagEnd::CodeBlock if !metadata && !table => {
                 #[cfg(feature = "highlighter")]
                 {
-                    highlighter = None;
+                    state.borrow_mut().highlighter = highlighter.take();
                 }
 
                 produce(
+                    state.borrow_mut(),
                     &mut lists,
                     Item::CodeBlock(Text::new(spans.drain(..).collect())),
                     source,
@@ -504,20 +591,16 @@ fn parse_with<'a>(
         pulldown_cmark::Event::Text(text) if !metadata && !table => {
             #[cfg(feature = "highlighter")]
             if let Some(highlighter) = &mut highlighter {
-                use text::Highlighter as _;
+                let start = std::time::Instant::now();
 
-                for (range, highlight) in
+                for line in text.lines() {
-                    highlighter.highlight_line(text.as_ref())
+                    spans.extend_from_slice(
-                {
+                        highlighter.highlight_line(&format!("{line}\n")),
-                    let span = Span::Highlight {
+                    );
-                        text: text[range].to_owned(),
-                        color: highlight.color(),
-                        font: highlight.font(),
-                    };
-
-                    spans.push(span);
                 }
 
+                dbg!(start.elapsed());
+
                 return None;
             }