diff --git a/README.de.md b/README.de.md index 105f375..2c3419f 100644 --- a/README.de.md +++ b/README.de.md @@ -37,6 +37,11 @@ Die Oberfläche gibt es auf Deutsch und Englisch; ink-notes folgt der Systemspra - Erkannt werden Linien, Pfeile, Kreise, Ellipsen, Rechtecke, Dreiecke, Rauten, Vielecke, Koordinatensysteme (L-Form), Achsenkreuze (ein „+“ in einem Zug oder zwei gekreuzte Geraden, Pfeilspitzen an allen vier Enden) und Oktaeder (Skizze als 3D-Körper). - Die Erkennung arbeitet rein geometrisch (Bogenlänge, Douglas-Peucker, Kreis-Fit), ohne Machine Learning. Nicht alles wird erkannt – was nicht passt, bleibt Handschrift. +### Formeln ausrechnen (experimentell) +- Einschalten im ☰-Menü unter Eingabe („Formeln nach „=“ ausrechnen“). Nach einem handgeschriebenen `=` und einer kurzen Pause erscheint daneben das Ergebnis samt dem Gelesenen, z. B. `2⁴ = 16` oder `(x + 3)² = x² + 6x + 9`; Antippen mit dem Stift oder Klicken fügt es als Textfeld in der Stiftfarbe und in der Höhe der Handschrift ein (Rückgängig entfernt es). Finger und Handballen lösen nichts aus. +- Gelesen werden Ziffern, `+ − · : /`, Klammern, Hochzahlen (hochgestellt geschrieben), Bruchstriche, Dezimalkommas und die Variablen `x`, `y`, `a`, `b`. Zahlen werden exakt als Brüche gerechnet, Ausdrücke mit Variablen ausmultipliziert. Vor der Formel etwas Abstand lassen, damit weiter links Geschriebenes nicht mitgelesen wird; in einer Kette wie `x = 2 · 3 =` wird der Teil nach dem vorigen `=` gerechnet. +- Die Zeichen werden mit Punktwolken-Vorlagen ($P) erkannt, ohne Machine Learning und offline. Zeigt der Vorschlag ein falsches Zeichen, ihn einfach ignorieren. + ### Auswählen und Bearbeiten - **Auswahlwerkzeug** (`V`): antippen oder Rahmen aufziehen, dann verschieben, an den Ecken skalieren, am Griff drehen oder mit `R` um 90° drehen. Funktioniert mit Maus, Finger und Stift. - Auswahl über die Palette **umfärben**, **kopieren/einfügen** (`Strg + C/V`), **löschen** (`Entf`). @@ -193,6 +198,7 @@ src/ ├── library.rs # Ordner der Notizbücher (library.json) ├── math.rs # Exaktes Ausrechnen handgeschriebener Formeln (Issue #46) ├── math_ink.rs # Erkennung handgeschriebener Formeln ($P-Punktwolken) +├── math_assist.rs # Ergebnis neben einem handgeschriebenen „=“ (Auslöser, Vorschlag) ├── autosave.rs # Speichern im Hintergrund ├── export.rs # PDF-Export ├── page_export.rs # Seite als PNG/SVG, Auswahl als Bild diff --git a/README.md b/README.md index f5e53f4..6afe5a9 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,11 @@ The interface is available in English and German; ink-notes follows the system l - Recognized shapes: lines, arrows, circles, ellipses, rectangles, triangles, rhombuses, polygons, coordinate systems (L shape), axis crosses (a "+" in one stroke or two crossing straight lines, arrowheads on all four ends) and octahedra (sketch as a 3D solid). - Recognition is purely geometric (arc length, Douglas-Peucker, circle fitting), without machine learning. Not everything is recognized – whatever does not fit stays handwriting. +### Computing formulas (experimental) +- Turn on in the ☰ menu under input ("Compute formulas after “=”"). After a handwritten `=` and a short pause, the result appears next to it together with what was read, e.g. `2⁴ = 16` or `(x + 3)² = x² + 6x + 9`; tapping it with the pen or clicking inserts it as a text box in the ink color, as high as the handwriting (undo removes it). Fingers and palms do not trigger it. +- Read are digits, `+ − · : /`, parentheses, exponents (written raised), fraction bars, decimal commas and the variables `x`, `y`, `a`, `b`. Numbers are computed exactly as fractions, expressions with variables are multiplied out. Leave some space before the formula, so that writing further to the left is not read with it; in a chain such as `x = 2 · 3 =` the part after the previous `=` is computed. +- The symbols are recognized with point-cloud templates ($P), without machine learning and offline. If the suggestion shows a wrong symbol, just ignore it. + ### Selecting and editing - **Selection tool** (`V`): tap or drag a frame, then move, scale by the corners, rotate by the handle, or rotate by 90° with `R`. Works with mouse, finger and pen. - **Recolor** the selection with the palette, **copy/paste** (`Ctrl + C/V`), **delete** (`Del`). @@ -195,6 +200,7 @@ src/ ├── library.rs # Notebook folders (library.json) ├── math.rs # Exact evaluation of handwritten math (Issue #46) ├── math_ink.rs # Recognition of handwritten math ($P point clouds) +├── math_assist.rs # Result next to a handwritten "=" (trigger, suggestion) ├── autosave.rs # Saving in the background ├── export.rs # PDF export ├── page_export.rs # Page as PNG/SVG, selection as image diff --git a/src/app.rs b/src/app.rs index a66537f..5528a06 100644 --- a/src/app.rs +++ b/src/app.rs @@ -335,7 +335,7 @@ fn commit_stroke_to_notebook( notebook: &mut Notebook, manifest: &mut NotebookManifest, autosave: &mut AutosaveController, - ui_state: &UiState, + ui_state: &mut UiState, window: &Window, snapped_shape: Option, ) { @@ -357,6 +357,7 @@ fn commit_stroke_to_notebook( let page = notebook.current_page_mut(); page.replace_stroke(previous_id, stroke); canvas.sync_page(page); + ui_state.math.stroke_committed(page, Instant::now()); autosave.mark_dirty(notebook, manifest); window.request_redraw(); return; @@ -383,6 +384,8 @@ fn commit_stroke_to_notebook( let page = notebook.current_page_mut(); page.add_stroke(stroke); canvas.sync_page(page); + // An `=` may have been completed (Issue #46) + ui_state.math.stroke_committed(page, Instant::now()); manifest.next_stroke_id = notebook.next_stroke_id; autosave.mark_dirty(notebook, manifest); // Durchgehende Ansicht: unten auf der letzten Seite geht es auf einer neuen weiter @@ -915,6 +918,54 @@ fn insert_text_on_page( ui_state.set_export_message(t!("Text eingefügt – mit dem Text-Werkzeug (T) änderbar")); } +/// Inserts the result of a math suggestion as a text box next to its `=`, in the ink color +/// and as high as the handwriting, as one undo step (Issue #46) +fn insert_math_result( + suggestion: &crate::math_assist::Suggestion, + canvas: &mut CanvasState, + notebook: &mut Notebook, + manifest: &mut NotebookManifest, + autosave: &mut AutosaveController, +) { + let page = notebook.current_page(); + if page.id != suggestion.page_id || page.is_readonly() { + return; + } + let font = crate::text::TextFont::Sans; + let ascent_per_em = crate::text::TextBox::new(String::new(), 1.0, font, 1.0) + .layout() + .ascent; + let (origin, size) = suggestion.text_placement(ascent_per_em); + let mut edit = crate::text::TextEditing::new_at( + page.id, + origin, + page.text_right_edge(origin.x), + size, + font, + suggestion.color, + ); + edit.origin = origin; + edit.text.content = suggestion.result.clone(); + // As wide as the result, so that it does not wrap near the edge of the page + edit.text.wrap_width = edit + .text + .layout() + .lines + .iter() + .map(|line| line.width) + .fold(0.0, f32::max) + + size; + let id = notebook.next_stroke_id(); + let Some(stroke) = edit.to_stroke(id) else { + return; + }; + let page = notebook.current_page_mut(); + page.add_stroke(stroke); + canvas.sync_page(page); + manifest.next_stroke_id = notebook.next_stroke_id; + autosave.mark_dirty(notebook, manifest); +} + /// Beendet eine laufende Textbearbeitung: übernimmt das Feld (neu, geändert oder – wenn /// geleert – entfernt) als einen Undo-Schritt auf seiner Seite. fn finish_text_edit( @@ -1419,6 +1470,7 @@ pub fn run_with( ui_state.pen_presets = ui_settings.pen_presets; ui_state.stylus_buttons = ui_settings.stylus_buttons.clone(); ui_state.set_eraser_radius_mm(ui_settings.eraser_radius_mm); + ui_state.math.set_enabled(ui_settings.math_assist); ui_state.select_pen(ui_settings.active_pen); ui_state.highlighter_colors = ui_settings.highlighter_colors; ui_state.select_highlighter_color(ui_settings.active_highlighter); @@ -1864,7 +1916,7 @@ pub fn run_with( &mut notebook, &mut manifest, &mut autosave, - &ui_state, + &mut ui_state, &window, stroke_snapped_shape.take(), ); @@ -2048,6 +2100,26 @@ pub fn run_with( stylus_tool_id = Some(tool); let (sx, sy) = stylus_screen_pos; + // Tapping a math suggestion inserts its result (Issue #46) + if let Some(suggestion) = (!ui_state.view_mode) + .then(|| { + ui_state.math.take_hit(egui::pos2(sx / scale, sy / scale)) + }) + .flatten() + { + insert_math_result( + &suggestion, + &mut canvas, + &mut notebook, + &mut manifest, + &mut autosave, + ); + // The rest of this tap goes nowhere + stylus_ui_capture = true; + window.request_redraw(); + continue; + } + if ui::is_screen_point_in_ui(&egui_ctx, &ui_state, (sx, sy), scale) { // In die Eingabe-Warteschlange des NÄCHSTEN egui-Frames @@ -2275,7 +2347,7 @@ pub fn run_with( &mut notebook, &mut manifest, &mut autosave, - &ui_state, + &mut ui_state, &window, snapped_shape, ); @@ -2351,6 +2423,16 @@ pub fn run_with( window.request_redraw(); } + // Pause after an `=` is over: compute the expression (Issue #46) + if ui_state.math.update( + notebook.current_page(), + Instant::now(), + is_drawing || is_stylus_drawing, + crate::i18n::language() == crate::i18n::Language::German, + ) { + window.request_redraw(); + } + // Nachgleiten nach dem Wischen if let Some(delta) = fling.step(Instant::now()) { canvas.pan(delta.x, delta.y); @@ -2377,6 +2459,7 @@ pub fn run_with( } else if let Some(at) = [ egui_repaint_at, long_press.deadline(), + ui_state.math.deadline(), Some(disk_watch.deadline()), // The updater thread does not wake the loop: look for its results now and then matches!( @@ -3176,6 +3259,25 @@ pub fn run_with( return; } + // Clicking a math suggestion inserts its result (Issue #46) + if button == MouseButton::Left && !ui_state.view_mode { + let (sx, sy) = cursor_screen_pos; + let scale = window.scale_factor() as f32; + if let Some(suggestion) = + ui_state.math.take_hit(egui::pos2(sx / scale, sy / scale)) + { + insert_math_result( + &suggestion, + &mut canvas, + &mut notebook, + &mut manifest, + &mut autosave, + ); + window.request_redraw(); + return; + } + } + fling.stop(); // 1. Pan via mittlere Maustaste oder Leertaste + Linksklick // In view-only mode the left and right buttons move the page too (Issue #35) @@ -3380,7 +3482,7 @@ pub fn run_with( &mut notebook, &mut manifest, &mut autosave, - &ui_state, + &mut ui_state, &window, snapped_shape, ); @@ -3958,7 +4060,7 @@ pub fn run_with( &mut notebook, &mut manifest, &mut autosave, - &ui_state, + &mut ui_state, &window, snapped_shape, ); @@ -5187,6 +5289,14 @@ pub fn run_with( } } } + UiAction::SetMathAssist(on) => { + ui_settings.math_assist = on; + save_ui_settings( + &ui_settings, + ui_settings_path.as_deref(), + &mut ui_state, + ); + } UiAction::SetSidebarSide(side) => { ui_settings.sidebar_side = side; save_ui_settings( diff --git a/src/i18n/en.rs b/src/i18n/en.rs index 7d205c2..f22ce9e 100644 --- a/src/i18n/en.rs +++ b/src/i18n/en.rs @@ -107,6 +107,7 @@ pub const EN: &[(&str, &str)] = &[ ("Fehler beim Öffnen: {err}", "Error while opening: {err}"), ("Fehler: {err}", "Error: {err}"), ("Fehlermeldung: {message}", "Error message: {message}"), + ("Formeln nach „=“ ausrechnen", "Compute formulas after “=”"), ("Formen (Shape)", "Shapes"), ("Formen: Striche werden zu Linien, Rechtecken, Kreisen … (S/F)", "Shapes: strokes become lines, rectangles, circles … (S/F)"), ("Fünfeck (Pentagon)", "Pentagon"), @@ -186,6 +187,7 @@ pub const EN: &[(&str, &str)] = &[ ("Mindestdruck", "Minimum pressure"), ("Mit dem Finger zeichnen", "Draw with finger"), ("Mittel", "Medium"), + ("Nach einem handgeschriebenen „=“ erscheint das Ergebnis daneben, z. B. 2⁴ = 16. Antippen mit dem Stift oder Klicken fügt es als Text ein.\nGelesen werden Ziffern, + − · : /, Klammern, Hochzahlen, Brüche und x, y, a, b.", "After a handwritten “=” the result appears next to it, e.g. 2⁴ = 16. Tapping it with the pen or clicking inserts it as text.\nDigits, + − · : /, parentheses, exponents, fractions and x, y, a, b are read."), ("Name …", "Name …"), ("Neon-Gelb", "Neon yellow"), ("Netzwerk-Transportfehler: {t}", "Network transport error: {t}"), diff --git a/src/lib.rs b/src/lib.rs index 35994c4..86cca22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod image_layer; pub mod import; pub mod library; pub mod math; +pub mod math_assist; pub mod math_ink; pub mod osk; pub mod page_export; diff --git a/src/math.rs b/src/math.rs index 62dfa5e..0b27cfd 100644 --- a/src/math.rs +++ b/src/math.rs @@ -51,6 +51,14 @@ pub fn solve(expression: &str, decimal_comma: bool) -> Option { Some(text) } +/// `expression` written neatly for showing it, e.g. `2⁴` for `2^(4)`, `(x + 1)/2` for +/// `((x+1)/(2))` or `2,5 · 3` for `2,5*3`. `None` if it cannot be read. +pub fn display(expression: &str, decimal_comma: bool) -> Option { + let tokens = tokenize(expression)?; + let ast = Parser::new(&tokens).parse()?; + Some(ast.display(decimal_comma)) +} + // --- Rational numbers --------------------------------------------------------------------- /// Exact fraction `num / den` with `den > 0`, always reduced @@ -351,7 +359,8 @@ enum Token { Plus, Minus, Times, - Divide, + /// `/` for a fraction, `:` for a division sign (`:` and `÷`) + Divide(char), Power, Open, Close, @@ -396,7 +405,8 @@ fn tokenize(input: &str) -> Option> { '+' => Some(Token::Plus), '-' | '−' | '–' => Some(Token::Minus), '*' | '·' | '⋅' | '∙' | '×' => Some(Token::Times), - '/' | ':' | '÷' => Some(Token::Divide), + '/' => Some(Token::Divide('/')), + ':' | '÷' => Some(Token::Divide(':')), '^' => Some(Token::Power), '(' | '[' | '{' => Some(Token::Open), ')' | ']' | '}' => Some(Token::Close), @@ -450,7 +460,8 @@ enum Expr { Add(Box, Box), Sub(Box, Box), Mul(Box, Box), - Div(Box, Box), + /// With the sign it was written with, for [`display`] + Div(Box, Box, char), Pow(Box, Box), } @@ -463,12 +474,79 @@ impl Expr { Expr::Add(a, b) => a.eval()?.add(&b.eval()?)?, Expr::Sub(a, b) => a.eval()?.add(&b.eval()?.neg()?)?, Expr::Mul(a, b) => a.eval()?.mul(&b.eval()?)?, - Expr::Div(a, b) => a.eval()?.div(&b.eval()?)?, + Expr::Div(a, b, _) => a.eval()?.div(&b.eval()?)?, Expr::Pow(a, b) => a.eval()?.pow(&b.eval()?)?, }) } } +impl Expr { + /// Binding strength for deciding on parentheses: sums < products < signs < powers < atoms + fn precedence(&self) -> u8 { + match self { + Expr::Add(..) | Expr::Sub(..) => 1, + Expr::Mul(..) | Expr::Div(..) => 2, + Expr::Neg(_) => 3, + Expr::Pow(..) => 4, + Expr::Number(_) | Expr::Variable(_) => 5, + } + } + + /// The text of this expression, in parentheses if it binds weaker than `min` + fn display_at(&self, min: u8, decimal_comma: bool) -> String { + let text = self.display(decimal_comma); + if self.precedence() < min { + format!("({text})") + } else { + text + } + } + + fn display(&self, decimal_comma: bool) -> String { + let dc = decimal_comma; + match self { + Expr::Number(value) => value.format_abs(dc), + Expr::Variable(name) => name.to_string(), + Expr::Neg(e) => format!("−{}", e.display_at(3, dc)), + Expr::Add(a, b) => format!("{} + {}", a.display_at(1, dc), b.display_at(1, dc)), + Expr::Sub(a, b) => format!("{} − {}", a.display_at(1, dc), b.display_at(2, dc)), + Expr::Mul(a, b) => { + let left = a.display_at(2, dc); + let right = b.display_at(3, dc); + // 2x, 3(x + 1), x², but 2 · 3 and x · 2 + let implicit = !matches!(**a, Expr::Neg(_)) + && (matches!(**b, Expr::Variable(_)) + || matches!(&**b, Expr::Pow(base, _) if matches!(**base, Expr::Variable(_))) + || right.starts_with('(')); + if implicit { + format!("{left}{right}") + } else { + format!("{left} · {right}") + } + } + Expr::Div(a, b, sign) => match sign { + '/' => format!("{}/{}", a.display_at(4, dc), b.display_at(4, dc)), + _ => format!("{} : {}", a.display_at(2, dc), b.display_at(3, dc)), + }, + Expr::Pow(base, exponent) => { + let base = base.display_at(5, dc); + let integer = |e: &Expr| match e { + Expr::Number(v) => v.as_integer().map(|n| superscript(n as u32)), + _ => None, + }; + let small = match &**exponent { + Expr::Neg(e) => integer(e).map(|s| format!("⁻{s}")), + e => integer(e), + }; + match small { + Some(small) => format!("{base}{small}"), + None => format!("{base}^({})", exponent.display(dc)), + } + } + } + } +} + /// Recursive descent with the usual precedence: `+ −` < `· :` and implicit /// multiplication < unary minus < `^` (right-associative, so `−2² = −4` and `2^3^2 = 2^9`). struct Parser<'a> { @@ -516,8 +594,9 @@ impl<'a> Parser<'a> { loop { if self.eat(&Token::Times) { left = Expr::Mul(Box::new(left), Box::new(self.unary()?)); - } else if self.eat(&Token::Divide) { - left = Expr::Div(Box::new(left), Box::new(self.unary()?)); + } else if let Some(&Token::Divide(sign)) = self.peek() { + self.pos += 1; + left = Expr::Div(Box::new(left), Box::new(self.unary()?), sign); } else if matches!(self.peek(), Some(Token::Variable(_) | Token::Open)) { // Implicit multiplication: 2x, 3(x + 1), (a + b)(a − b). Two numbers in a // row are not multiplied, they are more likely a misread. @@ -644,6 +723,28 @@ mod tests { assert_eq!(de("1,2,3"), None); } + #[test] + fn displays_expressions() { + let show = |e: &str| display(e, true).unwrap(); + assert_eq!(show("2^(4)"), "2⁴"); + assert_eq!(show("(x+3)^(2)"), "(x + 3)²"); + assert_eq!(show("2,5·3"), "2,5 · 3"); + assert_eq!(show("9:3"), "9 : 3"); + assert_eq!(show("((x+1)/(2))"), "(x + 1)/2"); + assert_eq!(show("((1)/(2))+((1)/(2·3))"), "1/2 + 1/(2 · 3)"); + assert_eq!(show("2^(-1)"), "2⁻¹"); + assert_eq!(show("2^(x+1)"), "2^(x + 1)"); + assert_eq!(show("3(x+1)"), "3(x + 1)"); + assert_eq!(show("2x^(2)"), "2x²"); + assert_eq!(show("x·2"), "x · 2"); + assert_eq!(show("(a+b)(a-b)"), "(a + b)(a − b)"); + assert_eq!(show("7-(5-2)"), "7 − (5 − 2)"); + assert_eq!(show("-2^(2)"), "−2²"); + assert_eq!(show("(-2)^(2)"), "(−2)²"); + assert_eq!(show("0.5"), "0,5"); + assert!(display("1 +", true).is_none()); + } + #[test] fn overflow_and_limits() { assert_eq!(de("10^100"), None); diff --git a/src/math_assist.rs b/src/math_assist.rs new file mode 100644 index 0000000..f7fafb1 --- /dev/null +++ b/src/math_assist.rs @@ -0,0 +1,465 @@ +//! Computes handwritten math after an `=` (Issue #46, stage 3). +//! +//! When the last two strokes on a page form an `=` and the pen then rests for +//! [`TRIGGER_DELAY`], the strokes left of it are recognized ([`crate::math_ink`]) and computed +//! ([`crate::math::solve`]). The result is offered next to the `=` as a suggestion that also +//! shows what was read (`2⁴ = 16`), so that a misread symbol is noticed. Tapping it with the +//! pen or clicking it inserts the result as a text box; fingers and palms are ignored. Any new +//! stroke drops the suggestion. Off by default (☰ → Input). + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use glam::Vec2; + +use crate::document::{Page, Stroke, StrokeTool}; +use crate::math_ink::Recognizer; + +/// Pause after the `=` before its expression is computed; a new stroke cancels it +pub const TRIGGER_DELAY: Duration = Duration::from_millis(500); +/// Estimated symbol height relative to the width of the `=` +const SYMBOL_PER_EQUALS: f32 = 1.3; +/// Strokes left of the `=` belong to the expression up to a gap of this many symbol heights +const MAX_GAP: f32 = 2.0; +/// At most this many strokes are read as the expression +const MAX_STROKES: usize = 80; +/// Cap height of the text font relative to its size (Ubuntu: 0.69 em) +const CAP_HEIGHT_PER_EM: f32 = 0.7; + +/// A result offered next to an `=` +#[derive(Debug, Clone, PartialEq)] +pub struct Suggestion { + pub page_id: u64, + /// The two strokes of the `=` + pub equals: [u64; 2], + /// What was read, written neatly (`2⁴`) + pub expression: String, + /// The result (`16`) + pub result: String, + /// Where the result goes: left end of the text on the vertical middle of the `=` + pub anchor: Vec2, + /// Height of the handwritten digits in page units + pub symbol_height: f32, + /// Ink color of the `=` + pub color: [f32; 4], +} + +impl Suggestion { + /// Top left corner and size (em) of a text box that puts the result next to the `=`, as + /// high as the handwriting + pub fn text_placement(&self, ascent_per_em: f32) -> (Vec2, f32) { + let size = self.symbol_height / CAP_HEIGHT_PER_EM; + // Baseline at the bottom of digits centered on the `=` + let baseline = self.anchor.y + self.symbol_height / 2.0; + ( + Vec2::new(self.anchor.x, baseline - ascent_per_em * size), + size, + ) + } +} + +#[derive(Debug, Clone)] +struct Pending { + page_id: u64, + equals: [u64; 2], + due: Instant, +} + +/// State of the math assistant: waiting after an `=`, and the suggestion shown +#[derive(Default, Clone)] +pub struct MathAssist { + pub enabled: bool, + /// Built on first use + recognizer: Option>, + pending: Option, + suggestion: Option, + /// Where the UI drew the suggestion (egui points), for [`Self::take_hit`] + pub chip_rect: Option, +} + +impl std::fmt::Debug for MathAssist { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MathAssist") + .field("enabled", &self.enabled) + .field("pending", &self.pending) + .field("suggestion", &self.suggestion) + .finish_non_exhaustive() + } +} + +impl MathAssist { + /// After a stroke was added to `page`: drops the suggestion and waits if the stroke + /// completed an `=` + pub fn stroke_committed(&mut self, page: &Page, now: Instant) { + self.suggestion = None; + self.chip_rect = None; + self.pending = None; + if !self.enabled { + return; + } + if let Some((a, b)) = equals_at_end(&page.strokes) { + self.pending = Some(Pending { + page_id: page.id, + equals: [page.strokes[a].id, page.strokes[b].id], + due: now + TRIGGER_DELAY, + }); + } + } + + /// When [`Self::update`] has something to do + pub fn deadline(&self) -> Option { + self.pending.as_ref().map(|p| p.due) + } + + /// Computes the expression once the pause after the `=` is over (not while a stroke is + /// being drawn). True if a suggestion appeared. + pub fn update( + &mut self, + page: &Page, + now: Instant, + drawing: bool, + decimal_comma: bool, + ) -> bool { + let Some(pending) = self.pending.as_ref() else { + return false; + }; + if drawing || now < pending.due { + return false; + } + let pending = self.pending.take().expect("checked above"); + if pending.page_id != page.id || !self.enabled { + return false; + } + let recognizer = self + .recognizer + .get_or_insert_with(|| Arc::new(Recognizer::new())); + self.suggestion = suggest(recognizer, page, pending.equals, decimal_comma); + self.suggestion.is_some() + } + + /// The suggestion if it belongs to `page` and its `=` is still there + pub fn suggestion_for(&self, page: &Page) -> Option<&Suggestion> { + self.suggestion.as_ref().filter(|s| { + s.page_id == page.id + && s.equals + .iter() + .all(|id| page.strokes.iter().any(|stroke| stroke.id == *id)) + }) + } + + /// The suggestion if `pos` (egui points) is on it; it is then taken away + pub fn take_hit(&mut self, pos: egui::Pos2) -> Option { + if !self.chip_rect.is_some_and(|r| r.contains(pos)) { + return None; + } + self.chip_rect = None; + self.suggestion.take() + } + + pub fn set_enabled(&mut self, enabled: bool) { + self.enabled = enabled; + if !enabled { + self.pending = None; + self.suggestion = None; + self.chip_rect = None; + } + } +} + +/// Handwriting that can be read: pen strokes (also snapped lines), no highlighter, text or +/// images +fn is_ink(stroke: &Stroke) -> bool { + !stroke.is_object() && stroke.tool != StrokeTool::Highlighter && !stroke.points.is_empty() +} + +fn positions(stroke: &Stroke) -> Vec { + stroke.points.iter().map(|s| s.position).collect() +} + +/// A straight, roughly horizontal stroke: one bar of an `=` +fn is_bar(stroke: &Stroke) -> bool { + let size = stroke.aabb.max - stroke.aabb.min; + let length: f32 = positions(stroke) + .windows(2) + .map(|w| w[0].distance(w[1])) + .sum(); + size.x > 0.0 && size.x >= 2.5 * size.y && length <= 1.3 * size.x +} + +/// Indices of the last two ink strokes if they form an `=`: two bars of similar length, one +/// above the other +fn equals_at_end(strokes: &[Stroke]) -> Option<(usize, usize)> { + let mut ink = strokes.iter().enumerate().rev().filter(|(_, s)| is_ink(s)); + let (b, second) = ink.next()?; + let (a, first) = ink.next()?; + if !is_bar(first) || !is_bar(second) { + return None; + } + let (w1, w2) = ( + first.aabb.max.x - first.aabb.min.x, + second.aabb.max.x - second.aabb.min.x, + ); + let (short, long) = (w1.min(w2), w1.max(w2)); + let overlap = first.aabb.max.x.min(second.aabb.max.x) - first.aabb.min.x.max(second.aabb.min.x); + let dy = ((first.aabb.min.y + first.aabb.max.y) - (second.aabb.min.y + second.aabb.max.y)) + .abs() + / 2.0; + (overlap >= 0.5 * short && long <= 2.0 * short && dy >= 0.1 * long && dy <= long) + .then_some((a, b)) +} + +/// Indices of the strokes left of the `=` that make up its expression, in the order they were +/// written: ink in a band around the `=`, going left until a wide gap +fn expression_strokes(strokes: &[Stroke], equals: (usize, usize)) -> Vec { + let (e1, e2) = (&strokes[equals.0], &strokes[equals.1]); + let min = e1.aabb.min.min(e2.aabb.min); + let max = e1.aabb.max.max(e2.aabb.max); + let height = (max.x - min.x) * SYMBOL_PER_EQUALS; + let middle = (min.y + max.y) / 2.0; + let mut candidates: Vec = (0..strokes.len()) + .filter(|&i| i != equals.0 && i != equals.1 && is_ink(&strokes[i])) + .filter(|&i| { + let s = &strokes[i].aabb; + let center = (s.min.y + s.max.y) / 2.0; + (center - middle).abs() <= 2.0 * height + && s.max.y - s.min.y <= 3.0 * height + && s.max.x <= min.x + 0.2 * height + }) + .collect(); + candidates.sort_by(|&a, &b| strokes[b].aabb.max.x.total_cmp(&strokes[a].aabb.max.x)); + let mut left = min.x; + let mut chosen = Vec::new(); + for i in candidates { + let s = &strokes[i].aabb; + if s.max.x < left - MAX_GAP * height || chosen.len() >= MAX_STROKES { + break; + } + left = left.min(s.min.x); + chosen.push(i); + } + chosen.sort_unstable(); + chosen +} + +/// Reads and computes the expression in front of the `=` made of the strokes `equals` +fn suggest( + recognizer: &Recognizer, + page: &Page, + equals: [u64; 2], + decimal_comma: bool, +) -> Option { + let strokes = &page.strokes; + let index = |id: u64| strokes.iter().position(|s| s.id == id); + let pair = (index(equals[0])?, index(equals[1])?); + let mut chosen = expression_strokes(strokes, pair); + if chosen.is_empty() { + return None; + } + chosen.extend([pair.0, pair.1]); + let points: Vec> = chosen.iter().map(|&i| positions(&strokes[i])).collect(); + let recognition = recognizer.recognize(&points)?; + // The `=` must be read as the last symbol, from exactly its two strokes + let last = recognition.symbols.last()?; + let equals_strokes = [points.len() - 2, points.len() - 1]; + let mut last_strokes = last.strokes.clone(); + last_strokes.sort_unstable(); + if last.ch != '=' || last_strokes != equals_strokes { + return None; + } + // In a chain like `x = 2 + 3 =` only the part after the previous `=` is computed + let text = recognition.text.strip_suffix('=')?; + let expression = text.rsplit('=').next()?; + // A dot or comma that was written decides the separator of the result + let decimal_comma = if expression.contains(',') { + true + } else if expression.contains('.') { + false + } else { + decimal_comma + }; + let result = crate::math::solve(expression, decimal_comma)?; + let shown = crate::math::display(expression, decimal_comma)?; + + let (e1, e2) = (&strokes[pair.0], &strokes[pair.1]); + let right = e1.aabb.max.x.max(e2.aabb.max.x); + let middle = (e1.aabb.min.y.min(e2.aabb.min.y) + e1.aabb.max.y.max(e2.aabb.max.y)) / 2.0; + let mut heights: Vec = recognition + .symbols + .iter() + .filter(|s| s.ch.is_ascii_digit() || s.ch.is_ascii_alphabetic()) + .map(|s| s.max.y - s.min.y) + .collect(); + heights.sort_by(f32::total_cmp); + let width = right - e1.aabb.min.x.min(e2.aabb.min.x); + let symbol_height = heights + .last() + .copied() + .unwrap_or(width * SYMBOL_PER_EQUALS) + .max(1.0); + Some(Suggestion { + page_id: page.id, + equals, + expression: shown, + result, + anchor: Vec2::new(right + 0.4 * symbol_height, middle), + symbol_height, + color: e2.color, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::StylusSample; + use crate::math_ink::tests::{write_line, Jitter}; + + const BLACK: [f32; 4] = [0.0, 0.0, 0.0, 1.0]; + + fn stroke(id: u64, points: &[Vec2]) -> Stroke { + let samples = points + .iter() + .enumerate() + .map(|(i, &position)| StylusSample { + position, + pressure: 0.5, + timestamp: i as f64 * 0.01, + }) + .collect(); + Stroke::new(id, samples, Vec::new(), BLACK) + } + + /// A page with `text` written at `origin`, scaled by `scale` + fn page_with(lines: &[(&str, Vec2)], scale: f32) -> Page { + let mut page = Page::new(7, crate::document::PageBackground::Blank); + let mut id = 1; + for (seed, (text, origin)) in lines.iter().enumerate() { + for points in write_line(text, &mut Jitter(seed as u64 + 3)) { + let points: Vec = points.iter().map(|p| *origin + *p * scale).collect(); + page.add_stroke(stroke(id, &points)); + id += 1; + } + } + page + } + + fn run(page: &Page) -> Option { + let mut assist = MathAssist { + enabled: true, + ..Default::default() + }; + let start = Instant::now(); + assist.stroke_committed(page, start); + assert!(!assist.update(page, start, false, true), "too early"); + assist.update(page, start + TRIGGER_DELAY, false, true); + assist.suggestion_for(page).cloned() + } + + #[test] + fn suggests_result_after_equals() { + let page = page_with(&[("(x+3)^2=", Vec2::new(100.0, 200.0))], 3.0); + let suggestion = run(&page).unwrap(); + assert_eq!(suggestion.expression, "(x + 3)²"); + assert_eq!(suggestion.result, "x² + 6x + 9"); + let equals_right = page + .strokes + .iter() + .rev() + .take(2) + .map(|s| s.aabb.max.x) + .fold(f32::MIN, f32::max); + assert!(suggestion.anchor.x > equals_right); + // The handwriting is 14 units high, scaled by 3 + assert!( + (suggestion.symbol_height - 42.0).abs() < 6.0, + "{suggestion:?}" + ); + } + + #[test] + fn ignores_writing_far_to_the_left_and_on_other_lines() { + let page = page_with( + &[ + ("7", Vec2::new(0.0, 200.0)), + ("5·5", Vec2::new(0.0, 300.0)), + ("2^4=", Vec2::new(150.0, 200.0)), + ], + 2.0, + ); + let suggestion = run(&page).unwrap(); + assert_eq!(suggestion.expression, "2⁴"); + assert_eq!(suggestion.result, "16"); + } + + #[test] + fn computes_the_last_part_of_a_chain() { + let page = page_with(&[("x=2·3=", Vec2::new(0.0, 0.0))], 2.0); + assert_eq!(run(&page).unwrap().result, "6"); + } + + #[test] + fn nothing_without_equals_or_when_disabled() { + let page = page_with(&[("2+3", Vec2::ZERO)], 2.0); + assert!(run(&page).is_none()); + let page = page_with(&[("2+3=", Vec2::ZERO)], 2.0); + let mut assist = MathAssist::default(); + assist.stroke_committed(&page, Instant::now()); + assert!(assist.deadline().is_none()); + } + + #[test] + fn new_stroke_or_removed_equals_drops_the_suggestion() { + let mut page = page_with(&[("1+1=", Vec2::ZERO)], 2.0); + let mut assist = MathAssist { + enabled: true, + ..Default::default() + }; + let start = Instant::now(); + assist.stroke_committed(&page, start); + // A stroke still being drawn when the time is up: nothing yet + assert!(!assist.update(&page, start + TRIGGER_DELAY, true, true)); + assert!(assist.update(&page, start + TRIGGER_DELAY, false, true)); + assert_eq!(assist.suggestion_for(&page).unwrap().result, "2"); + let last = page.strokes.last().unwrap().id; + page.remove_strokes_by_ids(&[last]); + assert!(assist.suggestion_for(&page).is_none()); + assist.stroke_committed(&page, start); + assert!(assist.suggestion.is_none()); + } + + #[test] + fn hit_takes_the_suggestion() { + let page = page_with(&[("1+1=", Vec2::ZERO)], 2.0); + let mut assist = MathAssist { + enabled: true, + ..Default::default() + }; + let start = Instant::now(); + assist.stroke_committed(&page, start); + assist.update(&page, start + TRIGGER_DELAY, false, true); + assist.chip_rect = Some(egui::Rect::from_min_size( + egui::pos2(10.0, 10.0), + egui::vec2(50.0, 20.0), + )); + assert!(assist.take_hit(egui::pos2(5.0, 5.0)).is_none()); + assert_eq!(assist.take_hit(egui::pos2(20.0, 20.0)).unwrap().result, "2"); + assert!(assist.take_hit(egui::pos2(20.0, 20.0)).is_none()); + } + + #[test] + fn result_text_sits_on_the_baseline() { + let suggestion = Suggestion { + page_id: 1, + equals: [1, 2], + expression: "1 + 1".into(), + result: "2".into(), + anchor: Vec2::new(100.0, 50.0), + symbol_height: 14.0, + color: BLACK, + }; + let (top_left, size) = suggestion.text_placement(0.9); + assert!((size - 20.0).abs() < 0.01); + // Baseline (top + ascent) at the bottom of digits centered on the `=` + assert!((top_left.y + 0.9 * size - 57.0).abs() < 0.01); + assert_eq!(top_left.x, 100.0); + } +} diff --git a/src/math_ink.rs b/src/math_ink.rs index d27be6e..5d2e1f5 100644 --- a/src/math_ink.rs +++ b/src/math_ink.rs @@ -942,11 +942,11 @@ fn layout(elements: &[Element], line_height: f32) -> String { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; /// Deterministic pseudo-random numbers in [-1, 1] - struct Jitter(u64); + pub(crate) struct Jitter(pub u64); impl Jitter { fn next(&mut self) -> f32 { @@ -1050,7 +1050,7 @@ mod tests { } /// Writes the symbols of `text` from left to right; `^` raises the next symbol - fn write_line(text: &str, jitter: &mut Jitter) -> Vec> { + pub(crate) fn write_line(text: &str, jitter: &mut Jitter) -> Vec> { let mut strokes = Vec::new(); let mut x = 0.0; let mut raised = false; diff --git a/src/settings.rs b/src/settings.rs index 1c22589..2b483f4 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -232,6 +232,8 @@ pub struct UiSettings { /// Folder of the notebooks, e.g. inside a Nextcloud folder; default location if unset /// (Issue #49) pub storage_dir: Option, + /// Compute handwritten math after `=` (Issue #46) + pub math_assist: bool, } impl Default for UiSettings { @@ -253,6 +255,7 @@ impl Default for UiSettings { language: None, eraser_radius_mm: 3.0, storage_dir: None, + math_assist: false, } } } @@ -350,6 +353,7 @@ mod tests { language: Some(Language::English), eraser_radius_mm: 6.0, storage_dir: Some(PathBuf::from("/home/pia/Nextcloud/ink-notes")), + math_assist: true, }; settings.save_to(&path).unwrap(); assert_eq!(UiSettings::load_from(&path), settings); diff --git a/src/ui.rs b/src/ui.rs index 9d67631..45300dd 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -317,6 +317,8 @@ pub enum UiAction { UnlockCorruptedPage(u64), /// Werkzeugleiste an die andere Fensterseite verlegen (wird gespeichert) SetSidebarSide(SidebarSide), + /// Compute handwritten math after `=` on or off (Issue #46) + SetMathAssist(bool), /// Sprache der Oberfläche; `None` = Sprache des Systems SetLanguage(Option), /// Papierfarbe der Anzeige und Aufhellen dunkler Tinte (wird gespeichert) @@ -413,6 +415,8 @@ pub struct UiState { pub update_modal_rect: Option, /// Steuert, ob Touch-Eingaben mit einem Finger zeichnen oder das Dokument verschieben (Default: false = reine Gesten-Navigation & Zoom) pub touch_draw_enabled: bool, + /// Computes handwritten math after an `=` (Issue #46) + pub math: crate::math_assist::MathAssist, /// Zuletzt gelöschtes Notizbuch für Rückgängig während der Sitzung (Papierkorb-Pfad, Titel) pub last_deleted_notebook: Option<(std::path::PathBuf, String)>, /// Stifte der Werkzeugleiste (Farbe und Stärke) und der gewählte davon; `pen_color` und @@ -546,6 +550,7 @@ impl Default for UiState { show_updater_config: false, update_modal_rect: None, touch_draw_enabled: false, + math: crate::math_assist::MathAssist::default(), last_deleted_notebook: None, pen_presets: DEFAULT_PEN_PRESETS, active_pen: 0, @@ -1652,6 +1657,57 @@ fn text_style_buttons(ui: &mut egui::Ui, btn: f32, state: &mut UiState) { /// Bearbeitung eines Textfelds: egui-Textfeld an der Stelle und in der Größe des Felds auf /// dem Blatt (ungedreht), mit derselben Schrift. Esc beendet die Bearbeitung. +/// The result offered next to a handwritten `=` (Issue #46): what was read and the result, +/// e.g. `2⁴ = 16`. Not interactive for egui, so that fingers and palms pass through to the +/// page; the pen and the mouse take it via [`crate::math_assist::MathAssist::take_hit`]. +fn math_suggestion(ctx: &egui::Context, state: &mut UiState, notebook: &Notebook) { + state.math.chip_rect = None; + if state.view_mode { + return; + } + let (Some(view), Some(suggestion)) = ( + state.page_view, + state.math.suggestion_for(notebook.current_page()).cloned(), + ) else { + return; + }; + let size = (suggestion.symbol_height * view.scale * 0.8).clamp(14.0, 48.0); + let response = egui::Area::new(egui::Id::new("math_suggestion")) + .fixed_pos(view.to_screen(suggestion.anchor)) + .pivot(egui::Align2::LEFT_CENTER) + .order(egui::Order::Foreground) + .interactable(false) + .show(ctx, |ui| { + egui::Frame::none() + .fill(egui::Color32::from_rgba_unmultiplied(30, 34, 44, 235)) + .stroke(egui::Stroke::new( + 1.5_f32, + egui::Color32::from_rgb(70, 130, 230), + )) + .rounding(size * 0.4) + .inner_margin(egui::vec2(size * 0.45, size * 0.2)) + .show(ui, |ui| { + ui.horizontal(|ui| { + ui.spacing_mut().item_spacing.x = size * 0.25; + ui.label( + egui::RichText::new(format!("{} =", suggestion.expression)) + .size(size * 0.7) + .color(egui::Color32::from_gray(170)), + ); + ui.label( + egui::RichText::new(&suggestion.result) + .size(size) + .strong() + .color(egui::Color32::WHITE), + ); + }); + }); + }) + .response; + // A little larger than drawn, as the pen does not hit exactly + state.math.chip_rect = Some(response.rect.expand(6.0)); +} + fn text_edit_overlay(ctx: &egui::Context, state: &mut UiState) { let adapt = state.adapt_ink_colors && state.paper_theme.is_dark(); let (Some(view), Some(edit)) = (state.page_view, state.text_edit.as_mut()) else { @@ -3150,6 +3206,7 @@ pub fn render_ui( space_preview(ctx, state, notebook); ruler_overlay(ctx, state); text_edit_overlay(ctx, state); + math_suggestion(ctx, state, notebook); context_menu(ctx, state, &mut actions); let mut pen_btn_rect = None; @@ -4383,6 +4440,15 @@ pub fn render_ui( .on_hover_text( t!("Aus: ein Finger verschiebt, zwei Finger zoomen (empfohlen mit Stift).\nAn: ein Finger zeichnet."), ); + let mut math = state.math.enabled; + if ui + .checkbox(&mut math, t!("Formeln nach „=“ ausrechnen")) + .on_hover_text(t!("Nach einem handgeschriebenen „=“ erscheint das Ergebnis daneben, z. B. 2⁴ = 16. Antippen mit dem Stift oder Klicken fügt es als Text ein.\nGelesen werden Ziffern, + − · : /, Klammern, Hochzahlen, Brüche und x, y, a, b.")) + .changed() + { + state.math.set_enabled(math); + actions.push(UiAction::SetMathAssist(math)); + } if let Some(curve) = pressure_curve_editor(ui, state) { actions.push(UiAction::SavePressureCurve(curve)); }