From d16cd2db7222e91c9da5081ae1bfd837ab522c39 Mon Sep 17 00:00:00 2001 From: Harald Date: Wed, 28 Oct 2026 14:27:25 +0100 Subject: [PATCH] feat(math): recognize handwritten expressions (Issue #46, stage 2) New module math_ink.rs turns the strokes of an expression into text for math::solve. Consecutive overlapping strokes are grouped into symbols, each symbol is classified with the $P point-cloud recognizer (plus the turning angle and a small stroke-count penalty) against built-in templates for digits, + ( ) / x y a b, and the structure comes from the positions: stacked lines are =, stacked dots :, lines with symbols above and below are fraction bars, low dots between digits decimal commas and raised symbols exponents. Own templates can be added for stage 4. Tested with synthetic handwriting (slanted, rotated, warped); not used by the UI yet. Co-Authored-By: Claude Opus 5.5 --- README.de.md | 1 + README.md | 1 + src/lib.rs | 1 + src/math_ink.rs | 1259 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1262 insertions(+) create mode 100644 src/math_ink.rs diff --git a/README.de.md b/README.de.md index e6c002a..105f375 100644 --- a/README.de.md +++ b/README.de.md @@ -192,6 +192,7 @@ src/ ├── disk_watch.rs # Änderungen anderer Geräte übernehmen (Sync-Clients) ├── library.rs # Ordner der Notizbücher (library.json) ├── math.rs # Exaktes Ausrechnen handgeschriebener Formeln (Issue #46) +├── math_ink.rs # Erkennung handgeschriebener Formeln ($P-Punktwolken) ├── autosave.rs # Speichern im Hintergrund ├── export.rs # PDF-Export ├── page_export.rs # Seite als PNG/SVG, Auswahl als Bild diff --git a/README.md b/README.md index 520d4af..f5e53f4 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ src/ ├── disk_watch.rs # Taking over changes from other devices (sync clients) ├── library.rs # Notebook folders (library.json) ├── math.rs # Exact evaluation of handwritten math (Issue #46) +├── math_ink.rs # Recognition of handwritten math ($P point clouds) ├── autosave.rs # Saving in the background ├── export.rs # PDF export ├── page_export.rs # Page as PNG/SVG, selection as image diff --git a/src/lib.rs b/src/lib.rs index 85172c0..35994c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod image_layer; pub mod import; pub mod library; pub mod math; +pub mod math_ink; pub mod osk; pub mod page_export; pub mod paths; diff --git a/src/math_ink.rs b/src/math_ink.rs new file mode 100644 index 0000000..d27be6e --- /dev/null +++ b/src/math_ink.rs @@ -0,0 +1,1259 @@ +//! Recognition of handwritten math (Issue #46, stage 2). +//! +//! Turns the strokes of a handwritten expression into text for [`crate::math::solve`], e.g. +//! the strokes of `(x+3)²=` into `(x+3)^(2)=`. It works on the stroke points, not on an +//! image, in three steps: +//! +//! 1. **Grouping**: consecutive strokes that overlap form one symbol (the two strokes of `+`, +//! `x` or `4`). Straight horizontal strokes and dots are kept apart, as their meaning +//! depends on their neighbours. +//! 2. **Classifying** each symbol with the $P point-cloud recognizer (Vatavu, Anthony, +//! Wobbrock 2012): its points are compared with templates of each symbol. It does not +//! depend on stroke order or direction and needs no trained model; besides the built-in +//! templates, one's own handwriting can be added as templates. +//! 3. **Structure** from the positions: two stacked lines are `=`, two stacked dots `:`, a +//! line with symbols above and below is a fraction bar, a dot low between digits a +//! decimal comma, and smaller raised symbols an exponent. + +use glam::{Vec2, Vec3}; + +/// Points of a symbol after resampling, for $P +const CLOUD_POINTS: usize = 32; +/// Symbols whose best template is further away are not recognized ($P distance) +const MAX_DISTANCE: f32 = 3.0; +/// Factor on the distance per stroke more or less than the template has +const STROKE_PENALTY: f32 = 0.15; +/// Slants of the built-in templates (horizontal shift of the top against the bottom) +const TEMPLATE_SLANTS: [f32; 2] = [0.0, 0.25]; +/// Weight of the turning angle against the position of a point ($P+) +const ANGLE_WEIGHT: f32 = 1.0; + +/// A recognized symbol with the area it covers and the strokes it consists of +#[derive(Debug, Clone, PartialEq)] +pub struct Symbol { + /// `0`–`9`, `x`, `y`, `a`, `b`, `+`, `-`, `·`, `:`, `/`, `(`, `)`, `=`, `,` or `?` + pub ch: char, + pub min: Vec2, + pub max: Vec2, + /// Indices into the strokes passed to [`Recognizer::recognize`] + pub strokes: Vec, +} + +/// Result of [`Recognizer::recognize`] +#[derive(Debug, Clone, PartialEq)] +pub struct Recognition { + /// The expression as text, e.g. `(x+3)^(2)=` or `((1)/(2))+3=` + pub text: String, + /// All symbols, from left to right + pub symbols: Vec, +} + +struct Template { + symbol: char, + strokes: usize, + cloud: Vec, +} + +/// $P recognizer with templates for the symbols of simple math +pub struct Recognizer { + templates: Vec