diff --git a/README.de.md b/README.de.md index e6c002a..105f375 100644 --- a/README.de.md +++ b/README.de.md @@ -192,6 +192,7 @@ src/ ├── disk_watch.rs # Änderungen anderer Geräte übernehmen (Sync-Clients) ├── library.rs # Ordner der Notizbücher (library.json) ├── math.rs # Exaktes Ausrechnen handgeschriebener Formeln (Issue #46) +├── math_ink.rs # Erkennung handgeschriebener Formeln ($P-Punktwolken) ├── autosave.rs # Speichern im Hintergrund ├── export.rs # PDF-Export ├── page_export.rs # Seite als PNG/SVG, Auswahl als Bild diff --git a/README.md b/README.md index 520d4af..f5e53f4 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ src/ ├── disk_watch.rs # Taking over changes from other devices (sync clients) ├── library.rs # Notebook folders (library.json) ├── math.rs # Exact evaluation of handwritten math (Issue #46) +├── math_ink.rs # Recognition of handwritten math ($P point clouds) ├── autosave.rs # Saving in the background ├── export.rs # PDF export ├── page_export.rs # Page as PNG/SVG, selection as image diff --git a/src/lib.rs b/src/lib.rs index 85172c0..35994c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod image_layer; pub mod import; pub mod library; pub mod math; +pub mod math_ink; pub mod osk; pub mod page_export; pub mod paths; diff --git a/src/math_ink.rs b/src/math_ink.rs new file mode 100644 index 0000000..d27be6e --- /dev/null +++ b/src/math_ink.rs @@ -0,0 +1,1259 @@ +//! Recognition of handwritten math (Issue #46, stage 2). +//! +//! Turns the strokes of a handwritten expression into text for [`crate::math::solve`], e.g. +//! the strokes of `(x+3)²=` into `(x+3)^(2)=`. It works on the stroke points, not on an +//! image, in three steps: +//! +//! 1. **Grouping**: consecutive strokes that overlap form one symbol (the two strokes of `+`, +//! `x` or `4`). Straight horizontal strokes and dots are kept apart, as their meaning +//! depends on their neighbours. +//! 2. **Classifying** each symbol with the $P point-cloud recognizer (Vatavu, Anthony, +//! Wobbrock 2012): its points are compared with templates of each symbol. It does not +//! depend on stroke order or direction and needs no trained model; besides the built-in +//! templates, one's own handwriting can be added as templates. +//! 3. **Structure** from the positions: two stacked lines are `=`, two stacked dots `:`, a +//! line with symbols above and below is a fraction bar, a dot low between digits a +//! decimal comma, and smaller raised symbols an exponent. + +use glam::{Vec2, Vec3}; + +/// Points of a symbol after resampling, for $P +const CLOUD_POINTS: usize = 32; +/// Symbols whose best template is further away are not recognized ($P distance) +const MAX_DISTANCE: f32 = 3.0; +/// Factor on the distance per stroke more or less than the template has +const STROKE_PENALTY: f32 = 0.15; +/// Slants of the built-in templates (horizontal shift of the top against the bottom) +const TEMPLATE_SLANTS: [f32; 2] = [0.0, 0.25]; +/// Weight of the turning angle against the position of a point ($P+) +const ANGLE_WEIGHT: f32 = 1.0; + +/// A recognized symbol with the area it covers and the strokes it consists of +#[derive(Debug, Clone, PartialEq)] +pub struct Symbol { + /// `0`–`9`, `x`, `y`, `a`, `b`, `+`, `-`, `·`, `:`, `/`, `(`, `)`, `=`, `,` or `?` + pub ch: char, + pub min: Vec2, + pub max: Vec2, + /// Indices into the strokes passed to [`Recognizer::recognize`] + pub strokes: Vec, +} + +/// Result of [`Recognizer::recognize`] +#[derive(Debug, Clone, PartialEq)] +pub struct Recognition { + /// The expression as text, e.g. `(x+3)^(2)=` or `((1)/(2))+3=` + pub text: String, + /// All symbols, from left to right + pub symbols: Vec, +} + +struct Template { + symbol: char, + strokes: usize, + cloud: Vec, +} + +/// $P recognizer with templates for the symbols of simple math +pub struct Recognizer { + templates: Vec