From 2d14c64c3ed105badd1ffda871ef718eb6036ea2 Mon Sep 17 00:00:00 2001 From: harald Date: Mon, 7 Sep 2026 21:59:42 +0200 Subject: [PATCH] feat(compression): implement transparent LZ4 chunk compression with V1 backwards compatibility --- Cargo.lock | 16 +++++ Cargo.toml | 1 + src/crypto.rs | 115 +++++++++++++++++++++++++++++--- src/mount.rs | 2 +- src/storage.rs | 15 ++++- src/vfs.rs | 17 +++-- tests/integration_test.rs | 135 ++++++++++++++++++++++++++++++++++++-- 7 files changed, 279 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 72be4cb..6225c48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -896,6 +896,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1192,6 +1201,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "lz4_flex", "rand", "rpassword", "rusqlite", @@ -1504,6 +1514,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + [[package]] name = "typenum" version = "1.20.1" diff --git a/Cargo.toml b/Cargo.toml index 9253af1..6d6b092 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" dyn-clone = "1.0" +lz4_flex = "0.11" [profile.release] opt-level = 3 diff --git a/src/crypto.rs b/src/crypto.rs index 0963175..3e4ea30 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -10,9 +10,15 @@ use serde::{Deserialize, Serialize}; use zeroize::Zeroizing; pub const MAGIC_BYTES: &[u8; 8] = b"SANCTUM\0"; -pub const FORMAT_VERSION: u32 = 1; +pub const FORMAT_VERSION_V1: u32 = 1; +pub const FORMAT_VERSION_V2: u32 = 2; +pub const FORMAT_VERSION: u32 = FORMAT_VERSION_V2; pub const CHUNK_SIZE: usize = 1024 * 1024; // 1 MB +/// Kompressions-Flags für Chunk-Payloads in Formatversion >= 2 +pub const COMPRESSION_NONE: u8 = 0x00; +pub const COMPRESSION_LZ4: u8 = 0x01; + pub const DEFAULT_MEMORY_COST_KIB: u32 = 64 * 1024; // 64 MB pub const DEFAULT_TIME_COST: u32 = 3; pub const DEFAULT_PARALLELISM: u32 = 4; @@ -134,12 +140,15 @@ pub fn build_chunk_aad(node_id: i64, chunk_index: u32) -> [u8; 16] { } /// Verschlüsselt einen Payload-Chunk mit dem DEK via AES-256-GCM unter Einbindung von AAD. +/// In Formatversion >= 2 wird der Chunk vor der Verschlüsselung transparent mit LZ4 komprimiert, +/// sofern dadurch eine Größenreduktion erzielt wird. /// Gibt (ciphertext, nonce_12_bytes, tag_16_bytes) zurück. pub fn encrypt_chunk( dek: &[u8; 32], node_id: i64, chunk_index: u32, plaintext: &[u8], + format_version: u32, ) -> Result<(Vec, [u8; 12], [u8; 16])> { let cipher = Aes256Gcm::new_from_slice(dek) .map_err(|e| anyhow::anyhow!("AES-GCM Initialisierungsfehler: {e}"))?; @@ -150,7 +159,28 @@ pub fn encrypt_chunk( let aad = build_chunk_aad(node_id, chunk_index); - let mut buffer = plaintext.to_vec(); + let mut buffer = if format_version >= FORMAT_VERSION_V2 { + if plaintext.is_empty() { + vec![COMPRESSION_NONE] + } else { + let compressed = lz4_flex::compress_prepend_size(plaintext); + // Nur komprimieren, wenn tatsächlich Bytes gespart werden (+1 Byte für das Flag) + if compressed.len() + 1 < plaintext.len() { + let mut buf = Vec::with_capacity(compressed.len() + 1); + buf.push(COMPRESSION_LZ4); + buf.extend_from_slice(&compressed); + buf + } else { + let mut buf = Vec::with_capacity(plaintext.len() + 1); + buf.push(COMPRESSION_NONE); + buf.extend_from_slice(plaintext); + buf + } + } + } else { + plaintext.to_vec() + }; + let tag = cipher .encrypt_in_place_detached(nonce, &aad, &mut buffer) .map_err(|e| anyhow::anyhow!("Chunk-Verschlüsselung fehlgeschlagen: {e}"))?; @@ -162,6 +192,7 @@ pub fn encrypt_chunk( } /// Entschlüsselt und authentifiziert einen Payload-Chunk mit dem DEK via AES-256-GCM. +/// Dekomprimiert LZ4-gepackte Chunks automatisch (in Formatversion >= 2). pub fn decrypt_chunk( dek: &[u8; 32], node_id: i64, @@ -169,6 +200,7 @@ pub fn decrypt_chunk( ciphertext: &[u8], nonce_bytes: &[u8; 12], tag_bytes: &[u8; 16], + format_version: u32, ) -> Result> { let cipher = Aes256Gcm::new_from_slice(dek) .map_err(|e| anyhow::anyhow!("AES-GCM Initialisierungsfehler: {e}"))?; @@ -182,9 +214,25 @@ pub fn decrypt_chunk( .decrypt_in_place_detached(nonce, &aad, &mut buffer, tag) .map_err(|_| anyhow::anyhow!("Chunk-Integritätsprüfung fehlgeschlagen (AEAD Auth-Fehler oder Swap-Angriff)"))?; - Ok(buffer) + if format_version >= FORMAT_VERSION_V2 { + if buffer.is_empty() { + return Ok(Vec::new()); + } + match buffer[0] { + COMPRESSION_NONE => Ok(buffer[1..].to_vec()), + COMPRESSION_LZ4 => { + let decompressed = lz4_flex::decompress_size_prepended(&buffer[1..]) + .map_err(|e| anyhow::anyhow!("LZ4-Dekomprimierungsfehler im Chunk: {e}"))?; + Ok(decompressed) + } + other => bail!("Unbekannte Chunk-Kompressionsmethode: 0x{:02x}", other), + } + } else { + Ok(buffer) + } } + #[cfg(test)] mod tests { use super::*; @@ -226,23 +274,72 @@ mod tests { let node_id = 42i64; let chunk_index = 0u32; - let (ciphertext, nonce, tag) = encrypt_chunk(&dek, node_id, chunk_index, plaintext).unwrap(); + let (ciphertext, nonce, tag) = + encrypt_chunk(&dek, node_id, chunk_index, plaintext, FORMAT_VERSION_V2).unwrap(); - // Reguläre Entschlüsselung - let decrypted = decrypt_chunk(&dek, node_id, chunk_index, &ciphertext, &nonce, &tag).unwrap(); + // Reguläre Entschlüsselung (v2) + let decrypted = + decrypt_chunk(&dek, node_id, chunk_index, &ciphertext, &nonce, &tag, FORMAT_VERSION_V2).unwrap(); assert_eq!(decrypted, plaintext); // Swap Attack 1: Falsche node_id (Chunk in andere Datei verschoben) - let swap_node_err = decrypt_chunk(&dek, 99i64, chunk_index, &ciphertext, &nonce, &tag); + let swap_node_err = + decrypt_chunk(&dek, 99i64, chunk_index, &ciphertext, &nonce, &tag, FORMAT_VERSION_V2); assert!(swap_node_err.is_err()); // Swap Attack 2: Falscher chunk_index (Chunk innerhalb derselben Datei verschoben) - let swap_idx_err = decrypt_chunk(&dek, node_id, 1u32, &ciphertext, &nonce, &tag); + let swap_idx_err = + decrypt_chunk(&dek, node_id, 1u32, &ciphertext, &nonce, &tag, FORMAT_VERSION_V2); assert!(swap_idx_err.is_err()); // Manipulation des Ciphertexts let mut tampered_ct = ciphertext.clone(); tampered_ct[0] ^= 0x01; - assert!(decrypt_chunk(&dek, node_id, chunk_index, &tampered_ct, &nonce, &tag).is_err()); + assert!( + decrypt_chunk(&dek, node_id, chunk_index, &tampered_ct, &nonce, &tag, FORMAT_VERSION_V2).is_err() + ); } + + #[test] + fn test_lz4_chunk_compression_efficiency() { + let dek = generate_dek(); + // Stark komprimierbarer Text (z.B. Logdatei, JSON, Quellcode) + let repeated_text = "Sanctum Secure Vault Storage System ".repeat(500); + let plaintext = repeated_text.as_bytes(); + let node_id = 10i64; + let chunk_index = 0u32; + + let (ciphertext, nonce, tag) = + encrypt_chunk(&dek, node_id, chunk_index, plaintext, FORMAT_VERSION_V2).unwrap(); + + // Der komprimierte Ciphertext muss signifikant kleiner sein als der Klartext + assert!( + ciphertext.len() < plaintext.len() / 5, + "Ciphertext ({}) sollte drastisch kleiner als Plaintext ({}) sein", + ciphertext.len(), + plaintext.len() + ); + + let decrypted = + decrypt_chunk(&dek, node_id, chunk_index, &ciphertext, &nonce, &tag, FORMAT_VERSION_V2).unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn test_v1_backward_compatibility() { + let dek = generate_dek(); + let plaintext = b"Uncompressed Legacy V1 Chunk Payload"; + let node_id = 5i64; + let chunk_index = 0u32; + + // V1 Format: Reine Verschlüsselung ohne Kompressionspräfix + let (ciphertext, nonce, tag) = + encrypt_chunk(&dek, node_id, chunk_index, plaintext, FORMAT_VERSION_V1).unwrap(); + assert_eq!(ciphertext.len(), plaintext.len()); + + let decrypted = + decrypt_chunk(&dek, node_id, chunk_index, &ciphertext, &nonce, &tag, FORMAT_VERSION_V1).unwrap(); + assert_eq!(decrypted, plaintext); + } + } diff --git a/src/mount.rs b/src/mount.rs index ee11e39..ccb7d0f 100644 --- a/src/mount.rs +++ b/src/mount.rs @@ -107,7 +107,7 @@ pub async fn mount_container( .context("Ungültiges Master-Passwort oder Container beschädigt")?; // WebDAV Filesystem und Handler konfigurieren - let fs = SanctumFs::new(db.clone(), dek); + let fs = SanctumFs::new(db.clone(), dek, meta.version); let dav_server = DavHandler::builder() .filesystem(Box::new(fs)) .locksystem(FakeLs::new()) diff --git a/src/storage.rs b/src/storage.rs index 0f0a71d..9fc4f66 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -5,7 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Context, Result}; use rusqlite::{params, Connection, OptionalExtension}; -use crate::crypto::{KdfParams, FORMAT_VERSION, MAGIC_BYTES}; +use crate::crypto::{KdfParams, FORMAT_VERSION, FORMAT_VERSION_V1, FORMAT_VERSION_V2, MAGIC_BYTES}; #[allow(dead_code)] #[derive(Debug, Clone)] @@ -190,10 +190,11 @@ impl Database { bail!("Ungültige Sanctum-Containerdatei: Magic Bytes stimmen nicht überein"); } - if meta.1 != FORMAT_VERSION { + if meta.1 != FORMAT_VERSION_V1 && meta.1 != FORMAT_VERSION_V2 { bail!("Nicht unterstützte Sanctum-Formatversion: {}", meta.1); } + if meta.2.len() != 16 { bail!("Ungültige Salt-Länge im Header"); } @@ -254,6 +255,16 @@ impl Database { Ok(()) } + /// Aktualisiert die Version in der meta-Tabelle (z. B. für Migrationen oder Tests). + pub fn set_meta_version(&self, version: u32) -> Result<()> { + let conn = self.conn.lock().unwrap(); + let rows_affected = conn.execute("UPDATE meta SET version = ?1", params![version])?; + if rows_affected == 0 { + bail!("Konnte Container-Version nicht aktualisieren: meta-Tabelle ist leer"); + } + Ok(()) + } + /// Löst einen hierarchischen Pfad (z. B. "/ordner/datei.txt") in den entsprechenden NodeRecord auf. pub fn resolve_path(&self, raw_path: &str) -> Result> { let normalized = raw_path.trim_matches('/'); diff --git a/src/vfs.rs b/src/vfs.rs index 73d59a5..d22c55f 100644 --- a/src/vfs.rs +++ b/src/vfs.rs @@ -86,6 +86,7 @@ pub struct SanctumFile { meta: SanctumMetaData, // (chunk_index, decrypted_payload, is_dirty) cached_chunk: Option<(u32, Vec, bool)>, + format_version: u32, } impl Debug for SanctumFile { @@ -94,6 +95,7 @@ impl Debug for SanctumFile { .field("node_id", &self.node_id) .field("file_size", &self.file_size) .field("cursor", &self.cursor) + .field("format_version", &self.format_version) .finish() } } @@ -103,6 +105,7 @@ impl SanctumFile { node: NodeRecord, db: Database, dek: Arc>, + format_version: u32, ) -> Self { let meta = SanctumMetaData { is_dir: node.is_dir, @@ -119,14 +122,16 @@ impl SanctumFile { dek, meta, cached_chunk: None, + format_version, } } + /// Schreibt den aktuell im RAM gehaltenen Chunk verschlüsselt in die SQLite-Datenbank zurück. fn flush_cached_chunk(&mut self) -> Result<(), FsError> { if let Some((idx, ref data, true)) = self.cached_chunk { let (ciphertext, nonce, tag) = - encrypt_chunk(&self.dek, self.node_id, idx, data).map_err(|e| { + encrypt_chunk(&self.dek, self.node_id, idx, data, self.format_version).map_err(|e| { error!("Verschlüsselungsfehler beim Chunk-Flush: {e}"); FsError::GeneralFailure })?; @@ -166,6 +171,7 @@ impl SanctumFile { &record.ciphertext, &record.nonce, &record.tag, + self.format_version, ) .map_err(|e| { error!("AEAD-Entschlüsselungsfehler bei Chunk #{chunk_index}: {e}"); @@ -339,13 +345,15 @@ impl DavFile for SanctumFile { pub struct SanctumFs { db: Database, dek: Arc>, + format_version: u32, } impl SanctumFs { - pub fn new(db: Database, dek: Zeroizing<[u8; 32]>) -> Self { + pub fn new(db: Database, dek: Zeroizing<[u8; 32]>, format_version: u32) -> Self { Self { db, dek: Arc::new(dek), + format_version, } } @@ -432,7 +440,7 @@ impl DavFileSystem for SanctumFs { } }; - let file = SanctumFile::new(node, self.db.clone(), self.dek.clone()); + let file = SanctumFile::new(node, self.db.clone(), self.dek.clone(), self.format_version); Ok(Box::new(file) as Box) }) } @@ -678,11 +686,12 @@ impl DavFileSystem for SanctumFs { &record.ciphertext, &record.nonce, &record.tag, + self.format_version, ) .map_err(|_| FsError::GeneralFailure)?; let (new_ct, new_nonce, new_tag) = - encrypt_chunk(&self.dek, dest_node.id, idx, &plaintext) + encrypt_chunk(&self.dek, dest_node.id, idx, &plaintext, self.format_version) .map_err(|_| FsError::GeneralFailure)?; self.db diff --git a/tests/integration_test.rs b/tests/integration_test.rs index b9f246f..b8bece8 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -9,8 +9,8 @@ use dav_server::{ use futures_util::StreamExt; use sanctum::{ crypto::{ - derive_kek, generate_dek, generate_salt, unwrap_dek, wrap_dek, KdfParams, CHUNK_SIZE, - FORMAT_VERSION, + derive_kek, encrypt_chunk, generate_dek, generate_salt, unwrap_dek, wrap_dek, KdfParams, + CHUNK_SIZE, FORMAT_VERSION, FORMAT_VERSION_V1, }, storage::Database, vfs::SanctumFs, @@ -62,7 +62,7 @@ async fn test_sanctum_full_container_lifecycle() { assert_eq!(*dek, *unwrapped_dek); // 3. VFS Filesystem-Operationen (WebDAV Trait) - let fs = SanctumFs::new(db.clone(), unwrapped_dek); + let fs = SanctumFs::new(db.clone(), unwrapped_dek, meta.version); // Ordner erstellen let docs_path = DavPath::new("/documents").unwrap(); @@ -189,7 +189,7 @@ async fn test_sanctum_password_change() { db.checkpoint().expect("Checkpoint"); // Datei im Container mit DEK anlegen - let fs_v1 = SanctumFs::new(db.clone(), original_dek.clone()); + let fs_v1 = SanctumFs::new(db.clone(), original_dek.clone(), FORMAT_VERSION); let file_path = DavPath::new("/secret.txt").unwrap(); let file_data = b"Verschluesselte Geheimdaten vor der Passwortaenderung!"; let mut opts_write = OpenOptions::default(); @@ -234,14 +234,13 @@ async fn test_sanctum_password_change() { let active_dek = unwrap_dek(&new_kek, &meta_after.wrapped_dek, &meta_after.header_nonce, &meta_after.header_tag) .expect("Unwrap with new password"); - let fs_v2 = SanctumFs::new(db.clone(), active_dek); + let fs_v2 = SanctumFs::new(db.clone(), active_dek, meta_after.version); let mut opts_read = OpenOptions::default(); opts_read.read = true; let mut read_file = fs_v2.open(&file_path, opts_read).await.expect("Open read with new password"); let read_bytes = read_file.read_bytes(file_data.len()).await.expect("Read bytes"); assert_eq!(&read_bytes[..], file_data, "Daten muessen nach Passwortaenderung unveraendert lesbar sein!"); - drop(read_file); drop(fs_v2); drop(db); @@ -249,3 +248,127 @@ async fn test_sanctum_password_change() { let _ = std::fs::remove_file(&container_path); } +#[tokio::test] +async fn test_sanctum_lz4_compression_efficiency() { + let temp_dir = std::env::temp_dir(); + let container_path: PathBuf = temp_dir.join(format!("test_sanctum_compress_{}.sanctum", std::process::id())); + + if container_path.exists() { + let _ = std::fs::remove_file(&container_path); + } + + let password = "CompressTestPassword2026!"; + let salt = generate_salt(); + let kdf_params = KdfParams { + memory_cost: 1024, + time_cost: 1, + parallelism: 1, + }; + let kek = derive_kek(password, &salt, &kdf_params).expect("KEK"); + let dek = generate_dek(); + let (wrapped_dek, header_nonce, header_tag) = wrap_dek(&kek, &dek).expect("wrap"); + + let db = Database::open(&container_path).expect("open db"); + db.init_schema(&salt, &kdf_params, &wrapped_dek, &header_nonce, &header_tag).expect("init schema"); + + let fs = SanctumFs::new(db.clone(), dek.clone(), FORMAT_VERSION); + + // 2 MB hochkompressible Textdaten erzeugen (z. B. wiederholende Zeilen) + let pattern = b"Sanctum high performance encrypted container storage with transparent LZ4 compression.\n"; + let target_size = 2 * 1024 * 1024; + let mut repetitive_data = Vec::with_capacity(target_size); + while repetitive_data.len() < target_size { + repetitive_data.extend_from_slice(pattern); + } + repetitive_data.truncate(target_size); + + // Datei über VFS schreiben + let test_file_path = DavPath::new("/compressed_test.log").unwrap(); + let mut opts = OpenOptions::default(); + opts.write = true; + opts.create_new = true; + let mut file = fs.open(&test_file_path, opts).await.expect("open write"); + file.write_bytes(Bytes::copy_from_slice(&repetitive_data)).await.expect("write"); + file.flush().await.expect("flush"); + drop(file); + db.checkpoint().expect("checkpoint"); + + // Chunk-Knoten ermitteln und Ciphertext-Größe in der Datenbank prüfen + let node = db.resolve_path("/compressed_test.log").expect("resolve").expect("found"); + assert_eq!(node.size, target_size as u64); + + let chunk0 = db.read_chunk(node.id, 0).expect("read chunk").expect("chunk 0 exists"); + // 1 MB Rohdaten komprimiert mit LZ4 sollte typischerweise < 100 KB sein + assert!( + chunk0.ciphertext.len() < 100_000, + "Ciphertext should be compressed from 1MB to < 100KB, was {}", + chunk0.ciphertext.len() + ); + + // Datei zurücklesen und mit Original vergleichen + let mut read_opts = OpenOptions::default(); + read_opts.read = true; + let mut read_file = fs.open(&test_file_path, read_opts).await.expect("open read"); + let read_back = read_file.read_bytes(target_size).await.expect("read back"); + assert_eq!(read_back.len(), target_size); + assert_eq!(&read_back[..], &repetitive_data[..]); + + drop(read_file); + drop(fs); + drop(db); + let _ = std::fs::remove_file(&container_path); +} + +#[tokio::test] +async fn test_sanctum_v1_backward_compatibility() { + let temp_dir = std::env::temp_dir(); + let container_path: PathBuf = temp_dir.join(format!("test_sanctum_v1_compat_{}.sanctum", std::process::id())); + + if container_path.exists() { + let _ = std::fs::remove_file(&container_path); + } + + let password = "V1LegacyPassword!"; + let salt = generate_salt(); + let kdf_params = KdfParams { + memory_cost: 1024, + time_cost: 1, + parallelism: 1, + }; + let kek = derive_kek(password, &salt, &kdf_params).expect("KEK"); + let dek = generate_dek(); + let (wrapped_dek, header_nonce, header_tag) = wrap_dek(&kek, &dek).expect("wrap"); + + let db = Database::open(&container_path).expect("open db"); + db.init_schema(&salt, &kdf_params, &wrapped_dek, &header_nonce, &header_tag).expect("init schema"); + // Formatversion auf V1 setzen + db.set_meta_version(FORMAT_VERSION_V1).expect("set v1"); + + let meta = db.read_meta().expect("read meta"); + assert_eq!(meta.version, FORMAT_VERSION_V1); + + // Datei-Knoten direkt in DB erstellen + let node = db.create_node(1, "legacy_v1.txt", false).expect("create node"); + + // V1 Chunk mit encrypt_chunk(..., FORMAT_VERSION_V1) erzeugen und direkt in DB schreiben + let v1_plaintext = b"Legacy Sanctum V1 uncompressed data payload."; + let (ct, nonce, tag) = encrypt_chunk(&dek, node.id, 0, v1_plaintext, FORMAT_VERSION_V1).expect("encrypt v1"); + db.write_chunk(node.id, 0, &nonce, &tag, &ct).expect("write chunk"); + db.update_node_size_and_time(node.id, v1_plaintext.len() as u64, 12345678).expect("update size"); + db.checkpoint().expect("checkpoint"); + + // Öffnen über SanctumFs konfiguriert für V1 + let fs = SanctumFs::new(db.clone(), dek.clone(), FORMAT_VERSION_V1); + let path = DavPath::new("/legacy_v1.txt").unwrap(); + let mut opts = OpenOptions::default(); + opts.read = true; + let mut file = fs.open(&path, opts).await.expect("open v1 file"); + let read_data = file.read_bytes(v1_plaintext.len()).await.expect("read v1"); + assert_eq!(&read_data[..], v1_plaintext); + + drop(file); + drop(fs); + drop(db); + let _ = std::fs::remove_file(&container_path); +} +