harald/sanctum
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 14 Wiki Activity

feat(carrier): implement carrier format V2 paged manifest and secondary index

Browse Source
This commit is contained in:
harald
2026-09-20 00:15:11 +02:00
parent 02f731f57d
commit 167e4b65fc
6 changed files with 1241 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/carrier_v2_paged_manifest_test.rs
+686
View File
@@ -0,0 +1,686 @@
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::Instant;
use bytes::Bytes;
use dav_server::davpath::DavPath;
use dav_server::fs::{DavFileSystem, OpenOptions};
use rand::rngs::OsRng;
use rand::RngCore;
use sanctum::carrier::{
write_carrier_block, CarrierFs, CarrierFsInner, CarrierInode, CarrierManifest, CARRIER_MAGIC,
CARRIER_VERSION, CARRIER_VERSION_V2,
};
use sanctum::crypto::{
derive_kek, generate_dek, generate_salt, wrap_slot0_payload, wrap_slot1_payload, KdfParams,
MIN_MEMORY_COST_KIB, MIN_TIME_COST,
};
use sanctum::storage::{Database, UnlockedKeys};
fn temp_db_path(prefix: &str) -> PathBuf {
let mut path = std::env::temp_dir();
let id: u64 = OsRng.next_u64();
path.push(format!("sanctum_test_v2_{}_{}.sanctum", prefix, id));
path
}
struct TestEnv {
pub path: PathBuf,
pub db: Database,
pub carrier_node_id: i64,
pub auth_hidden: UnlockedKeys,
}
impl Drop for TestEnv {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
fn setup_carrier_env(prefix: &str, size_mb: usize) -> TestEnv {
let path = temp_db_path(prefix);
let carrier_size_bytes = (size_mb * 1024 * 1024) as u64;
let carrier_name = "carrier_v2.dat";
let pass_decoy = "DecoyPassword2026!";
let pass_hidden = "SuperSecretHiddenPassword2026!";
let kdf_params = KdfParams {
memory_cost: MIN_MEMORY_COST_KIB,
time_cost: MIN_TIME_COST,
parallelism: 1,
};
let salt_0 = generate_salt();
let kek_0 = derive_kek(pass_decoy, &salt_0, &kdf_params).unwrap();
let dek_0 = generate_dek();
let salt_1 = generate_salt();
let kek_1 = derive_kek(pass_hidden, &salt_1, &kdf_params).unwrap();
let dek_1 = generate_dek();
let carrier_node_id = 3i64;
let (wrapped_0, nonce_0, tag_0) = wrap_slot0_payload(&kek_0, &dek_0, carrier_node_id).unwrap();
let (wrapped_1, nonce_1, tag_1) =
wrap_slot1_payload(&kek_1, &dek_1, &dek_0, carrier_node_id).unwrap();
let db = Database::open(&path).expect("Open database");
db.init_schema_with_carrier(
&salt_0,
&kdf_params,
&wrapped_0,
&nonce_0,
&tag_0,
Some((
carrier_name,
carrier_size_bytes,
&salt_1,
&kdf_params,
&wrapped_1,
&nonce_1,
&tag_1,
&dek_0,
&dek_1,
)),
)
.expect("Init carrier schema");
db.checkpoint().unwrap();
let meta = db.read_meta().unwrap();
let auth_hidden = meta.authenticate(pass_hidden).expect("Auth hidden");
TestEnv {
path,
db,
carrier_node_id,
auth_hidden,
}
}
fn load_carrier_fs(env: &TestEnv) -> CarrierFs {
CarrierFs::load(
env.db.clone(),
env.carrier_node_id,
std::sync::Arc::new(env.auth_hidden.carrier_dek().unwrap().clone()),
std::sync::Arc::new(env.auth_hidden.dek().clone()),
env.auth_hidden.version(),
true,
)
.expect("Load CarrierFs")
}
/// 1. Kapazitätstest: >10.000 Inodes künstlich erzeugen, über mehrere Seiten speichern und neu laden
#[tokio::test]
async fn test_capacity_large_inode_table() {
let env = setup_carrier_env("capacity_large", 25);
let cfs = load_carrier_fs(&env);
// Initial hat ein frischer Container 1 Seite (Block 2)
assert_eq!(cfs.page_count(), 1);
// 10.000 Inodes direkt in den Speicher einfügen
{
let mut inner = cfs.inner.lock().unwrap();
for i in 0..10_000 {
let id = inner.manifest.next_inode_id;
inner.manifest.next_inode_id += 1;
let inode = CarrierInode {
id,
parent_id: Some(1),
name: format!("file_{:05}.txt", i),
is_dir: false,
size: i % 500,
created_at: 1_700_000_000 + i,
modified_at: 1_700_000_000 + i,
blocks: Vec::new(),
};
inner.manifest.inodes.insert(id, inode);
inner.index_add_child(1, id);
}
inner.mark_dirty();
}
// Speichern erzwingen
cfs.sync_manifest().expect("Sync 10.000 inodes");
// In V2 müssen die 10.000 Inodes auf mindestens 2 Seiten aufgeteilt worden sein
let pages_after_save = cfs.page_count();
assert!(
pages_after_save >= 2,
"10.000 Inodes müssen auf mindestens 2 Seiten aufgeteilt werden, tatsächlich: {}",
pages_after_save
);
drop(cfs);
// Neu aus Trägerdatei laden
let cfs_reloaded = load_carrier_fs(&env);
assert_eq!(cfs_reloaded.page_count(), pages_after_save);
assert_eq!(cfs_reloaded.corrupted_pages(), 0);
// Prüfe, dass beliebige Dateien korrekt existieren und Pfade auflösen
let test_file_1 = cfs_reloaded.resolve_path("/file_00000.txt");
assert!(test_file_1.is_some());
assert_eq!(test_file_1.unwrap().name, "file_00000.txt");
let test_file_mid = cfs_reloaded.resolve_path("/file_04999.txt");
assert!(test_file_mid.is_some());
assert_eq!(test_file_mid.unwrap().name, "file_04999.txt");
let test_file_end = cfs_reloaded.resolve_path("/file_09999.txt");
assert!(test_file_end.is_some());
assert_eq!(test_file_end.unwrap().name, "file_09999.txt");
let nonexistent = cfs_reloaded.resolve_path("/file_10000.txt");
assert!(nonexistent.is_none());
}
/// 2. Migrationstest: Einen V1-Container laden, modifizieren, als V2 speichern und verifizieren
#[tokio::test]
async fn test_v1_to_v2_migration() {
let env = setup_carrier_env("migration_v1_v2", 10);
// Einen V1-Container konstruieren: Block 0 und Block 1 halten CarrierManifest mit version = 1
let mut v1_manifest = CarrierManifest {
magic: *CARRIER_MAGIC,
version: CARRIER_VERSION, // 1
manifest_generation: 1,
total_blocks: 10,
free_blocks: (2..10).collect(),
next_inode_id: 3,
page_block_indices: Vec::new(),
inodes: HashMap::new(),
};
v1_manifest.inodes.insert(
1,
CarrierInode {
id: 1,
parent_id: None,
name: String::new(),
is_dir: true,
size: 0,
created_at: 1000,
modified_at: 1000,
blocks: Vec::new(),
},
);
v1_manifest.inodes.insert(
2,
CarrierInode {
id: 2,
parent_id: Some(1),
name: "original_v1.txt".to_string(),
is_dir: false,
size: 128,
created_at: 1000,
modified_at: 1000,
blocks: Vec::new(),
},
);
let v1_bytes = serde_json::to_vec(&v1_manifest).unwrap();
let dek_outer = env.auth_hidden.carrier_dek().unwrap();
let dek_inner = env.auth_hidden.dek();
write_carrier_block(
&env.db,
env.carrier_node_id,
0,
&dek_outer,
&dek_inner,
&v1_bytes,
env.auth_hidden.version(),
)
.unwrap();
write_carrier_block(
&env.db,
env.carrier_node_id,
1,
&dek_outer,
&dek_inner,
&v1_bytes,
env.auth_hidden.version(),
)
.unwrap();
// 1. Unter v0.9.0 laden: V1-Format wird transparent erkannt
let cfs = load_carrier_fs(&env);
{
let inner = cfs.inner.lock().unwrap();
assert_eq!(inner.manifest.version, CARRIER_VERSION);
}
let orig = cfs.resolve_path("/original_v1.txt");
assert!(orig.is_some());
assert_eq!(orig.unwrap().id, 2);
// 2. Änderung vornehmen: Datei hinzufügen (löst Dirty-Tracking aus)
let new_file_path = DavPath::new("/new_v2.txt").unwrap();
let mut file = cfs
.open(
&new_file_path,
OpenOptions {
create_new: true,
write: true,
..Default::default()
},
)
.await
.expect("Create new file in migrated container");
file.write_bytes(Bytes::from_static(b"migrated to v2"))
.await
.unwrap();
file.flush().await.unwrap();
drop(file);
// Explizit synchronisieren
cfs.sync_manifest().expect("Sync manifest during migration");
drop(cfs);
// 3. Neu laden: Jetzt muss das Dateisystem im Format V2 vorliegen
let cfs_reloaded = load_carrier_fs(&env);
{
let inner = cfs_reloaded.inner.lock().unwrap();
assert_eq!(inner.manifest.version, CARRIER_VERSION_V2);
assert!(!inner.manifest.page_block_indices.is_empty());
}
// Beide Dateien müssen vorhanden sein
assert!(cfs_reloaded.resolve_path("/original_v1.txt").is_some());
assert!(cfs_reloaded.resolve_path("/new_v2.txt").is_some());
}
/// 3. Teilausfalltest (D-01 Fail-Soft): Beschädigung einer einzelnen Inode-Seite führt nicht zum Totalverlust
#[tokio::test]
async fn test_partial_page_corruption_resilience() {
let env = setup_carrier_env("partial_corruption", 20);
let cfs = load_carrier_fs(&env);
// Genügend Inodes erzeugen, um 2 Seiten zu belegen
{
let mut inner = cfs.inner.lock().unwrap();
for i in 0..1500 {
let id = inner.manifest.next_inode_id;
inner.manifest.next_inode_id += 1;
let name = if i < 750 {
format!("alpha_{:04}_{}", i, "a".repeat(800))
} else {
format!("omega_{:04}_{}", i, "z".repeat(800))
};
let inode = CarrierInode {
id,
parent_id: Some(1),
name,
is_dir: false,
size: 10,
created_at: 2000,
modified_at: 2000,
blocks: Vec::new(),
};
inner.manifest.inodes.insert(id, inode);
inner.index_add_child(1, id);
}
inner.mark_dirty();
}
cfs.sync_manifest().expect("Sync 2 pages");
let page_blocks = {
let inner = cfs.inner.lock().unwrap();
inner.manifest.page_block_indices.clone()
};
assert!(
page_blocks.len() >= 2,
"Benötigt mindestens 2 Seiten, hat: {}",
page_blocks.len()
);
let second_page_block = page_blocks[1];
drop(cfs);
// 2. Gezielte Beschädigung: Zweite Seite in SQLite mit Rauschen überschreiben
{
let conn = rusqlite::Connection::open(&env.path).unwrap();
let mut noise = vec![0u8; 500];
OsRng.fill_bytes(&mut noise);
conn.execute(
"UPDATE chunks SET ciphertext = ?1 WHERE node_id = ?2 AND chunk_index = ?3",
rusqlite::params![noise, env.carrier_node_id, second_page_block],
)
.unwrap();
}
// 3. Neu laden: Darf NICHT abbrechen (Fail-Soft), sondern überspringt die defekte Seite
let cfs_reloaded = load_carrier_fs(&env);
assert_eq!(
cfs_reloaded.corrupted_pages(),
1,
"Muss genau 1 beschädigte Seite protokollieren"
);
// Wurzelverzeichnis '/' muss intakt sein
assert!(cfs_reloaded.resolve_path("/").is_some());
// Einträge der ersten, unbeschädigten Seite müssen weiterhin vollständig auffindbar sein
let alpha_0 = cfs_reloaded.resolve_path(&format!("/alpha_0000_{}", "a".repeat(800)));
assert!(
alpha_0.is_some(),
"Dateien aus der intakten Seite 0 müssen lesbar bleiben"
);
}
/// 4. Index-Konsistenztest (D-02): Operationsfolge gegen frisch aufgebauten Referenzindex abgleichen
#[tokio::test]
async fn test_children_index_consistency() {
let env = setup_carrier_env("children_consistency", 10);
let cfs = load_carrier_fs(&env);
// 1. Verzeichnisse anlegen
cfs.create_dir(&DavPath::new("/docs").unwrap())
.await
.unwrap();
cfs.create_dir(&DavPath::new("/docs/work").unwrap())
.await
.unwrap();
cfs.create_dir(&DavPath::new("/photos").unwrap())
.await
.unwrap();
// 2. Dateien anlegen
let opt = OpenOptions {
create_new: true,
write: true,
..Default::default()
};
let mut f1 = cfs
.open(&DavPath::new("/docs/report.txt").unwrap(), opt.clone())
.await
.unwrap();
f1.write_bytes(Bytes::from_static(b"report")).await.unwrap();
f1.flush().await.unwrap();
drop(f1);
let mut f2 = cfs
.open(&DavPath::new("/docs/work/notes.txt").unwrap(), opt.clone())
.await
.unwrap();
f2.write_bytes(Bytes::from_static(b"notes")).await.unwrap();
f2.flush().await.unwrap();
drop(f2);
let mut f3 = cfs
.open(&DavPath::new("/photos/pic.jpg").unwrap(), opt.clone())
.await
.unwrap();
f3.write_bytes(Bytes::from_static(b"pic")).await.unwrap();
f3.flush().await.unwrap();
drop(f3);
// 3. Verschieben/Umbenennen
cfs.rename(
&DavPath::new("/docs/work/notes.txt").unwrap(),
&DavPath::new("/photos/notes_moved.txt").unwrap(),
)
.await
.unwrap();
// 4. Datei löschen
cfs.remove_file(&DavPath::new("/docs/report.txt").unwrap())
.await
.unwrap();
// 5. Verzeichnis löschen
cfs.remove_dir(&DavPath::new("/docs/work").unwrap())
.await
.unwrap();
// 6. Abgleich: children_index gegen frisch aufgebauten Referenzindex prüfen
{
let inner = cfs.inner.lock().unwrap();
let reference_index = CarrierFsInner::build_children_index(&inner.manifest.inodes);
for (parent_id, ref_children) in &reference_index {
let actual_children = inner.children_index.get(parent_id);
assert!(
actual_children.is_some(),
"Parent {} fehlt im children_index",
parent_id
);
let mut ref_sorted = ref_children.clone();
ref_sorted.sort();
let mut act_sorted = actual_children.unwrap().clone();
act_sorted.sort();
assert_eq!(
ref_sorted, act_sorted,
"Inkonsistenz der Kinder für Parent {}",
parent_id
);
}
// Auch die Umkehrung prüfen (keine Geister-Einträge)
for (parent_id, actual_children) in &inner.children_index {
if actual_children.is_empty() {
continue;
}
assert!(
reference_index.contains_key(parent_id),
"children_index enthält verwaisten Parent {}",
parent_id
);
}
}
}
/// 5. Performance-Sanity-Check: Pfadauflösung bei 5.000 Geschwistern muss sub-millisekündlich sein (D-02)
#[tokio::test]
async fn test_secondary_index_path_resolution_speed() {
let env = setup_carrier_env("perf_resolution", 15);
let cfs = load_carrier_fs(&env);
cfs.create_dir(&DavPath::new("/big_dir").unwrap())
.await
.unwrap();
let big_dir_node = cfs.resolve_path("/big_dir").expect("big_dir node");
// 5.000 Kindknoten in /big_dir anlegen
{
let mut inner = cfs.inner.lock().unwrap();
for i in 0..5_000 {
let id = inner.manifest.next_inode_id;
inner.manifest.next_inode_id += 1;
let inode = CarrierInode {
id,
parent_id: Some(big_dir_node.id),
name: format!("child_{:04}.txt", i),
is_dir: false,
size: 10,
created_at: 3000,
modified_at: 3000,
blocks: Vec::new(),
};
inner.manifest.inodes.insert(id, inode);
inner.index_add_child(big_dir_node.id, id);
}
inner.mark_dirty();
}
// Messung: Pfadauflösung für den letzten Eintrag
let start = Instant::now();
let target = cfs.resolve_path("/big_dir/child_4999.txt");
let elapsed = start.elapsed();
assert!(target.is_some());
assert_eq!(target.unwrap().name, "child_4999.txt");
// Der Sekundärindex muss den Eintrag in unter 5 Millisekunden finden (typisch < 0.2 ms)
assert!(
elapsed.as_millis() < 5,
"Pfadauflösung dauerte zu lange: {:?}",
elapsed
);
}
/// 6. Platzmangel-Migrationstest: Migration eines Containers ohne freie Blöcke bricht sauber ab (D-03)
#[tokio::test]
async fn test_out_of_space_migration_aborts_cleanly() {
let env = setup_carrier_env("migration_no_space", 5);
// V1-Manifest mit 0 freien Blöcken (free_blocks ist leer)
let mut v1_manifest = CarrierManifest {
magic: *CARRIER_MAGIC,
version: CARRIER_VERSION,
manifest_generation: 1,
total_blocks: 2,
free_blocks: Vec::new(), // Keine freien Blöcke!
next_inode_id: 2,
page_block_indices: Vec::new(),
inodes: HashMap::new(),
};
v1_manifest.inodes.insert(
1,
CarrierInode {
id: 1,
parent_id: None,
name: String::new(),
is_dir: true,
size: 0,
created_at: 1000,
modified_at: 1000,
blocks: Vec::new(),
},
);
let v1_bytes = serde_json::to_vec(&v1_manifest).unwrap();
let dek_outer = env.auth_hidden.carrier_dek().unwrap();
let dek_inner = env.auth_hidden.dek();
write_carrier_block(
&env.db,
env.carrier_node_id,
0,
&dek_outer,
&dek_inner,
&v1_bytes,
env.auth_hidden.version(),
)
.unwrap();
write_carrier_block(
&env.db,
env.carrier_node_id,
1,
&dek_outer,
&dek_inner,
&v1_bytes,
env.auth_hidden.version(),
)
.unwrap();
let cfs = load_carrier_fs(&env);
// Versuch zu speichern muss mit klarer Fehlermeldung fehlschlagen
let res = {
let mut inner = cfs.inner.lock().unwrap();
inner.mark_dirty();
inner.save_manifest()
};
assert!(res.is_err());
let err_msg = res.unwrap_err().to_string();
assert!(
err_msg.contains("Nicht genügend freie Blöcke"),
"Fehlermeldung muss auf Speichermangel hinweisen: {}",
err_msg
);
drop(cfs);
// Trägerdatei muss weiterhin intakt als V1 ladbar sein
let cfs_recheck = load_carrier_fs(&env);
{
let inner = cfs_recheck.inner.lock().unwrap();
assert_eq!(inner.manifest.version, CARRIER_VERSION);
}
}
/// 7. Seitenfreigabe (D-05): Überzählige Seitenblöcke werden bei Schrumpfung geschreddert und freigegeben
#[tokio::test]
async fn test_page_block_reclaiming_on_shrink() {
let env = setup_carrier_env("page_reclaiming", 25);
let cfs = load_carrier_fs(&env);
let initial_free = {
let inner = cfs.inner.lock().unwrap();
inner.manifest.free_blocks.len()
};
// 1. Viele Inodes erzeugen, sodass mindestens 3 Seiten belegt werden
let mut added_ids = Vec::new();
{
let mut inner = cfs.inner.lock().unwrap();
for i in 0..2500 {
let id = inner.manifest.next_inode_id;
inner.manifest.next_inode_id += 1;
added_ids.push(id);
let inode = CarrierInode {
id,
parent_id: Some(1),
name: format!("large_item_{:04}_{}", i, "x".repeat(800)),
is_dir: false,
size: 10,
created_at: 4000,
modified_at: 4000,
blocks: Vec::new(),
};
inner.manifest.inodes.insert(id, inode);
inner.index_add_child(1, id);
}
inner.mark_dirty();
}
cfs.sync_manifest().expect("Sync 3 pages");
let pages_before = cfs.page_count();
assert!(
pages_before >= 3,
"Muss mindestens 3 Seiten belegen, hat: {}",
pages_before
);
let free_after_expand = {
let inner = cfs.inner.lock().unwrap();
inner.manifest.free_blocks.len()
};
assert_eq!(free_after_expand, initial_free - (pages_before - 1));
// 2. Fast alle Inodes entfernen, sodass nur noch 1 Seite benötigt wird
{
let mut inner = cfs.inner.lock().unwrap();
for id in added_ids {
inner.manifest.inodes.remove(&id);
inner.index_remove_child(1, id);
}
inner.mark_dirty();
}
cfs.sync_manifest().expect("Sync after shrink");
// 3. Verifizieren: page_count ist wieder 1, freie Blöcke wurden zurückgegeben
let pages_after = cfs.page_count();
assert_eq!(
pages_after, 1,
"Nach Schrumpfung darf nur noch 1 Seite belegt sein"
);
let free_after_shrink = {
let inner = cfs.inner.lock().unwrap();
inner.manifest.free_blocks.len()
};
assert_eq!(
free_after_shrink, initial_free,
"Alle überzähligen Seitenblöcke müssen an free_blocks zurückgegeben worden sein"
);
drop(cfs);
// Trägerdatei neu laden und Integrität prüfen
let cfs_reloaded = load_carrier_fs(&env);
assert_eq!(cfs_reloaded.page_count(), 1);
assert_eq!(cfs_reloaded.corrupted_pages(), 0);
assert!(cfs_reloaded.resolve_path("/").is_some());
}