Merge pull request #490 from freeflowuniverse/development_ourdb_fixes

wip: fix ourdb

research/datastructures/ourdb/backend.v

@@ -1,116 +1,144 @@
 module ourdb
+
 import os
 import hash.crc32
 
-//this is the implementation of the lowlevel datastor, doesnt know about ACL or signature, it just stores a binary blob
+// this is the implementation of the lowlevel datastor, doesnt know about ACL or signature, it just stores a binary blob
 
-//the backend can be in 1 file or multiple files, if multiple files each file has a nr, we support max 65536 files in 1 dir
+// the backend can be in 1 file or multiple files, if multiple files each file has a nr, we support max 65536 files in 1 dir
 
 // calculate_crc computes CRC32 for the data
 fn calculate_crc(data []u8) u32 {
 	return crc32.sum(data)
 }
 
-fn (mut db OurDB) db_file_select(file_nr u16) !{
-	//open file for read/write
+fn (mut db OurDB) db_file_select(file_nr u16) ! {
+	// open file for read/write
 	// file is in "${db.path}/${nr}.db"
 	// return the file for read/write
 	if file_nr > 65535 {
-		return error("file_nr needs to be < 65536")
+		return error('file_nr needs to be < 65536')
+	}
+
+	path := '${db.path}/${file_nr}.db'
+
+	if db.file_nr == file_nr {
+		// make sure file is opened
+		if !db.file.is_opened {
+			mut file := os.open_file(path, 'r+') or {
+				// Create if doesn't exist with read/write permissions
+				mut f := os.create(path)!
+				f.write([u8(0)])! // to make all positions start from 1
+				f
+			}
+			db.file = file
+		}
+		return
+	}
+
+	if db.file.is_opened {
+		db.file.close()
 	}
 
-	path := "${db.path}/${file_nr}.db"
-
-	if db.file_nr != file_nr{
-		if db.file_nr >0{
-			db.file.close()		
-		}		
-		mut file2 := os.open_file(path, 'c+') or { 
-			// Create if doesn't exist with read/write permissions
-			os.create(path)!
-		}	
-		db.file=file2
-		db.file_nr = file_nr
+	mut file := os.open_file(path, 'r+') or {
+		// Create if doesn't exist with read/write permissions
+		mut f := os.create(path)!
+		f.write([u8(0)])! // to make all positions start from 1
+		f
 	}
+
+	db.file = file
+	db.file_nr = file_nr
 }
 
 // set stores data at position x
-pub fn (mut db OurDB) set_(location Location, data []u8) ! {
+pub fn (mut db OurDB) set_(x u32, old_location Location, data []u8) ! {
 	// Convert u64 to Location
-
-	db.db_file_select(location.file_nr)!
+	// TODO: can't file_nr change between two revisions?
+	db.db_file_select(old_location.file_nr)!
 
 	// Get current file position for lookup
 	db.file.seek(0, .end)!
-
-	// Get previous position if exists
-	prev_location := db.lookup.get(x_)!
-
+	new_location := Location{
+		file_nr: old_location.file_nr
+		position: u32(db.file.tell()!)
+	}
+
+	// // Get previous position if exists
+	// prev_location := db.lookup.get(x)!
+
 	// Calculate CRC of data
 	crc := calculate_crc(data)
-	
+
 	// Write size as u16 (2 bytes)
 	size := u16(data.len)
 	mut header := []u8{len: header_size, init: 0}
-	
+
 	// Write size (2 bytes)
 	header[0] = u8(size & 0xFF)
 	header[1] = u8((size >> 8) & 0xFF)
-	
+
 	// Write CRC (4 bytes)
 	header[2] = u8(crc & 0xFF)
 	header[3] = u8((crc >> 8) & 0xFF)
 	header[4] = u8((crc >> 16) & 0xFF)
 	header[5] = u8((crc >> 24) & 0xFF)
-	
+
 	// Convert previous location to bytes and store in header
-	prev_bytes := prev_location.to_bytes()!
+	prev_bytes := old_location.to_bytes()!
 	for i := 0; i < 6; i++ {
 		header[6 + i] = prev_bytes[i]
 	}
-	
+
 	// Write header
 	db.file.write(header)!
-	
+
 	// Write actual data
 	db.file.write(data)!
 
+	// TODO: how to update position?
 	// Update lookup table with new position
-	db.lookup.set(x_, location)!
+	db.lookup.set(x, new_location)!
 }
 
 // get retrieves data at specified location
 fn (mut db OurDB) get_(location Location) ![]u8 {
 	db.db_file_select(location.file_nr)!
 
+	// TODO: what if data really starts from position 0???
 	if location.position == 0 {
 		return error('Record not found')
 	}
-	
+
 	// Seek to position
 	db.file.seek(i64(location.position), .start)!
-	
+
 	// Read header
-	header := db.file.read_bytes(header_size)
-
+	mut header := []u8{len: header_size}
+	header_read_bytes := db.file.read(mut header)!
+	if header_read_bytes != header_size {
+		return error('failed to read header')
+	}
+
 	// Parse size (2 bytes)
 	size := u16(header[0]) | (u16(header[1]) << 8)
-	
+
 	// Parse CRC (4 bytes)
-	stored_crc := u32(header[2]) | 
-				  (u32(header[3]) << 8) |
-				  (u32(header[4]) << 16) |
-				  (u32(header[5]) << 24)
-
+	stored_crc := u32(header[2]) | (u32(header[3]) << 8) | (u32(header[4]) << 16) | (u32(header[5]) << 24)
+
 	// Read data
-	data := db.file.read_bytes(int(size))
-
+	mut data := []u8{len: int(size)}
+	data_read_bytes := db.file.read(mut data)!
+	if data_read_bytes != int(size) {
+		return error('failed to read data bytes')
+	}
+
 	// Verify CRC
 	calculated_crc := calculate_crc(data)
 	if calculated_crc != stored_crc {
 		return error('CRC mismatch: data corruption detected')
 	}
-	
+
 	return data
 }
 
@@ -119,89 +147,89 @@ fn (mut db OurDB) get_prev_pos_(location Location) !Location {
 	if location.position == 0 {
 		return error('Record not found')
 	}
-	
+
 	// Seek to position
 	db.file.seek(i64(location.position), .start)!
-	
+
 	// Skip size and CRC (6 bytes)
 	db.file.seek(i64(location.position + 6), .start)!
-	
+
 	// Read previous location (6 bytes)
 	prev_bytes := db.file.read_bytes(6)
 	return db.lookup.location_new(prev_bytes)!
 }
 
 // delete zeros out the record at specified location
-fn (mut db OurDB) delete_(location Location) ! {
+fn (mut db OurDB) delete_(x u32, location Location) ! {
 	if location.position == 0 {
 		return error('Record not found')
 	}
-	
+
 	// Seek to position
 	db.file.seek(i64(location.position), .start)!
-	
+
 	// Read size first
 	size_bytes := db.file.read_bytes(2)
 	size := u16(size_bytes[0]) | (u16(size_bytes[1]) << 8)
-	
+
 	// Write zeros for the entire record (header + data)
 	zeros := []u8{len: int(size) + header_size, init: 0}
 	db.file.seek(i64(location.position), .start)!
 	db.file.write(zeros)!
-	
+
 	// Clear lookup entry
-	db.lookup.delete(location)!
+	db.lookup.delete(x)!
 }
 
 // condense removes empty records and updates positions
 fn (mut db OurDB) condense() ! {
 	temp_path := db.path + '.temp'
 	mut temp_file := os.create(temp_path)!
-	
+
 	// Track current position in temp file
 	mut new_pos := Location{
 		file_nr: 0
 		position: 0
 	}
-	
+
 	// Iterate through lookup table
 	entry_size := int(db.lookup.keysize)
 	for i := 0; i < db.lookup.data.len / entry_size; i++ {
 		location := db.lookup.get(u32(i)) or { continue }
 		if location.position == 0 {
 			continue
 		}
-		
+
 		// Read record from original file
 		db.file.seek(i64(location.position), .start)!
 		header := db.file.read_bytes(header_size)
 		size := u16(header[0]) | (u16(header[1]) << 8)
-		
+
 		if size == 0 {
 			continue
 		}
-		
+
 		data := db.file.read_bytes(int(size))
-		
+
 		// Write to temp file
 		temp_file.write(header)!
 		temp_file.write(data)!
-		
+
 		// Update lookup with new position
 		db.lookup.set(u32(i), new_pos)!
-		
+
 		// Update position counter
 		new_pos.position += u32(size) + header_size
 	}
-	
+
 	// Close both files
 	temp_file.close()
 	db.file.close()
-	
+
 	// Replace original with temp
 	os.rm(db.path)!
 	os.mv(temp_path, db.path)!
-	
+
 	// Reopen the file
 	db.file = os.open_file(db.path, 'c+')!
 }

research/datastructures/ourdb/db.v

@@ -1,6 +1,6 @@
 module ourdb
+
 import os
-import hash.crc32
 
 // OurDB is a simple key-value database implementation that provides:
 // - Efficient key-value storage with history tracking
@@ -18,7 +18,7 @@ import hash.crc32
 // and maintains a linked list of previous values for history tracking
 pub fn (mut db OurDB) set(x u32, data []u8) ! {
 	location := db.lookup.get(x)! // Get location from lookup table
-	db.set_(location,data)!
+	db.set_(x, location, data)!
 }
 
 // get retrieves data stored at the specified key position
@@ -34,20 +34,20 @@ pub fn (mut db OurDB) get(x u32) ![]u8 {
 pub fn (mut db OurDB) get_history(x u32, depth u8) ![][]u8 {
 	mut result := [][]u8{}
 	mut current_location := db.lookup.get(x)! // Start with current location
-	
+
 	// Traverse the history chain up to specified depth
 	for i := u8(0); i < depth; i++ {
 		// Get current value
 		data := db.get_(current_location)!
 		result << data
-		
+
 		// Try to get previous location
 		current_location = db.get_prev_pos_(current_location) or { break }
 		if current_location.position == 0 {
 			break
 		}
 	}
-	
+
 	return result
 }
 
@@ -56,37 +56,33 @@ pub fn (mut db OurDB) get_history(x u32, depth u8) ![][]u8 {
 // Use condense() to reclaim space from deleted records (happens in step after)
 pub fn (mut db OurDB) delete(x u32) ! {
 	location := db.lookup.get(x)! // Get location from lookup table
-	db.delete_(location)!
+	db.delete_(x, location)!
 }
 
-
 // close closes the database file
 fn (mut db OurDB) lookup_dump_path() string {
-	return "${db.path}/lookup_dump.db"
+	return '${db.path}/lookup_dump.db'
 }
 
-//load metadata i exists
-fn (mut db OurDB) load() {	
-	if os.exists(db.lookup_dump_path()){
-		db.import_sparse(db.lookup_dump_path())!
+// load metadata i exists
+fn (mut db OurDB) load() ! {
+	if os.exists(db.lookup_dump_path()) {
+		db.lookup.import_sparse(db.lookup_dump_path())!
 	}
 }
 
-//make sure we have the metata stored on disk
-fn (mut db OurDB) save() {
-	//make sure we remember the data
-	db.export_sparse(db.lookup_dump_path())!
+// make sure we have the metata stored on disk
+fn (mut db OurDB) save() ! {
+	// make sure we remember the data
+	db.lookup.export_sparse(db.lookup_dump_path())!
 }
 
 // close closes the database file
-fn (mut db OurDB) close() {
-	db.save()
+fn (mut db OurDB) close() ! {
+	db.save()!
 	db.close_()
 }
 
-fn (mut db OurDB) destroy() {
-	panic("implement rm all self.path")
+fn (mut db OurDB) destroy() ! {
+	os.rmdir_all(db.path)!
 }
-
-
-