[cdb] Fix optional saving and dataFiles changes not being saved sometimes (#268)

hide/comp/cdb/DataFiles.hx

@@ -835,7 +835,12 @@ class DataFiles {
 		for( file => pf in prefabs ) {
 			skip++;
 			var path = ide.getPath(file);
+
+			var backup = [];
+			hide.view.Prefab.cleanupPrefabCdb(pf, backup);
 			@:privateAccess var out = ide.toJSON(pf.serialize());
+			hide.comp.cdb.Editor.restoreOptionals(backup);
+
 			if( force ) {
 				var txt = try sys.io.File.getContent(path) catch( e : Dynamic ) null;
 				if( txt == out ) continue;

hide/comp/cdb/Editor.hx

@@ -45,6 +45,8 @@ typedef EditorSheetProps = {
 	var ?categories : Array<String>;
 }
 
+typedef OptionalBackup = Array<{line: Dynamic, colName: String, data: Dynamic}>;
+
 @:allow(hide.comp.cdb)
 class Editor extends Component {
 	static var CLIPBOARD_PREFIX = "[CDB_FORMAT]";
@@ -865,7 +867,6 @@ class Editor extends Component {
 				var line = cursor.table.lines[y];
 				if(!cursor.table.lines[y].element.hasClass("filtered")) {
 					sheet.deleteLine(line.index);
-					cursor.table.refreshCellValue();
 					modifiedTables.pushUnique(cursor.table);
 				}
 				y--;
@@ -888,7 +889,6 @@ class Editor extends Component {
 					if( old == def )
 						continue;
 					changeObject(line,c,def);
-					cursor.table.refreshCellValue();
 					modifiedTables.pushUnique(cursor.table);
 				}
 			}
@@ -1059,6 +1059,49 @@ class Editor extends Component {
 		}
 	}
 
+	/**
+		Recursively removes empty list/properties from the optional columns of `sheet` in the given `lines` objects.
+		A record of the removed items are stored in `optionalBackup` so they can be restored later using `restoreOptionals()`.
+	**/
+	public static function cleanupOptionalLines(lines: Array<Dynamic>, sheet: cdb.Sheet, optionalBackup: OptionalBackup) {
+		if (lines == null)
+			return;
+		for (column in sheet.columns) {
+				for (line in lines) {
+					var data = Reflect.field(line, column.name);
+					if (data == null)
+						continue;
+					switch (column.type) {
+						case TList:
+							var list : Array<Dynamic> = cast data;
+							if (list.length == 0 && column.opt) {
+								optionalBackup.push({line: line, colName: column.name, data: list});
+								Reflect.deleteField(line, column.name);
+							} else {
+								var sub = sheet.getSub(column);
+								cleanupOptionalLines(list, sub, optionalBackup);
+							}
+						case TProperties:
+							var props : Dynamic = cast data;
+							if	(Reflect.fields(props).length == 0 && column.opt) {
+								optionalBackup.push({line: line, colName: column.name, data: props});
+								Reflect.deleteField(line, column.name);
+							} else {
+								var sub = sheet.getSub(column);
+								cleanupOptionalLines([props], sub, optionalBackup);
+							}
+						default:
+					}
+				}
+		}
+	}
+
+	public static function restoreOptionals(optionalBackup: OptionalBackup) {
+		for (backup in optionalBackup) {
+			Reflect.setField(backup.line, backup.colName, backup.data);
+		}
+	}
+
 	public static var inRefreshAll(default,null) : Bool;
 	public static function refreshAll( eraseUndo = false, loadDataFiles = true) {
 		var editors : Array<Editor> = [for( e in new Element(".is-cdb-editor").elements() ) e.data("cdb")];
@@ -1971,7 +2014,6 @@ class Editor extends Component {
 					if( table.displayMode == Properties ) {
 						beginChanges();
 						changeObject(cell.line, col, base.getDefault(col,sheet));
-						cursor.table.refreshCellValue();
 					} else {
 						beginChanges(true);
 						sheet.deleteColumn(col.name);
@@ -2041,7 +2083,6 @@ class Editor extends Component {
 					return;
 				beginChanges(true);
 				table.sheet.deleteColumn(col.name);
-				cursor.table.refreshCellValue();
 				endChanges();
 				refresh();
 			}});

hide/comp/cdb/SubTable.hx

@@ -67,28 +67,29 @@ class SubTable extends Table {
 		return cell.table.sheet.getSub(cell.column);
 	}
 
-	override function refreshCellValue() {
-		if (cell.column.opt == true) {
-			var doSet = true;
-			var value : Dynamic = null;
-			switch( cell.column.type ) {
-				case TList:
-					value = sheet.lines;
-					doSet = value != null && value.length > 0;
-				case TProperties:
-					value = sheet.lines[0];
-					doSet = value != null && Reflect.fields(value).length > 0;
-				default:
-					return;
-			}
-			if (doSet) {
-				Reflect.setField(cell.line.obj, cell.column.name, value);
-			} else {
-				Reflect.deleteField(cell.line.obj, cell.column.name);
-			}
+	override function refresh() {
+		super.refresh();
+
+		checkIntegrity();
+	}
+
+	function checkIntegrity() {
+		var v = Reflect.field(cell.line.obj, cell.column.name);
+		switch(cell.column.type) {
+			case TList:
+				if (v != sheet.lines) {
+					hide.Ide.inst.error("Editor integrity compromised, please refresh the editor and contact someone in the tool team");
+				}
+			case TProperties:
+				if (v != sheet.lines[0]) {
+					hide.Ide.inst.error("Editor integrity compromised, please refresh the editor and contact someone in the tool team");
+				}
+			default:
+		}
+		var parSub = Std.downcast(parent, SubTable);
+		if (parSub != null) {
+			parSub.checkIntegrity();
 		}
-		parent.refreshCellValue();
-		super.refreshCellValue();
 	}
 
 	function makeSubSheet() {
@@ -97,22 +98,19 @@ class SubTable extends Table {
 		var index = cell.line.index;
 		var key = sheet.getPath() + "@" + c.name + ":" + index;
 		var psheet = sheet.getSub(c);
+		var value : Dynamic = Reflect.field(cell.line.obj, cell.column.name);
 		var lines = switch( cell.column.type ) {
 		case TList:
-			var value = cell.value;
 			if( value == null ) {
 				value = [];
-				if (!cell.column.opt)
-					Reflect.setField(cell.line.obj, cell.column.name, value);
+				Reflect.setField(cell.line.obj, cell.column.name, value);
 				// do not save for now
 			}
 			value;
 		case TProperties:
-			var value = cell.value;
 			if( value == null ) {
 				value = {};
-				if (!cell.column.opt)
-					Reflect.setField(cell.line.obj, cell.column.name, value);
+				Reflect.setField(cell.line.obj, cell.column.name, value);
 				// do not save for now
 			}
 			var lines = [for( f in psheet.columns ) value];
@@ -156,12 +154,6 @@ class SubTable extends Table {
 	override function dispose() {
 		super.dispose();
 		insertedTR.remove();
-		var prev = Reflect.getProperty(cell.line.obj, cell.column.name);
-		refreshCellValue();
-		var after = Reflect.getProperty(cell.line.obj, cell.column.name);
-		if (prev != after) {
-			editor.save();
-		}
 		cell.refresh();
 	}
 

hide/comp/cdb/Table.hx

@@ -113,10 +113,6 @@ class Table extends Component {
 		}
 	}
 
-	public function refreshCellValue() {
-
-	}
-
 	function setupTableElement() {
 		cloneTableHead();
 		#if js
@@ -538,7 +534,6 @@ class Table extends Component {
 				var val = editor.base.getDefault(c, true, sheet);
 				editor.beginChanges();
 				Reflect.setField(props, c.name, val);
-				refreshCellValue();
 				editor.endChanges();
 				refresh();
 				for( l in lines )
@@ -645,7 +640,6 @@ class Table extends Component {
 		}
 		editor.beginChanges();
 		sheet.newLine(index);
-		refreshCellValue();
 		editor.endChanges();
 		refresh();
 	}
@@ -706,7 +700,6 @@ class Table extends Component {
 		var srcObj = sheet.lines[index];
 		editor.beginChanges();
 		var obj = sheet.newLine(index);
-		refreshCellValue();
 		for(colId => c in columns ) {
 			var val = Reflect.field(srcObj, c.name);
 			if( val != null ) {

hide/tools/IdeData.hx

@@ -221,7 +221,19 @@ class IdeData {
 					hide.comp.cdb.Editor.refreshAll();
 					return;
 				}
+
+				var backup = [];
+				for (sheet in database.sheets) {
+					// only perform cleanup on root sheets as the function is recursive
+					if (sheet.parent == null) {
+						hide.comp.cdb.Editor.cleanupOptionalLines(sheet.lines, sheet, backup);
+					}
+				}
+
 				lastDBContent = database.save();
+
+				hide.comp.cdb.Editor.restoreOptionals(backup);
+
 				sys.io.File.saveContent(getPath(databaseFile), lastDBContent);
 				if ( dbWatcher != null )
 					fileWatcher.ignorePrevChange(dbWatcher);

hide/ui/View.hx

@@ -22,7 +22,7 @@ class View<T> extends hide.comp.Component {
 	var watches : Array<{ keep : Bool, path : String, callb : Void -> Void }> = [];
 	public var keys(get,null) : Keys;
 	public var state(default, null) : T;
-	public var undo(get, null) = new hide.ui.UndoHistory();
+	public var undo(get, never) : hide.ui.UndoHistory;
 
 	function get_undo() : hide.ui.UndoHistory {
 		return undoStack[undoStack.length-1];

hide/view/CdbTable.hx

@@ -19,7 +19,7 @@ class CdbTable extends hide.ui.View<{}> {
 				haxe.Timer.delay(syncTabs,0);
 			}
 		}, this);
-		undo = editor.undo;
+		undoStack[0] = editor.undo;
 		currentSheet = this.config.get("cdb.currentSheet");
 		view = cast this.config.get("cdb.view");
 	}

hide/view/Prefab.hx

@@ -433,7 +433,7 @@ class Prefab extends hide.view.FileView {
 	}
 
 	public function onSceneReady() {
-		data = hxd.res.Loader.currentInstance.load(state.path).toPrefab().load().clone();
+		data = hxd.res.Loader.currentInstance.load(state.path).toPrefab().loadBypassCache();
 		sceneEditor.setPrefab(cast data);
 
 		refreshSceneFilters();
@@ -598,6 +598,22 @@ class Prefab extends hide.view.FileView {
 		return data != null;
 	}
 
+	public static function cleanupPrefabCdb(prefab: hrt.prefab.Prefab, optionalBackup: hide.comp.cdb.Editor.OptionalBackup) {
+		var cdbType = prefab.getCdbType();
+
+		if (cdbType != null) {
+			if (prefab.name == "npc")
+				trace("break");
+			var db = hide.Ide.inst.database;
+			var sheet = db.getSheet(cdbType);
+			hide.comp.cdb.Editor.cleanupOptionalLines([prefab.props], sheet, optionalBackup);
+		}
+
+		for (child in prefab.children) {
+			cleanupPrefabCdb(child, optionalBackup);
+		}
+	}
+
 	override function save() {
 		if( !canSave() )
 			return;
@@ -606,7 +622,13 @@ class Prefab extends hide.view.FileView {
 		if (Ide.inst.currentConfig.get("sceneeditor.renderprops.edit", false) && sceneEditor.renderPropsRoot != null)
 			sceneEditor.renderPropsRoot.save();
 
+		var backup = [];
+		cleanupPrefabCdb(data, backup);
+
 		@:privateAccess var content = ide.toJSON(data.serialize());
+
+		hide.comp.cdb.Editor.restoreOptionals(backup);
+
 		var newSign = ide.makeSignature(content);
 		if(newSign != currentSign)
 			haxe.Timer.delay(saveBackup.bind(content), 0);

hrt/prefab/Resource.hx

@@ -31,11 +31,16 @@ class Resource extends hxd.res.Resource {
 		return isBSON ? new hxd.fmt.hbson.Reader(entry.getBytes(),false).read() : haxe.Json.parse(entry.getText());
 	}
 
+	public function loadBypassCache() {
+		var data = loadData();
+		var prefab = Prefab.createFromDynamic(data, new ContextShared(entry.path, false));
+		return cast prefab;
+	}
+
 	public function load() : Prefab {
-		if( prefab != null && cacheVersion == CACHE_VERSION )
+		if(prefab != null && cacheVersion == CACHE_VERSION )
 			return prefab;
-		var data = loadData();
-		prefab = Prefab.createFromDynamic(data, new ContextShared(entry.path, false));
+		prefab = loadBypassCache();
 		cacheVersion = CACHE_VERSION;
 		onPrefabLoaded(prefab);
 		watch(function() {}); // auto lib reload