Improve loading performance

plugins/com.minres.scviewer.database.sqlite/src/com/minres/scviewer/database/sqlite/SQLiteDbLoader.java

@@ -75,6 +75,7 @@ public class SQLiteDbLoader implements IWaveformDbLoader {
 
 	@Override
 	public boolean load(IWaveformDb db, File file) throws InputFormatException {
+		dispose();
 		if(file.isDirectory() || !file.exists()) return false;
 		try(FileInputStream fis = new FileInputStream(file)) {
 			byte[] buffer = new byte[x.length];
@@ -101,6 +102,11 @@ public class SQLiteDbLoader implements IWaveformDbLoader {
 		return false;
 	}
 
+	public void dispose() {
+		database=null;
+		usedRelationsList=null;
+	}
+
 	@Override
 	public Collection<RelationType> getAllRelationTypes(){
 		return usedRelationsList;

plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TextDbLoader.java

@@ -20,11 +20,17 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 
+import org.mapdb.DB;
+import org.mapdb.DB.TreeMapSink;
+import org.mapdb.DBMaker;
+import org.mapdb.Serializer;
+
 import com.google.common.collect.HashMultimap;
 import com.minres.scviewer.database.AssociationType;
 import com.minres.scviewer.database.DataType;
@@ -41,19 +47,25 @@ public class TextDbLoader implements IWaveformDbLoader{
 
 	private Long maxTime=0L;
 
-	Map<String, RelationType> relationTypes = null;
+	DB mapDb=null;
 
-    Map<Long, TxStream> txStreams = null;
+	final Map<String, RelationType> relationTypes=new HashMap<>();
     
-    Map<Long, TxGenerator> txGenerators = null;
+    final Map<Long, TxStream> txStreams = new HashMap<>();
+    
+    final Map<Long, TxGenerator> txGenerators = new HashMap<>();
     
     Map<Long, ScvTx> transactions = null;
 
-    Map<String, TxAttributeType> attributeTypes = null; 
+    final Map<String, TxAttributeType> attributeTypes = new HashMap<>(); 
 
-	HashMultimap<Long, ScvRelation> relationsIn = null;
+	final HashMultimap<Long, ScvRelation> relationsIn = HashMultimap.create();
 
-	HashMultimap<Long, ScvRelation> relationsOut = null;
+	final HashMultimap<Long, ScvRelation> relationsOut = HashMultimap.create();
+
+    HashMap<Long, Tx> txCache = new HashMap<>();
+
+    List<Thread> threads = new ArrayList<>();
     
     @Override
 	public Long getMaxTime() {
@@ -70,6 +82,7 @@ public class TextDbLoader implements IWaveformDbLoader{
 	@SuppressWarnings("unchecked")
 	@Override
 	public	boolean load(IWaveformDb db, File file) throws InputFormatException {
+		dispose();
 		if(file.isDirectory() || !file.exists()) return false;
 		TextDbParser parser = new TextDbParser(this);
 		boolean gzipped = isGzipped(file);
@@ -79,24 +92,71 @@ public class TextDbLoader implements IWaveformDbLoader{
 		} catch(Exception e) {
 			throw new InputFormatException();
 		}
-		relationTypes=new HashMap<String, RelationType>();
+		
-	    txStreams = new HashMap<>();
+		if(file.length() < 75000000*(gzipped?1:10) || "memory".equals(System.getProperty("ScvBackingDB", "file")))
-	    txGenerators = new HashMap<>();
+			mapDb = DBMaker
-	    transactions = new HashMap<>();
+			.memoryDirectDB()
-	    attributeTypes = new HashMap<>(); 
+			.allocateStartSize(512*1024*1024)
-		relationsIn = HashMultimap.create();
+			.allocateIncrement(128*1024*1024)
-		relationsOut = HashMultimap.create();
+			.cleanerHackEnable()
+			.make();
+		else {
+			File mapDbFile;
+			try {
+				mapDbFile = File.createTempFile("."+file.getName(), ".mapdb", null /*file.parentFile*/);
+			} catch (IOException e1) {
+				return false;
+			}
+			mapDbFile.delete(); // we just need a file name
+			mapDb = DBMaker
+			.fileDB(mapDbFile)
+			.fileMmapEnable()            // Always enable mmap
+			.fileMmapEnableIfSupported()
+			.fileMmapPreclearDisable()
+			.allocateStartSize(512*1024*1024)
+			.allocateIncrement(128*1024*1024)
+			.cleanerHackEnable()
+			.make();
+			mapDbFile.deleteOnExit();
+		}
 		try {
-		      parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file));
+			parser.txSink = mapDb.treeMap("transactions", Serializer.LONG,Serializer.JAVA).createFromSink();
+			parser.parseInput(gzipped?new GZIPInputStream(new FileInputStream(file)):new FileInputStream(file));
+			transactions = parser.txSink.create();
 		} catch(IllegalArgumentException|ArrayIndexOutOfBoundsException e) {
 		} catch(Exception e) {
 			System.out.println("---->>> Exception "+e.toString()+" caught while loading database");
 			e.printStackTrace();
 			return false;
 		}
+		for(TxStream stream:txStreams.values()) {
+			Thread t = new Thread() {
+				public void run() {
+					try {
+					stream.calculateConcurrency();
+					} catch (Exception e) {/* don't let exceptions bubble up */ }
+				}
+			};
+			threads.add(t);
+			t.start();
+		}
 		return true;
 	}
 
+	public void dispose() {
+		relationTypes.clear();
+	    txStreams.clear();
+	    txGenerators.clear();
+	    transactions = null;
+	    attributeTypes.clear(); 
+		relationsIn.clear();
+		relationsOut.clear();
+		if(mapDb!=null) {
+			mapDb.close();
+			mapDb=null;
+		}
+	}
+	
 	private static boolean isTxfile(InputStream istream) {
 		byte[] buffer = new byte[x.length];
 		try {
@@ -134,6 +194,10 @@ public class TextDbLoader implements IWaveformDbLoader{
 				
 		final TextDbLoader loader;
 		
+		HashMap<Long, ScvTx> transactionById = new HashMap<>();
+		
+		TreeMapSink<Long, ScvTx> txSink;
+		
 		BufferedReader reader = null;
 		
 		TxGenerator generator=null;
@@ -174,7 +238,7 @@ public class TextDbLoader implements IWaveformDbLoader{
 				DataType type = DataType.valueOf(tokens[3]);
 				String remaining = tokens.length>5?String.join(" ", Arrays.copyOfRange(tokens, 5, tokens.length)):"";
 				TxAttributeType attrType = getAttrType(name, type, AssociationType.RECORD);
-				loader.transactions.get(id).attributes.add(new TxAttribute(attrType, remaining));
+				transactionById.get(id).attributes.add(new TxAttribute(attrType, remaining));
 			} else if("tx_begin".equals(tokens[0])){
 				Long id = Long.parseLong(tokens[1]);
 				Long genId = Long.parseLong(tokens[2]);
@@ -189,26 +253,28 @@ public class TextDbLoader implements IWaveformDbLoader{
 					while(nextLine!=null && nextLine.charAt(0)=='a') {
 						String[] attrTokens=nextLine.split("\\s+");
 						TxAttribute attr = new TxAttribute(gen.beginAttrs.get(idx), attrTokens[1]);
-						tx.getAttributes().add(attr);
+						scvTx.attributes.add(attr);
 						idx++;
 						nextLine=reader.readLine();
 					}
 				}
-				loader.transactions.put(id, scvTx);
+				txSink.put(id, scvTx);
+				transactionById.put(id, scvTx);
 				gen.getTransactions().add(tx);
 			} else if("tx_end".equals(tokens[0])){
 				Long id = Long.parseLong(tokens[1]);
-				ScvTx tx = loader.transactions.get(id);
+				//ScvTx tx = loader.transactions.get(id);
-				assert Long.parseLong(tokens[2])==tx.generatorId;
+				ScvTx scvTx = transactionById.get(id);
-				tx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]);
+				assert Long.parseLong(tokens[2])==scvTx.generatorId;
-				loader.maxTime = loader.maxTime>tx.endTime?loader.maxTime:tx.endTime;
+				scvTx.endTime=Long.parseLong(tokens[3])*stringToScale(tokens[4]);
-				TxGenerator gen = loader.txGenerators.get(tx.generatorId);
+				loader.maxTime = loader.maxTime>scvTx.endTime?loader.maxTime:scvTx.endTime;
+				TxGenerator gen = loader.txGenerators.get(scvTx.generatorId);
 				TxStream stream = loader.txStreams.get(gen.stream.getId());
-				if(tx.beginTime==tx.endTime)
+				if(scvTx.beginTime==scvTx.endTime)
-					stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, tx.beginTime));
+					stream.addEvent(new TxEvent(loader, EventKind.SINGLE, id, scvTx.beginTime));
 				else {
-					stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, tx.beginTime));
+					stream.addEvent(new TxEvent(loader, EventKind.BEGIN, id, scvTx.beginTime));
-					stream.addEvent(new TxEvent(loader, EventKind.END, id, tx.endTime));
+					stream.addEvent(new TxEvent(loader, EventKind.END, id, scvTx.endTime));
 				}
 				stream.setConcurrency(stream.getConcurrency()-1);
 				if(nextLine!=null && nextLine.charAt(0)=='a') {
@@ -216,7 +282,7 @@ public class TextDbLoader implements IWaveformDbLoader{
 					while(nextLine!=null && nextLine.charAt(0)=='a') {
 						String[] attrTokens=nextLine.split("\\s+");
 						TxAttribute attr = new TxAttribute(gen.endAttrs.get(idx), attrTokens[1]);
-						tx.attributes.add(attr);
+						scvTx.attributes.add(attr);
 						idx++;
 						nextLine=reader.readLine();
 					}
@@ -278,8 +344,12 @@ public class TextDbLoader implements IWaveformDbLoader{
 		}
 	}
 
-	public ITx getTransaction(long source) {
+	public ITx getTransaction(long txId) {
-		return new Tx(this, transactions.get(source));
+		if(txCache.containsKey(txId))
+			return txCache.get(txId);
+		Tx tx = new Tx(this, txId);
+		txCache.put(txId, tx);
+		return tx;
 	}
 
 }

plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/Tx.java

@@ -12,7 +12,6 @@ package com.minres.scviewer.database.text;
 
 import java.util.Collection;
 import java.util.List;
-import java.util.NavigableMap;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -26,24 +25,33 @@ class Tx implements ITx {
 	
 	private final TextDbLoader loader;
 	
-	private ScvTx scvTx = null;
+	private long id;
+		
+	long beginTime=-1;
+	
+	long endTime=-1;
 	
 	private int concurrencyIndex;
 				
 	public Tx(TextDbLoader loader, ScvTx scvTx) {
 		this.loader=loader;
-		this.scvTx = scvTx;
+		id=scvTx.id;
+	}
+
+	public Tx(TextDbLoader loader, long txId) {
+		this.loader=loader;
+		id=txId;
 	}
 
 	@Override
 	public Collection<ITxRelation> getIncomingRelations() {
-		Set<ScvRelation> rels = loader.relationsIn.get(scvTx.getId());
+		Set<ScvRelation> rels = loader.relationsIn.get(id);
 		return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList());
 	}
 
 	@Override
 	public Collection<ITxRelation> getOutgoingRelations() {
-		Set<ScvRelation> rels = loader.relationsOut.get(scvTx.getId());
+		Set<ScvRelation> rels = loader.relationsOut.get(id);
 		return rels.stream().map(rel -> new TxRelation(loader, rel)).collect(Collectors.toList());
 	}
 
@@ -60,12 +68,12 @@ class Tx implements ITx {
     public boolean equals(Object obj) {
         if (this == obj) return true;
         if (obj == null || getClass() != obj.getClass()) return false;
-        return this.scvTx.equals(((Tx) obj).scvTx);
+        return this.getScvTx().equals(((Tx) obj).getScvTx());
     }
 
 	@Override
     public int hashCode() {
-		return scvTx.hashCode();
+		return getScvTx().hashCode();
 	}
 
 	@Override
@@ -75,31 +83,33 @@ class Tx implements ITx {
 
 	@Override
 	public Long getId() {
-		return scvTx.id;
+		return getScvTx().id;
 	}
 
 	@Override
 	public IWaveform getStream() {
-		return loader.txStreams.get(scvTx.streamId);
+		return loader.txStreams.get(getScvTx().streamId);
 	}
 
 	@Override
 	public ITxGenerator getGenerator() {
-		return loader.txGenerators.get(scvTx.generatorId);
+		return loader.txGenerators.get(getScvTx().generatorId);
 	}
 
 	@Override
 	public Long getBeginTime() {
-		return scvTx.beginTime;
+		if(beginTime<0)	beginTime=getScvTx().beginTime;
+		return beginTime;
 	}
 
 	@Override
 	public Long getEndTime() {
-		return scvTx.endTime;
+		if(endTime<0) endTime=getScvTx().endTime;
+		return endTime;
 	}
 
 	void setEndTime(Long time) {
-		 scvTx.endTime=time;
+		 getScvTx().endTime=time;
 	}
 
 	@Override
@@ -107,13 +117,16 @@ class Tx implements ITx {
 		return concurrencyIndex;
 	}
 
-	public void setConcurrencyIndex(int idx) {
+	void setConcurrencyIndex(int idx) {
 		concurrencyIndex=idx;
 	}
 
 	@Override
 	public List<ITxAttribute> getAttributes() {
-		return scvTx.attributes;
+		return getScvTx().attributes;
 	}
 
+	private ScvTx getScvTx() {
+		return loader.transactions.get(id);
+	}
 }

plugins/com.minres.scviewer.database.text/src/com/minres/scviewer/database/text/TxStream.java

@@ -111,7 +111,8 @@ class TxStream extends HierNode implements IWaveform {
 		return id;
 	}
 
-	private void calculateConcurrency() {
+	synchronized void calculateConcurrency() {
+		if(concurrencyCalculated) return;
 		ArrayList<Long> rowendtime = new ArrayList<>();
 		events.entrySet().stream().forEach( entry -> {
 			IEvent[] values = entry.getValue();

plugins/com.minres.scviewer.database.vcd/src/com/minres/scviewer/database/vcd/VCDDbLoader.java

@@ -68,6 +68,7 @@ public class VCDDbLoader implements IWaveformDbLoader, IVCDDatabaseBuilder {
 	@SuppressWarnings("unchecked")
 	@Override
 	public boolean load(IWaveformDb db, File file) throws InputFormatException {
+		dispose();
 		if(file.isDirectory() || !file.exists()) return false;
 		this.maxTime=0;
 		boolean res = false;
@@ -110,6 +111,11 @@ public class VCDDbLoader implements IWaveformDbLoader, IVCDDatabaseBuilder {
 		return true;
 	}
 
+	public void dispose() {
+		moduleStack=null;
+		signals=null;
+	}
+
 	/* (non-Javadoc)
 	 * @see com.minres.scviewer.database.ITrDb#getMaxTime()
 	 */

plugins/com.minres.scviewer.database/src/com/minres/scviewer/database/IWaveformDbLoader.java

@@ -23,4 +23,5 @@ public interface IWaveformDbLoader {
 	
 	public Collection<RelationType> getAllRelationTypes() ;
 	
+	public void dispose();
 }

plugins/com.minres.scviewer.e4.application/src/com/minres/scviewer/e4/application/parts/TransactionList.java

@@ -294,7 +294,7 @@ public class TransactionList extends Composite {
 						}
 					});
 				}
-			}.run();
+			}.start();
 		}
 	}